diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 728a27d322..0000000000 --- a/.coveragerc +++ /dev/null @@ -1,9 +0,0 @@ -[run] -omit = - zarr/meta_v1.py - bench/compress_normal.py - -[report] -exclude_lines = - pragma: no cover - pragma: ${PY_MAJOR_VERSION} no cover diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..9e0316032f --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,4 @@ +# lint codebase with black and ruff +4e348d6b80c96da461fd866576c971b8a659ba15 +# migrate from black to ruff format +22cea005629913208a85799372e045f353744add diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 0000000000..8fb235d704 --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,4 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ +ref-names: $Format:%D$ diff --git a/.gitattributes b/.gitattributes index b6115e6e49..57eb8a8807 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ *.py linguist-language=python *.ipynb linguist-documentation +.git_archival.txt export-subst diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5f574f4f59..9503f3df8a 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,5 +1,4 @@ Contributing ============ -Please see the [project documentation](http://zarr.readthedocs.io/en/stable/contributing.html) for information about contributing to Zarr. - +Please see the [project documentation](https://zarr.readthedocs.io/en/stable/developers/contributing.html) for information about contributing to Zarr. diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 9f8b2348cd..0000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,28 +0,0 @@ -For bug reports, please follow the template below. For enhancement proposals, feel free -to use whatever template makes sense (major new features should be discussed in the -Zarr specifications repository https://github.com/zarr-developers/zarr-specs). - -#### Minimal, reproducible code sample, a copy-pastable example if possible - -```python -# Your code here - -``` - -#### Problem description - -Explain why the current behavior is a problem, what the expected output/behaviour -is, and why the expected output/behaviour is a better solution. - -#### Version and installation information - -Please provide the following: - -* Value of ``zarr.__version__`` -* Value of ``numcodecs.__version__`` -* Version of Python interpreter -* Operating system (Linux/Windows/Mac) -* How Zarr was installed (e.g., "using pip into virtual environment", or "using conda") - -Also, if you think it might be relevant, please provide the output from ``pip freeze`` or -``conda env export`` depending on which was used to install Zarr. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000000..84bb89d82a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,82 @@ +name: Bug Report +description: Report incorrect behaviour in the library. +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Please provide the following information. + - type: input + id: Zarr-version + attributes: + label: Zarr version + description: Value of ``zarr.__version__`` + placeholder: v2.10.2, v2.11.3, v2.12.0, etc. + validations: + required: true + - type: input + id: Numcodecs-version + attributes: + label: Numcodecs version + description: Value of ``numcodecs.__version__`` + placeholder: v0.8.1, v0.9.0, v0.10.0, etc. + validations: + required: true + - type: input + id: Python-version + attributes: + label: Python Version + description: Version of Python interpreter + placeholder: 3.10, 3.11, 3.12 etc. + validations: + required: true + - type: input + id: OS + attributes: + label: Operating System + description: Operating System + placeholder: (Linux/Windows/Mac) + validations: + required: true + - type: input + id: installation + attributes: + label: Installation + description: How was Zarr installed? + placeholder: e.g., "using pip into virtual environment", or "using conda" + validations: + required: true + - type: textarea + id: description + attributes: + label: Description + description: Explain why the current behavior is a problem, what the expected output/behaviour is, and why the expected output/behaviour is a better solution. + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: Minimal, reproducible code sample. Must list dependencies in [inline script metadata](https://packaging.python.org/en/latest/specifications/inline-script-metadata/#example). When put in a file named `issue.py` calling `uv run issue.py` should show the issue. + value: | + ```python + # /// script + # requires-python = ">=3.11" + # dependencies = [ + # "zarr@git+https://github.com/zarr-developers/zarr-python.git@main", + # ] + # /// + # + # This script automatically imports the development branch of zarr to check for issues + + import zarr + # your reproducer code + # zarr.print_debug_info() + ``` + validations: + required: true + - type: textarea + id: additional-output + attributes: + label: Additional output + description: If you think it might be relevant, please provide the output from ``pip freeze`` or ``conda env export`` depending on which was used to install Zarr. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..27239f5861 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: true +contact_links: + - name: Propose a new major feature + url: https://github.com/zarr-developers/zarr-specs + about: A new major feature should be discussed in the Zarr specifications repository. + - name: Discuss something on ZulipChat + url: https://ossci.zulipchat.com/ + about: For questions like "How do I do X with Zarr?", consider posting your question to our developer chat. + - name: Discuss something on GitHub Discussions + url: https://github.com/zarr-developers/zarr-python/discussions + about: For questions like "How do I do X with Zarr?", you can move to GitHub Discussions. diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml new file mode 100644 index 0000000000..c240666d1f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -0,0 +1,18 @@ +name: Documentation Improvement +description: Report missing or wrong documentation. Alternatively, you can just open a pull request with the suggested change. +title: "DOC: " +labels: [documentation, help wanted] + +body: +- type: textarea + attributes: + label: Describe the issue linked to the documentation + description: > + Please provide a description of what documentation you believe needs to be fixed/improved. + validations: + required: true +- type: textarea + attributes: + label: Suggested fix for documentation + description: > + Please explain the suggested fix and why it's better than the existing documentation. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a0d41f9841..9b64c97d0a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,7 +3,7 @@ TODO: * [ ] Add unit tests and/or doctests in docstrings * [ ] Add docstrings and API docs for any new/modified user-facing classes and functions -* [ ] New/modified features documented in docs/tutorial.rst -* [ ] Changes documented in docs/release.rst +* [ ] New/modified features documented in `docs/user-guide/*.rst` +* [ ] Changes documented as a new file in `changes/` * [ ] GitHub Actions have all passed * [ ] Test coverage is 100% (Codecov passes) diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000000..e9b99c8214 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,15 @@ +coverage: + status: + project: + default: + target: 100 + threshold: 0.1 + patch: + default: + target: 100 +comment: + layout: "diff, files" + behavior: default + require_changes: true # if true: only post the comment if coverage changes + branches: # branch names that can post comment + - "main" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2db1489656..469b6a4d19 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,32 @@ --- version: 2 updates: + # Updates for main + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + actions: + patterns: + - "*" + + # Updates for support/v2 branch - package-ecosystem: "pip" directory: "/" + target-branch: "support/v2" + schedule: + interval: "weekly" + groups: + requirements: + patterns: + - "*" + - package-ecosystem: "github-actions" + directory: "/" + target-branch: "support/v2" schedule: - interval: "daily" + interval: "weekly" + groups: + actions: + patterns: + - "*" diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000000..ede89c9d35 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,4 @@ +needs release notes: + - all: + - changed-files: + - all-globs-to-all-files: '!changes/*.rst' diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml new file mode 100644 index 0000000000..752440719b --- /dev/null +++ b/.github/workflows/gpu_test.yml @@ -0,0 +1,72 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: GPU Test + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +env: + LD_LIBRARY_PATH: /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }} + + runs-on: gpu-runner + strategy: + matrix: + python-version: ['3.11'] + numpy-version: ['2.2'] + dependency-set: ["minimal"] + + steps: + - uses: actions/checkout@v4 + # - name: cuda-toolkit + # uses: Jimver/cuda-toolkit@v0.2.16 + # id: cuda-toolkit + # with: + # cuda: '12.4.1' + - name: Set up CUDA + run: | + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb + sudo dpkg -i cuda-keyring_1.1-1_all.deb + sudo apt-get update + sudo apt-get -y install cuda-toolkit-12-6 + echo "/usr/local/cuda/bin" >> $GITHUB_PATH + - name: GPU check + run: | + nvidia-smi + echo $PATH + echo $LD_LIBRARY_PATH + nvcc -V + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install Hatch and CuPy + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Set Up Hatch Env + run: | + hatch env create gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} + hatch env run -e gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env + - name: Run Tests + run: | + hatch env run --env gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage + + - name: Upload coverage + uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml new file mode 100644 index 0000000000..776f859d6e --- /dev/null +++ b/.github/workflows/hypothesis.yaml @@ -0,0 +1,96 @@ +name: Slow Hypothesis CI +on: + push: + branches: + - "main" + pull_request: + branches: + - "main" + types: [opened, reopened, synchronize, labeled] + schedule: + - cron: "0 0 * * *" # Daily “At 00:00” UTC + workflow_dispatch: # allows you to trigger manually + +env: + FORCE_COLOR: 3 + +jobs: + + hypothesis: + name: Slow Hypothesis Tests + runs-on: "ubuntu-latest" + defaults: + run: + shell: bash -l {0} + + strategy: + matrix: + python-version: ['3.12'] + numpy-version: ['2.2'] + dependency-set: ["optional"] + + steps: + - uses: actions/checkout@v4 + - name: Set HYPOTHESIS_PROFILE based on trigger + run: | + if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "HYPOTHESIS_PROFILE=nightly" >> $GITHUB_ENV + else + echo "HYPOTHESIS_PROFILE=ci" >> $GITHUB_ENV + fi + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install Hatch + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Set Up Hatch Env + run: | + hatch env create test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} + hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env + # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache + - name: Restore cached hypothesis directory + id: restore-hypothesis-cache + uses: actions/cache/restore@v4 + with: + path: .hypothesis/ + key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} + restore-keys: | + cache-hypothesis- + + - name: Run slow Hypothesis tests + if: success() + id: status + run: | + echo "Using Hypothesis profile: $HYPOTHESIS_PROFILE" + hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-hypothesis + + # explicitly save the cache so it gets updated, also do this even if it fails. + - name: Save cached hypothesis directory + id: save-hypothesis-cache + if: always() && steps.status.outcome != 'skipped' + uses: actions/cache/save@v4 + with: + path: .hypothesis/ + key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} + + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) + + - name: Generate and publish the report + if: | + failure() + && steps.status.outcome == 'failure' + && github.event_name == 'schedule' + && github.repository_owner == 'zarr-developers' + uses: xarray-contrib/issue-from-pytest-log@v1 + with: + log-path: output-${{ matrix.python-version }}-log.jsonl + issue-title: "Nightly Hypothesis tests failed" + issue-label: "topic-hypothesis" diff --git a/.github/workflows/issue-metrics.yml b/.github/workflows/issue-metrics.yml new file mode 100644 index 0000000000..34bda59ff6 --- /dev/null +++ b/.github/workflows/issue-metrics.yml @@ -0,0 +1,42 @@ +name: Monthly issue metrics +on: + workflow_dispatch: + schedule: + - cron: '3 2 1 * *' + +permissions: + contents: read + +jobs: + build: + name: issue metrics + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: read + steps: + - name: Get dates for last month + shell: bash + run: | + # Calculate the first day of the previous month + first_day=$(date -d "last month" +%Y-%m-01) + + # Calculate the last day of the previous month + last_day=$(date -d "$first_day +1 month -1 day" +%Y-%m-%d) + + #Set an environment variable with the date range + echo "$first_day..$last_day" + echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV" + + - name: Run issue-metrics tool + uses: github/issue-metrics@v3 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SEARCH_QUERY: 'repo:zarr-developers/zarr-python is:issue created:${{ env.last_month }} -reason:"not planned"' + + - name: Create issue + uses: peter-evans/create-issue-from-file@v5 + with: + title: Monthly issue metrics report + token: ${{ secrets.GITHUB_TOKEN }} + content-filepath: ./issue_metrics.md diff --git a/.github/workflows/needs_release_notes.yml b/.github/workflows/needs_release_notes.yml new file mode 100644 index 0000000000..7a6c5462b4 --- /dev/null +++ b/.github/workflows/needs_release_notes.yml @@ -0,0 +1,17 @@ +name: "Pull Request Labeler" + +on: + - pull_request_target + +jobs: + labeler: + if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' }} && ${{ github.event.pull_request.user.login != 'pre-commit-ci[bot]' }} + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + sync-labels: true diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index d9bc362d12..0000000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,89 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Linux Testing - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.7, 3.8] - numpy_version: ['!=1.21.0', '==1.17.*'] - services: - redis: - image: redis - # Set health checks to wait until redis has started - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 - mongodb: - image: mongo:3.4.23 - ports: - - 27017:27017 - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@master - with: - channels: conda-forge - python-version: ${{ matrix.python-version }} - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: true - # Runs a set of commands using the runners shell - - name: Create Conda environment with the rights deps - shell: "bash -l {0}" - run: | - conda create -n zarr-env python==${{matrix.python-version}} bsddb3 numcodecs==0.6.4 lmdb pip nodejs flake8 mypy - conda activate zarr-env - npm install -g azurite - - name: Install dependencies - shell: "bash -l {0}" - run: | - conda activate zarr-env - python -m pip install --upgrade pip - python -m pip install -U pip setuptools wheel codecov line_profiler - python -m pip install -rrequirements_dev_minimal.txt numpy${{ matrix.numpy_version}} -rrequirements_dev_optional.txt pymongo redis - python -m pip install -e . - python -m pip freeze - - name: Tests - shell: "bash -l {0}" - env: - COVERAGE_FILE: .coverage.${{matrix.python-version}}.${{matrix.numpy_version}} - ZARR_TEST_ABS: 1 - ZARR_TEST_MONGO: 1 - ZARR_TEST_REDIS: 1 - run: | - conda activate zarr-env - mkdir ~/blob_emulator - azurite -l ~/blob_emulator --debug debug.log 2>&1 > stdouterr.log & - pytest --cov=zarr --cov-config=.coveragerc --doctest-plus --cov-report xml --cov=./ - - uses: codecov/codecov-action@v1 - with: - #token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - #files: ./coverage1.xml,./coverage2.xml # optional - #flags: unittests # optional - #name: codecov-umbrella # optional - #fail_ci_if_error: true # optional (default = false) - verbose: true # optional (default = false) - - name: Linting - shell: "bash -l {0}" - run: | - conda activate zarr-env - flake8 zarr - mypy zarr - - diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index d97320ff8a..c8903aa779 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -11,31 +11,23 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 with: submodules: true + fetch-depth: 0 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5.2.0 name: Install Python with: - python-version: '3.8' + python-version: '3.11' - name: Install PyBuild run: | - python -m pip install 'build!=0.1' setuptools-scm - + python -m pip install --upgrade pip + pip install hatch - name: Build wheel and sdist - run: | - python -m build - git describe - pwd - if [ -f dist/zarr-0.0.0.tar.gz ]; then - echo "WRONG VERSION NUMBER" - exit 1 - else - echo "All seem good" - fi - - uses: actions/upload-artifact@v1 + run: hatch build + - uses: actions/upload-artifact@v4 with: name: releases path: dist @@ -44,7 +36,7 @@ jobs: needs: [build_artifacts] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v1 + - uses: actions/download-artifact@v4 with: name: releases path: dist @@ -59,11 +51,11 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') steps: - - uses: actions/download-artifact@v1 + - uses: actions/download-artifact@v4 with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@master + - uses: pypa/gh-action-pypi-publish@v1.12.4 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..7cfce41312 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,156 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Test + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: os=${{ matrix.os }}, py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }} + + strategy: + matrix: + python-version: ['3.11', '3.12', '3.13'] + numpy-version: ['1.25', '2.2'] + dependency-set: ["minimal", "optional"] + os: ["ubuntu-latest"] + include: + - python-version: '3.11' + numpy-version: '1.25' + dependency-set: 'optional' + os: 'macos-latest' + - python-version: '3.13' + numpy-version: '2.2' + dependency-set: 'optional' + os: 'macos-latest' + - python-version: '3.11' + numpy-version: '1.25' + dependency-set: 'optional' + os: 'windows-latest' + - python-version: '3.13' + numpy-version: '2.2' + dependency-set: 'optional' + os: 'windows-latest' + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # grab all branches and tags + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install Hatch + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Set Up Hatch Env + run: | + hatch env create test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} + hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env + - name: Run Tests + env: + HYPOTHESIS_PROFILE: ci + run: | + hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage + - name: Upload coverage + if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) + + test-upstream-and-min-deps: + name: py=${{ matrix.python-version }}-${{ matrix.dependency-set }} + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.11', "3.13"] + dependency-set: ["upstream", "min_deps"] + exclude: + - python-version: "3.13" + dependency-set: min_deps + - python-version: "3.11" + dependency-set: upstream + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install Hatch + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Set Up Hatch Env + run: | + hatch env create ${{ matrix.dependency-set }} + hatch env run -e ${{ matrix.dependency-set }} list-env + - name: Run Tests + run: | + hatch env run --env ${{ matrix.dependency-set }} run-coverage + - name: Upload coverage + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) + + doctests: + name: doctests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # required for hatch version discovery, which is needed for numcodecs.zarr3 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: 'pip' + - name: Install Hatch + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Set Up Hatch Env + run: | + hatch env create doctest + hatch env run -e doctest list-env + - name: Run Tests + run: | + hatch env run --env doctest run + + test-complete: + name: Test complete + + needs: + [ + test, + test-upstream-and-min-deps, + doctests + ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check failure + if: | + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') + run: exit 1 + - name: Success + run: echo Success! diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml deleted file mode 100644 index 0d746ced99..0000000000 --- a/.github/workflows/windows-testing.yml +++ /dev/null @@ -1,60 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python package - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - windows: - name: Windows Tests - runs-on: "windows-latest" - strategy: - fail-fast: True - matrix: - python-version: ["3.8"] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} - channels: conda-forge - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: true - - name: Create Conda environment with the rights deps - shell: bash -l {0} - run: | - conda create -n zarr-env python==${{matrix.python-version}} numcodecs==0.6.4 pip nodejs - - name: Install dependencies - shell: bash -l {0} - run: | - conda activate zarr-env - python -m pip install --upgrade pip - python -m pip install -U pip setuptools wheel - python -m pip install -r requirements_dev_numpy.txt -r requirements_dev_minimal.txt -r requirements_dev_optional.txt - python -m pip install -e . - python -m pip freeze - npm install -g azurite - - name: Run Tests - shell: bash -l {0} - run: | - conda activate zarr-env - mkdir ~/blob_emulator - azurite -l ~/blob_emulator --debug debug.log 2>&1 > stdouterr.log & - pytest - env: - ZARR_TEST_ABS: 1 - - name: Conda info - shell: bash -l {0} - run: conda info - - name: Conda list - shell: pwsh - run: conda list - diff --git a/.gitignore b/.gitignore index f2f7edc348..1b2b63e651 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # Distribution / packaging .Python env/ +.venv/ build/ develop-eggs/ dist/ @@ -35,7 +36,6 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ -.tox/ .coverage .coverage.* .cache @@ -51,6 +51,10 @@ coverage.xml # Sphinx documentation docs/_build/ +docs/api +docs/data +data +data.zip # PyBuilder target/ @@ -61,12 +65,15 @@ target/ # Jupyter .ipynb_checkpoints/ -# setuptools-scm -zarr/version.py +# VCS versioning +src/zarr/_version.py # emacs *~ +# VSCode +.vscode/ + # test data #*.zarr #*.zip @@ -74,3 +81,12 @@ zarr/version.py #doesnotexist #test_sync* data/* +src/fixture/ +fixture/ +junit.xml + +.DS_Store +tests/.hypothesis +.hypothesis/ + +zarr/version.py diff --git a/.pep8speaks.yml b/.pep8speaks.yml deleted file mode 100644 index a000ded163..0000000000 --- a/.pep8speaks.yml +++ /dev/null @@ -1,4 +0,0 @@ -pycodestyle: - max-line-length: 100 - exclude: - - docs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..fd50366a1c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,58 @@ +ci: + autoupdate_commit_msg: "chore: update pre-commit hooks" + autoupdate_schedule: "monthly" + autofix_commit_msg: "style: pre-commit fixes" + autofix_prs: false +default_stages: [pre-commit, pre-push] +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.9 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + - id: ruff-format + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + args: ["-L", "fo,ihs,kake,te", "-S", "fixture"] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-yaml + - id: trailing-whitespace + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.15.0 + hooks: + - id: mypy + files: src|tests + additional_dependencies: + # Package dependencies + - packaging + - donfig + - numcodecs[crc32c] + - numpy==2.1 # until https://github.com/numpy/numpy/issues/28034 is resolved + - typing_extensions + - universal-pathlib + - obstore>=0.5.1 + # Tests + - pytest + - hypothesis + - s3fs + - repo: https://github.com/scientific-python/cookie + rev: 2025.05.02 + hooks: + - id: sp-repo-review + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-directive-colons + - id: rst-inline-touching-normal + - repo: https://github.com/numpy/numpydoc + rev: v1.8.0 + hooks: + - id: numpydoc-validation + - repo: https://github.com/twisted/towncrier + rev: 24.8.0 + hooks: + - id: towncrier-check diff --git a/.pyup.yml b/.pyup.yml index 04937ed0b0..a70df4e3fb 100644 --- a/.pyup.yml +++ b/.pyup.yml @@ -13,6 +13,3 @@ requirements: - requirements_dev_optional.txt: pin: True update: all - - requirements_rtfd.txt: - pin: False - update: False diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000000..6253a7196f --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,26 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.12" + jobs: + pre_build: + - | + if [ "$READTHEDOCS_VERSION_TYPE" != "tag" ]; + then + towncrier build --version Unreleased --yes; + fi + +sphinx: + configuration: docs/conf.py + fail_on_warning: true + +formats: all + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 93175dd661..0000000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,46 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at zarr.conduct@gmail.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] - -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ diff --git a/FUNDING.yml b/FUNDING.yml new file mode 100644 index 0000000000..a2faf1a285 --- /dev/null +++ b/FUNDING.yml @@ -0,0 +1,2 @@ +github: [numfocus] +custom: ['https://numfocus.org/donate-to-zarr'] diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 5bb4df8cf8..0000000000 --- a/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015-2018 Zarr Developers - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000000..a4de1c39d3 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015-2024 Zarr Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index de20ebbc3b..97f5617934 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
-
+
# Zarr @@ -31,7 +31,7 @@ License - + license @@ -39,16 +39,25 @@ Build Status - - travis build status + + build status + + Pre-commit Status + + + pre-commit status + + + + Coverage - coverage + coverage @@ -61,14 +70,21 @@ - Gitter + Developer Chat - - + + + Funding + + + CZI's Essential Open Source Software for Science + + + Citation @@ -85,13 +101,13 @@ Zarr is a Python package providing an implementation of compressed, chunked, N-d ## Main Features -- [**Create**](https://zarr.readthedocs.io/en/stable/tutorial.html#creating-an-array) N-dimensional arrays with any NumPy `dtype`. -- [**Chunk arrays**](https://zarr.readthedocs.io/en/stable/tutorial.html#chunk-optimizations) along any dimension. -- [**Compress**](https://zarr.readthedocs.io/en/stable/tutorial.html#compressors) and/or filter chunks using any NumCodecs codec. -- [**Store arrays**](https://zarr.readthedocs.io/en/stable/tutorial.html#tutorial-storage) in memory, on disk, inside a zip file, on S3, etc... -- [**Read**](https://zarr.readthedocs.io/en/stable/tutorial.html#reading-and-writing-data) an array [**concurrently**](https://zarr.readthedocs.io/en/stable/tutorial.html#parallel-computing-and-synchronization) from multiple threads or processes. -- Write to an array concurrently from multiple threads or processes. -- Organize arrays into hierarchies via [**groups**](https://zarr.readthedocs.io/en/stable/tutorial.html#groups). +- [**Create**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#creating-an-array) N-dimensional arrays with any NumPy `dtype`. +- [**Chunk arrays**](https://zarr.readthedocs.io/en/stable/user-guide/performance.html#chunk-optimizations) along any dimension. +- [**Compress**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#compressors) and/or filter chunks using any NumCodecs codec. +- [**Store arrays**](https://zarr.readthedocs.io/en/stable/user-guide/storage.html) in memory, on disk, inside a zip file, on S3, etc... +- [**Read**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#reading-and-writing-data) an array [**concurrently**](https://zarr.readthedocs.io/en/stable/user-guide/performance.html#parallel-computing-and-synchronization) from multiple threads or processes. +- [**Write**](https://zarr.readthedocs.io/en/stable/user-guide/arrays.html#reading-and-writing-data) to an array concurrently from multiple threads or processes. +- Organize arrays into hierarchies via [**groups**](https://zarr.readthedocs.io/en/stable/quickstart.html#hierarchical-groups). ## Where to get it @@ -107,4 +123,4 @@ or via `conda`: conda install -c conda-forge zarr ``` -For more details, including how to install from source, see the [installation documentation](https://zarr.readthedocs.io/en/stable/#installation). +For more details, including how to install from source, see the [installation documentation](https://zarr.readthedocs.io/en/stable/index.html#installation). diff --git a/TEAM.md b/TEAM.md new file mode 100644 index 0000000000..e6975d7c04 --- /dev/null +++ b/TEAM.md @@ -0,0 +1,29 @@ +## Active core-developers +- @joshmoore (Josh Moore) +- @jni (Juan Nunez-Iglesias) +- @rabernat (Ryan Abernathey) +- @jhamman (Joe Hamman) +- @d-v-b (Davis Bennett) +- @jakirkham (jakirkham) +- @martindurant (Martin Durant) +- @normanrz (Norman Rzepka) +- @dstansby (David Stansby) +- @dcherian (Deepak Cherian) +- @TomAugspurger (Tom Augspurger) + +## Emeritus core-developers +- @alimanfoo (Alistair Miles) +- @shoyer (Stephan Hoyer) +- @ryan-williams (Ryan Williams) +- @jrbourbeau (James Bourbeau) +- @mzjp2 (Zain Patel) +- @grlee77 (Gregory Lee) + +## Former core-developers +- @jeromekelleher (Jerome Kelleher) +- @tjcrone (Tim Crone) +- @funkey (Jan Funke) +- @shikharsg +- @Carreau (Matthias Bussonnier) +- @dazzag24 +- @WardF (Ward Fisher) diff --git a/bench/compress_normal.py b/bench/compress_normal.py index ce0a05b9ec..179520a0e4 100644 --- a/bench/compress_normal.py +++ b/bench/compress_normal.py @@ -1,44 +1,46 @@ import sys import timeit +import line_profiler import numpy as np -import line_profiler import zarr from zarr import blosc if __name__ == "__main__": - - sys.path.insert(0, '..') + sys.path.insert(0, "..") # setup - a = np.random.normal(2000, 1000, size=200000000).astype('u2') - z = zarr.empty_like(a, chunks=1000000, - compression='blosc', - compression_opts=dict(cname='lz4', clevel=5, shuffle=2)) + a = np.random.normal(2000, 1000, size=200000000).astype("u2") + z = zarr.empty_like( + a, + chunks=1000000, + compression="blosc", + compression_opts={"cname": "lz4", "clevel": 5, "shuffle": 2}, + ) print(z) - print('*' * 79) + print("*" * 79) # time - t = timeit.repeat('z[:] = a', repeat=10, number=1, globals=globals()) + t = timeit.repeat("z[:] = a", repeat=10, number=1, globals=globals()) print(t) print(min(t)) print(z) # profile profile = line_profiler.LineProfiler(blosc.compress) - profile.run('z[:] = a') + profile.run("z[:] = a") profile.print_stats() - print('*' * 79) + print("*" * 79) # time - t = timeit.repeat('z[:]', repeat=10, number=1, globals=globals()) + t = timeit.repeat("z[:]", repeat=10, number=1, globals=globals()) print(t) print(min(t)) # profile profile = line_profiler.LineProfiler(blosc.decompress) - profile.run('z[:]') + profile.run("z[:]") profile.print_stats() diff --git a/bench/compress_normal.txt b/bench/compress_normal.txt index 100d24bfdf..e5d6be6aeb 100644 --- a/bench/compress_normal.txt +++ b/bench/compress_normal.txt @@ -19,7 +19,7 @@ Line # Hits Time Per Hit % Time Line Contents ============================================================== 137 def compress(source, char* cname, int clevel, int shuffle): 138 """Compress data in a numpy array. - 139 + 139 140 Parameters 141 ---------- 142 source : array-like @@ -30,14 +30,14 @@ Line # Hits Time Per Hit % Time Line Contents 147 Compression level. 148 shuffle : int 149 Shuffle filter. - 150 + 150 151 Returns 152 ------- 153 dest : bytes-like 154 Compressed data. - 155 + 155 156 """ - 157 + 157 158 cdef: 159 char *source_ptr 160 char *dest_ptr @@ -45,18 +45,18 @@ Line # Hits Time Per Hit % Time Line Contents 162 size_t nbytes, cbytes, itemsize 163 200 506 2.5 0.2 array.array char_array_template = array.array('b', []) 164 array.array dest - 165 + 165 166 # setup source buffer 167 200 458 2.3 0.2 PyObject_GetBuffer(source, &source_buffer, PyBUF_ANY_CONTIGUOUS) 168 200 119 0.6 0.0 source_ptr = source_buffer.buf - 169 + 169 170 # setup destination 171 200 239 1.2 0.1 nbytes = source_buffer.len 172 200 103 0.5 0.0 itemsize = source_buffer.itemsize 173 200 2286 11.4 0.8 dest = array.clone(char_array_template, nbytes + BLOSC_MAX_OVERHEAD, 174 zero=False) 175 200 129 0.6 0.0 dest_ptr = dest.data.as_voidptr - 176 + 176 177 # perform compression 178 200 1734 8.7 0.6 if _get_use_threads(): 179 # allow blosc to use threads internally @@ -67,24 +67,24 @@ Line # Hits Time Per Hit % Time Line Contents 184 cbytes = blosc_compress(clevel, shuffle, itemsize, nbytes, 185 source_ptr, dest_ptr, 186 nbytes + BLOSC_MAX_OVERHEAD) - 187 + 187 188 else: 189 with nogil: 190 cbytes = blosc_compress_ctx(clevel, shuffle, itemsize, nbytes, 191 source_ptr, dest_ptr, 192 nbytes + BLOSC_MAX_OVERHEAD, cname, 193 0, 1) - 194 + 194 195 # release source buffer 196 200 616 3.1 0.2 PyBuffer_Release(&source_buffer) - 197 + 197 198 # check compression was successful 199 200 120 0.6 0.0 if cbytes <= 0: 200 raise RuntimeError('error during blosc compression: %d' % cbytes) - 201 + 201 202 # resize after compression 203 200 1896 9.5 0.6 array.resize(dest, cbytes) - 204 + 204 205 200 186 0.9 0.1 return dest ******************************************************************************* @@ -100,19 +100,19 @@ Line # Hits Time Per Hit % Time Line Contents ============================================================== 75 def decompress(source, dest): 76 """Decompress data. - 77 + 77 78 Parameters 79 ---------- 80 source : bytes-like 81 Compressed data, including blosc header. 82 dest : array-like 83 Object to decompress into. - 84 + 84 85 Notes 86 ----- 87 Assumes that the size of the destination buffer is correct for the size of 88 the uncompressed data. - 89 + 89 90 """ 91 cdef: 92 int ret @@ -122,7 +122,7 @@ Line # Hits Time Per Hit % Time Line Contents 96 array.array source_array 97 Py_buffer dest_buffer 98 size_t nbytes - 99 + 99 100 # setup source buffer 101 200 573 2.9 0.2 if PY2 and isinstance(source, array.array): 102 # workaround fact that array.array does not support new-style buffer @@ -134,13 +134,13 @@ Line # Hits Time Per Hit % Time Line Contents 108 200 112 0.6 0.0 release_source_buffer = True 109 200 144 0.7 0.1 PyObject_GetBuffer(source, &source_buffer, PyBUF_ANY_CONTIGUOUS) 110 200 98 0.5 0.0 source_ptr = source_buffer.buf - 111 + 111 112 # setup destination buffer 113 200 552 2.8 0.2 PyObject_GetBuffer(dest, &dest_buffer, 114 PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE) 115 200 100 0.5 0.0 dest_ptr = dest_buffer.buf 116 200 84 0.4 0.0 nbytes = dest_buffer.len - 117 + 117 118 # perform decompression 119 200 1856 9.3 0.8 if _get_use_threads(): 120 # allow blosc to use threads internally @@ -149,13 +149,12 @@ Line # Hits Time Per Hit % Time Line Contents 123 else: 124 with nogil: 125 ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1) - 126 + 126 127 # release buffers 128 200 754 3.8 0.3 if release_source_buffer: 129 200 326 1.6 0.1 PyBuffer_Release(&source_buffer) 130 200 165 0.8 0.1 PyBuffer_Release(&dest_buffer) - 131 + 131 132 # handle errors 133 200 128 0.6 0.1 if ret <= 0: 134 raise RuntimeError('error during blosc decompression: %d' % ret) - diff --git a/build.cmd b/build.cmd deleted file mode 100644 index 053894d249..0000000000 --- a/build.cmd +++ /dev/null @@ -1,45 +0,0 @@ -:: To build extensions for 64 bit Python 3, we need to configure environment -:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) -:: -:: To build extensions for 64 bit Python 2, we need to configure environment -:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) -:: -:: 32 bit builds do not require specific environment configurations. -:: -:: Note: this script needs to be run with the /E:ON and /V:ON flags for the -:: cmd interpreter, at least for (SDK v7.0) -:: -:: More details at: -:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows -:: http://stackoverflow.com/a/13751649/163740 -:: -:: Author: Olivier Grisel -:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ -@ECHO OFF - -SET COMMAND_TO_RUN=%* -SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows - -SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%" -IF %MAJOR_PYTHON_VERSION% == "3" ( - SET WINDOWS_SDK_VERSION="v7.1" -) ELSE ( - ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" - EXIT 1 -) - -IF "%DISTUTILS_USE_SDK%"=="1" ( - ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture - SET DISTUTILS_USE_SDK=1 - SET MSSdk=1 - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) ELSE ( - ECHO Using default MSVC build environment - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) diff --git a/changes/.gitignore b/changes/.gitignore new file mode 100644 index 0000000000..f935021a8f --- /dev/null +++ b/changes/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/changes/2774.feature.rst b/changes/2774.feature.rst new file mode 100644 index 0000000000..4df83f54ec --- /dev/null +++ b/changes/2774.feature.rst @@ -0,0 +1 @@ +Add `zarr.storage.FsspecStore.from_mapper()` so that `zarr.open()` supports stores of type `fsspec.mapping.FSMap`. \ No newline at end of file diff --git a/changes/2871.feature.rst b/changes/2871.feature.rst new file mode 100644 index 0000000000..a39f30c558 --- /dev/null +++ b/changes/2871.feature.rst @@ -0,0 +1,8 @@ +Added public API for Buffer ABCs and implementations. + +Use :mod:`zarr.buffer` to access buffer implementations, and +:mod:`zarr.abc.buffer` for the interface to implement new buffer types. + +Users previously importing buffer from ``zarr.core.buffer`` should update their +imports to use :mod:`zarr.buffer`. As a reminder, all of ``zarr.core`` is +considered a private API that's not covered by zarr-python's versioning policy. \ No newline at end of file diff --git a/changes/2874.feature.rst b/changes/2874.feature.rst new file mode 100644 index 0000000000..4c50532ae0 --- /dev/null +++ b/changes/2874.feature.rst @@ -0,0 +1,9 @@ +Adds zarr-specific data type classes. This replaces the internal use of numpy data types for zarr +v2 and a fixed set of string enums for zarr v3. This change is largely internal, but it does +change the type of the ``dtype`` and ``data_type`` fields on the ``ArrayV2Metadata`` and +``ArrayV3Metadata`` classes. It also changes the JSON metadata representation of the +variable-length string data type, but the old metadata representation can still be +used when reading arrays. The logic for automatically choosing the chunk encoding for a given data +type has also changed, and this necessitated changes to the ``config`` API. + +For more on this new feature, see the `documentation `_ \ No newline at end of file diff --git a/changes/2921.bugfix.rst b/changes/2921.bugfix.rst new file mode 100644 index 0000000000..65db48654f --- /dev/null +++ b/changes/2921.bugfix.rst @@ -0,0 +1 @@ +Ignore stale child metadata when reconsolidating metadata. diff --git a/changes/3021.feature.rst b/changes/3021.feature.rst new file mode 100644 index 0000000000..8805797ce3 --- /dev/null +++ b/changes/3021.feature.rst @@ -0,0 +1 @@ +Implemented ``move`` for ``LocalStore`` and ``ZipStore``. This allows users to move the store to a different root path. \ No newline at end of file diff --git a/changes/3066.feature.rst b/changes/3066.feature.rst new file mode 100644 index 0000000000..89d5ddb1c6 --- /dev/null +++ b/changes/3066.feature.rst @@ -0,0 +1 @@ +Added `~zarr.errors.GroupNotFoundError`, which is raised when attempting to open a group that does not exist. diff --git a/changes/3068.bugfix.rst b/changes/3068.bugfix.rst new file mode 100644 index 0000000000..9ada322c13 --- /dev/null +++ b/changes/3068.bugfix.rst @@ -0,0 +1 @@ +Trying to open an array with ``mode='r'`` when the store is not read-only now raises an error. diff --git a/changes/3081.feature.rst b/changes/3081.feature.rst new file mode 100644 index 0000000000..8cf83ea7c2 --- /dev/null +++ b/changes/3081.feature.rst @@ -0,0 +1 @@ +Adds ``fill_value`` to the list of attributes displayed in the output of the ``AsyncArray.info()`` method. \ No newline at end of file diff --git a/changes/3082.feature.rst b/changes/3082.feature.rst new file mode 100644 index 0000000000..e990d1f3a0 --- /dev/null +++ b/changes/3082.feature.rst @@ -0,0 +1 @@ +Use :py:func:`numpy.zeros` instead of :py:func:`np.full` for a performance speedup when creating a `zarr.core.buffer.NDBuffer` with `fill_value=0`. \ No newline at end of file diff --git a/changes/3100.bugfix.rst b/changes/3100.bugfix.rst new file mode 100644 index 0000000000..11f06628c0 --- /dev/null +++ b/changes/3100.bugfix.rst @@ -0,0 +1,3 @@ +For Zarr format 2, allow fixed-length string arrays to be created without automatically inserting a +``Vlen-UT8`` codec in the array of filters. Fixed-length string arrays do not need this codec. This +change fixes a regression where fixed-length string arrays created with Zarr Python 3 could not be read with Zarr Python 2.18. \ No newline at end of file diff --git a/changes/3103.bugfix.rst b/changes/3103.bugfix.rst new file mode 100644 index 0000000000..93aecce908 --- /dev/null +++ b/changes/3103.bugfix.rst @@ -0,0 +1,7 @@ +When creating arrays without explicitly specifying a chunk size using `zarr.create` and other +array creation routines, the chunk size will now set automatically instead of defaulting to the data shape. +For large arrays this will result in smaller default chunk sizes. +To retain previous behaviour, explicitly set the chunk shape to the data shape. + +This fix matches the existing chunking behaviour of +`zarr.save_array` and `zarr.api.asynchronous.AsyncArray.create`. diff --git a/changes/3127.bugfix.rst b/changes/3127.bugfix.rst new file mode 100644 index 0000000000..35d7f5d329 --- /dev/null +++ b/changes/3127.bugfix.rst @@ -0,0 +1,2 @@ +When `zarr.save` has an argument `path=some/path/` and multiple arrays in `args`, the path resulted in `some/path/some/path` due to using the `path` +argument twice while building the array path. This is now fixed. \ No newline at end of file diff --git a/changes/3128.bugfix.rst b/changes/3128.bugfix.rst new file mode 100644 index 0000000000..b93416070e --- /dev/null +++ b/changes/3128.bugfix.rst @@ -0,0 +1 @@ +Fix `zarr.open` default for argument `mode` when `store` is `read_only` \ No newline at end of file diff --git a/changes/3130.feature.rst b/changes/3130.feature.rst new file mode 100644 index 0000000000..7a64582f06 --- /dev/null +++ b/changes/3130.feature.rst @@ -0,0 +1 @@ +Port more stateful testing actions from `Icechunk `_. diff --git a/changes/3138.feature.rst b/changes/3138.feature.rst new file mode 100644 index 0000000000..ecd339bf9c --- /dev/null +++ b/changes/3138.feature.rst @@ -0,0 +1 @@ +Adds a `with_read_only` convenience method to the `Store` abstract base class (raises `NotImplementedError`) and implementations to the `MemoryStore`, `ObjectStore`, `LocalStore`, and `FsspecStore` classes. \ No newline at end of file diff --git a/changes/README.md b/changes/README.md new file mode 100644 index 0000000000..74ed9f94a9 --- /dev/null +++ b/changes/README.md @@ -0,0 +1,14 @@ +Writing a changelog entry +------------------------- + +Please put a new file in this directory named `xxxx..rst`, where + +- `xxxx` is the pull request number associated with this entry +- `` is one of: + - feature + - bugfix + - doc + - removal + - misc + +Inside the file, please write a short description of what you have changed, and how it impacts users of `zarr-python`. diff --git a/codecov.yml b/codecov.yml index e90b9e9fe1..83274aedec 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,15 +1,10 @@ coverage: status: + patch: + default: + target: auto project: default: - target: 100 + target: auto threshold: 0.1 - patch: - default: - target: 100 -comment: - layout: "diff, files" - behavior: default - require_changes: true # if true: only post the comment if coverage changes - branches: # branch names that can post comment - - "master" +comment: false diff --git a/data/donotdelete b/data/donotdelete deleted file mode 100644 index 1e9ef93e26..0000000000 --- a/data/donotdelete +++ /dev/null @@ -1 +0,0 @@ -This directory is used for data files created during testing. \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile index fe6a0bc4e9..f42ee840e9 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -2,14 +2,14 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = -W --keep-going SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) - $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) + $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from https://www.sphinx-doc.org/) endif # Internal variables. @@ -52,6 +52,7 @@ help: .PHONY: clean clean: rm -rf $(BUILDDIR)/* + rm -rf $(BUILDDIR)/../api .PHONY: html html: diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 9ad0555b95..1d32606f9a 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,32 +1,110 @@ +@import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); -/* Sidebar background color */ -.wy-nav-side, div.wy-side-nav-search { - background-color: rgb(38, 34, 98); +body { + font-family: 'Open Sans', sans-serif; } -/* Sidebar link click color */ -.wy-menu-vertical .toctree-l1 > a:active { - background-color: rgb(38, 34, 98); - color: rgb(252, 252, 252); +pre, code { + font-size: 100%; + line-height: 155%; } -/* Link color is darker to make hovering more clear */ -.wy-menu-vertical .toctree-l1 > a:hover { - background-color: rgb(25, 22, 65); - color: rgb(252, 252, 252); +/* Style the active version button. + +- dev: orange +- stable: green +- old, PR: red + +Colors from: + +Wong, B. Points of view: Color blindness. +Nat Methods 8, 441 (2011). https://doi.org/10.1038/nmeth.1618 +*/ + +/* If the active version has the name "dev", style it orange */ +#version_switcher_button[data-active-version-name*="dev"] { + background-color: #E69F00; + border-color: #E69F00; + color:#000000; +} + +/* green for `stable` */ +#version_switcher_button[data-active-version-name*="stable"] { + background-color: #009E73; + border-color: #009E73; +} + +/* red for `old` */ +#version_switcher_button:not([data-active-version-name*="stable"], [data-active-version-name*="dev"], [data-active-version-name=""]) { + background-color: #980F0F; + border-color: #980F0F; +} + +/* Main page overview cards */ + +.sd-card { + background: #fff; + border-radius: 0; + padding: 30px 10px 20px 10px; + margin: 10px 0px; +} + +.sd-card .sd-card-header { + text-align: center; +} + +.sd-card .sd-card-header .sd-card-text { + margin: 0px; +} + +.sd-card .sd-card-img-top { + height: 52px; + width: 52px; + margin-left: auto; + margin-right: auto; +} + +.sd-card .sd-card-header { + border: none; + background-color: white; + font-size: var(--pst-font-size-h5); + font-weight: bold; + padding: 2.5rem 0rem 0.5rem 0rem; +} + +.sd-card .sd-card-footer { + border: none; + background-color: white; +} + +.sd-card .sd-card-footer .sd-card-text { + max-width: 220px; + margin-left: auto; + margin-right: auto; +} + +/* Dark theme tweaking */ +html[data-theme=dark] .sd-card img[src*='.svg'] { + filter: invert(0.82) brightness(0.8) contrast(1.2); +} + +/* Main index page overview cards */ +html[data-theme=dark] .sd-card { + background-color:var(--pst-color-background); +} + +html[data-theme=dark] .sd-shadow-sm { + box-shadow: 0 .1rem 1rem rgba(250, 250, 250, .6) !important } -.wy-menu-vertical li.current > a:hover, .wy-menu-vertical li.current > a:active { - color: #404040; - background-color: #F5F5F5; +html[data-theme=dark] .sd-card .sd-card-header { + background-color:var(--pst-color-background); } -/* On hover over logo */ -.wy-side-nav-search > a:hover, .wy-side-nav-search .wy-dropdown > a:hover { - background: inherit; +html[data-theme=dark] .sd-card .sd-card-footer { + background-color:var(--pst-color-background); } -/* Border around search box */ -.wy-side-nav-search input[type="text"] { - border: 0px; +html[data-theme=dark] h1 { + color: var(--pst-color-primary); } diff --git a/docs/_static/custom.js b/docs/_static/custom.js new file mode 100644 index 0000000000..52f1cba9e0 --- /dev/null +++ b/docs/_static/custom.js @@ -0,0 +1,17 @@ +// handle redirects +(() => { + let anchorMap = { + "installation": "installation.html", + "getting-started": "getting_started.html#getting-started", + "highlights": "getting_started.html#highlights", + "contributing": "contributing.html", + "projects-using-zarr": "getting_started.html#projects-using-zarr", + "contents": "getting_started.html#contents", + "indices-and-tables": "api.html#indices-and-tables" + } + + let hash = window.location.hash.substring(1); + if (hash && hash in anchorMap) { + window.location.replace(anchorMap[hash]); + } +})(); diff --git a/docs/_static/index_api.svg b/docs/_static/index_api.svg new file mode 100644 index 0000000000..69f7ba1d2d --- /dev/null +++ b/docs/_static/index_api.svg @@ -0,0 +1,97 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + diff --git a/docs/_static/index_contribute.svg b/docs/_static/index_contribute.svg new file mode 100644 index 0000000000..de3d902379 --- /dev/null +++ b/docs/_static/index_contribute.svg @@ -0,0 +1,76 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + diff --git a/docs/_static/index_getting_started.svg b/docs/_static/index_getting_started.svg new file mode 100644 index 0000000000..2d36622cb7 --- /dev/null +++ b/docs/_static/index_getting_started.svg @@ -0,0 +1,66 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + diff --git a/docs/_static/index_user_guide.svg b/docs/_static/index_user_guide.svg new file mode 100644 index 0000000000..bd17053517 --- /dev/null +++ b/docs/_static/index_user_guide.svg @@ -0,0 +1,67 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + diff --git a/docs/_static/logo1.png b/docs/_static/logo1.png index 4d18ca6739..e6623c3ec9 100644 Binary files a/docs/_static/logo1.png and b/docs/_static/logo1.png differ diff --git a/docs/_static/logo_horizontal.svg b/docs/_static/logo_horizontal.svg new file mode 100644 index 0000000000..90fa4bc3a8 --- /dev/null +++ b/docs/_static/logo_horizontal.svg @@ -0,0 +1,135 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/about.rst b/docs/about.rst new file mode 100644 index 0000000000..7a0af998c0 --- /dev/null +++ b/docs/about.rst @@ -0,0 +1,24 @@ +About +===== + +Zarr is a format for the storage of chunked, compressed, N-dimensional arrays +inspired by `HDF5 `_, `h5py +`_ and `bcolz `_. + +These documents describe the Zarr-Python implementation. More information +about the Zarr format can be found on the `main website `_. + +Projects using Zarr +------------------- + +If you are using Zarr-Python, we would `love to hear about it +`_. + +Funding +------- +The project is fiscally sponsored by `NumFOCUS `_, a US +501(c)(3) public charity, and development is supported by the +`MRC Centre for Genomics and Global Health `_ +and the `Chan Zuckerberg Initiative `_. + +.. _NumCodecs: https://numcodecs.readthedocs.io/ diff --git a/docs/api.rst b/docs/api.rst deleted file mode 100644 index 8162ada965..0000000000 --- a/docs/api.rst +++ /dev/null @@ -1,15 +0,0 @@ -API reference -============= - -.. toctree:: - :maxdepth: 3 - - api/creation - api/core - api/hierarchy - api/storage - api/n5 - api/convenience - api/codecs - api/attrs - api/sync diff --git a/docs/api/attrs.rst b/docs/api/attrs.rst deleted file mode 100644 index f95e63af3a..0000000000 --- a/docs/api/attrs.rst +++ /dev/null @@ -1,16 +0,0 @@ -The Attributes class (``zarr.attrs``) -===================================== -.. module:: zarr.attrs - -.. autoclass:: Attributes - - .. automethod:: __getitem__ - .. automethod:: __setitem__ - .. automethod:: __delitem__ - .. automethod:: __iter__ - .. automethod:: __len__ - .. automethod:: keys - .. automethod:: asdict - .. automethod:: put - .. automethod:: update - .. automethod:: refresh diff --git a/docs/api/codecs.rst b/docs/api/codecs.rst deleted file mode 100644 index 2e176bb7ea..0000000000 --- a/docs/api/codecs.rst +++ /dev/null @@ -1,23 +0,0 @@ -Compressors and filters (``zarr.codecs``) -========================================= -.. module:: zarr.codecs - -This module contains compressor and filter classes for use with Zarr. Please note that this module -is provided for backwards compatibility with previous versions of Zarr. From Zarr version 2.2 -onwards, all codec classes have been moved to a separate package called Numcodecs_. The two -packages (Zarr and Numcodecs_) are designed to be used together. For example, a Numcodecs_ codec -class can be used as a compressor for a Zarr array:: - - >>> import zarr - >>> from numcodecs import Blosc - >>> z = zarr.zeros(1000000, compressor=Blosc(cname='zstd', clevel=1, shuffle=Blosc.SHUFFLE)) - -Codec classes can also be used as filters. See the tutorial section on :ref:`tutorial_filters` -for more information. - -Please note that it is also relatively straightforward to define and register custom codec -classes. See the Numcodecs `codec API `_ and -`codec registry `_ documentation for more -information. - -.. _Numcodecs: http://numcodecs.readthedocs.io/ diff --git a/docs/api/convenience.rst b/docs/api/convenience.rst deleted file mode 100644 index a70a90ce7c..0000000000 --- a/docs/api/convenience.rst +++ /dev/null @@ -1,14 +0,0 @@ -Convenience functions (``zarr.convenience``) -============================================ -.. automodule:: zarr.convenience -.. autofunction:: open -.. autofunction:: save -.. autofunction:: load -.. autofunction:: save_array -.. autofunction:: save_group -.. autofunction:: copy -.. autofunction:: copy_all -.. autofunction:: copy_store -.. autofunction:: tree -.. autofunction:: consolidate_metadata -.. autofunction:: open_consolidated diff --git a/docs/api/core.rst b/docs/api/core.rst deleted file mode 100644 index 5789fb996b..0000000000 --- a/docs/api/core.rst +++ /dev/null @@ -1,22 +0,0 @@ -The Array class (``zarr.core``) -=============================== -.. module:: zarr.core - -.. autoclass:: Array - - .. automethod:: __getitem__ - .. automethod:: __setitem__ - .. automethod:: get_basic_selection - .. automethod:: set_basic_selection - .. automethod:: get_mask_selection - .. automethod:: set_mask_selection - .. automethod:: get_coordinate_selection - .. automethod:: set_coordinate_selection - .. automethod:: get_orthogonal_selection - .. automethod:: set_orthogonal_selection - .. automethod:: digest - .. automethod:: hexdigest - .. automethod:: resize - .. automethod:: append - .. automethod:: view - .. automethod:: astype diff --git a/docs/api/creation.rst b/docs/api/creation.rst deleted file mode 100644 index 66422c0670..0000000000 --- a/docs/api/creation.rst +++ /dev/null @@ -1,15 +0,0 @@ -Array creation (``zarr.creation``) -================================== -.. module:: zarr.creation -.. autofunction:: create -.. autofunction:: empty -.. autofunction:: zeros -.. autofunction:: ones -.. autofunction:: full -.. autofunction:: array -.. autofunction:: open_array -.. autofunction:: empty_like -.. autofunction:: zeros_like -.. autofunction:: ones_like -.. autofunction:: full_like -.. autofunction:: open_like diff --git a/docs/api/hierarchy.rst b/docs/api/hierarchy.rst deleted file mode 100644 index 88b9c0fd88..0000000000 --- a/docs/api/hierarchy.rst +++ /dev/null @@ -1,41 +0,0 @@ -Groups (``zarr.hierarchy``) -=========================== -.. module:: zarr.hierarchy - -.. autofunction:: group -.. autofunction:: open_group - -.. autoclass:: Group - - .. automethod:: __len__ - .. automethod:: __iter__ - .. automethod:: __contains__ - .. automethod:: __getitem__ - .. automethod:: __enter__ - .. automethod:: __exit__ - .. automethod:: group_keys - .. automethod:: groups - .. automethod:: array_keys - .. automethod:: arrays - .. automethod:: visit - .. automethod:: visitkeys - .. automethod:: visitvalues - .. automethod:: visititems - .. automethod:: tree - .. automethod:: create_group - .. automethod:: require_group - .. automethod:: create_groups - .. automethod:: require_groups - .. automethod:: create_dataset - .. automethod:: require_dataset - .. automethod:: create - .. automethod:: empty - .. automethod:: zeros - .. automethod:: ones - .. automethod:: full - .. automethod:: array - .. automethod:: empty_like - .. automethod:: zeros_like - .. automethod:: ones_like - .. automethod:: full_like - .. automethod:: move \ No newline at end of file diff --git a/docs/api/n5.rst b/docs/api/n5.rst deleted file mode 100644 index b6a8d8c61e..0000000000 --- a/docs/api/n5.rst +++ /dev/null @@ -1,5 +0,0 @@ -N5 (``zarr.n5``) -================ -.. automodule:: zarr.n5 - -.. autoclass:: N5Store diff --git a/docs/api/storage.rst b/docs/api/storage.rst deleted file mode 100644 index 4321837449..0000000000 --- a/docs/api/storage.rst +++ /dev/null @@ -1,50 +0,0 @@ -Storage (``zarr.storage``) -========================== -.. automodule:: zarr.storage - -.. autoclass:: MemoryStore -.. autoclass:: DirectoryStore -.. autoclass:: TempStore -.. autoclass:: NestedDirectoryStore -.. autoclass:: ZipStore - - .. automethod:: close - .. automethod:: flush - -.. autoclass:: DBMStore - - .. automethod:: close - .. automethod:: flush - -.. autoclass:: LMDBStore - - .. automethod:: close - .. automethod:: flush - -.. autoclass:: SQLiteStore - - .. automethod:: close - -.. autoclass:: MongoDBStore -.. autoclass:: RedisStore -.. autoclass:: LRUStoreCache - - .. automethod:: invalidate - .. automethod:: invalidate_values - .. automethod:: invalidate_keys - -.. autoclass:: ABSStore - -.. autoclass:: FSStore - -.. autoclass:: ConsolidatedMetadataStore - -.. autofunction:: init_array -.. autofunction:: init_group -.. autofunction:: contains_array -.. autofunction:: contains_group -.. autofunction:: listdir -.. autofunction:: rmdir -.. autofunction:: getsize -.. autofunction:: rename -.. autofunction:: migrate_1to2 diff --git a/docs/api/sync.rst b/docs/api/sync.rst deleted file mode 100644 index a139805e78..0000000000 --- a/docs/api/sync.rst +++ /dev/null @@ -1,6 +0,0 @@ -Synchronization (``zarr.sync``) -=============================== -.. module:: zarr.sync - -.. autoclass:: ThreadSynchronizer -.. autoclass:: ProcessSynchronizer diff --git a/docs/conf.py b/docs/conf.py index f3e2e1327b..68bf003ad5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,105 +15,140 @@ import os import sys +from importlib.metadata import version as get_version +from typing import Any -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -import zarr +import sphinx +import sphinx.application # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append(os.path.abspath('..')) +sys.path.append(os.path.abspath("..")) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.viewcode', - 'sphinx.ext.intersphinx', - 'numpydoc', - 'sphinx_issues', + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.viewcode", + "sphinx.ext.intersphinx", + 'autoapi.extension', + "numpydoc", + "sphinx_issues", + "sphinx_copybutton", + "sphinx_design", + 'sphinx_reredirects', ] -numpydoc_show_class_members = False -numpydoc_class_members_toctree = False -issues_github_path = 'zarr-developers/zarr-python' +issues_github_path = "zarr-developers/zarr-python" + +autoapi_dirs = ['../src/zarr'] +autoapi_add_toctree_entry = False +autoapi_generate_api_docs = True +autoapi_member_order = "groupwise" +autoapi_root = "api" +autoapi_keep_files = True +autoapi_options = [ 'members', 'undoc-members', 'show-inheritance', 'show-module-summary', 'imported-members', ] + +def skip_submodules( + app: sphinx.application.Sphinx, + what: str, + name: str, + obj: object, + skip: bool, + options: dict[str, Any] + ) -> bool: + # Skip documenting zarr.codecs submodules + # codecs are documented in the main zarr.codecs namespace + if what == "module" and name.startswith("zarr.codecs.") or name.startswith("zarr.core"): + skip = True + return skip # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' -# The master toctree document. -master_doc = 'index' +# The main toctree document. +main_doc = "index" # General information about the project. -project = 'zarr' -copyright = '2018, Zarr Developers' -author = 'Zarr Developers' - -version = zarr.__version__ -# The full version, including alpha/beta/rc tags. -release = zarr.__version__ +project = "zarr" +copyright = "2025, Zarr Developers" +author = "Zarr Developers" + +version = get_version("zarr") +release = get_version("zarr") + +redirects = { + "spec": "https://zarr-specs.readthedocs.io", + "spec/v1": 'https://zarr-specs.readthedocs.io/en/latest/v1/v1.0.html', + "spec/v2": "https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html", + "spec/v3": "https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html", + "license": "https://github.com/zarr-developers/zarr-python/blob/main/LICENSE.txt", + "tutorial": "user-guide", + "getting-started": "quickstart", + "roadmap": "developers/roadmap.html", + "installation": "user-guide/installation.html", + "api": "api/zarr/index", + "release": "release-notes.html", +} # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'talks'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "talks"] # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -123,164 +158,178 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "pydata_sphinx_theme" + +html_favicon = "_static/logo1.png" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme_options = {'logo_only': True} +html_theme_options = { + "github_url": "https://github.com/zarr-developers/zarr-python", + "twitter_url": "https://twitter.com/zarr_dev", + "icon_links": [ + { + "name": "Zarr Dev", + "url": "https://zarr.dev/", + "icon": "_static/logo1.png", + "type": "local", + }, + ], + "collapse_navigation": True, + "navigation_with_keys": False, + "announcement": "Zarr-Python 3 is here! Check out the release announcement here.", +} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. # " v documentation" by default. -#html_title = 'zarr v@@' +# html_title = 'zarr v@@' # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -html_logo = '_static/logo1.png' +html_logo = "_static/logo_horizontal.svg" -# Add custom css -def setup(app): - app.add_css_file('custom.css') +def setup(app: sphinx.application.Sphinx) -> None: + app.add_css_file("custom.css") + app.connect("autoapi-skip-member", skip_submodules) # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] +html_js_files = [ + "custom.js", +] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not None, a 'Last updated on:' timestamp is inserted at every page # bottom, using the given strftime format. # The empty string is equivalent to '%b %d, %Y'. -#html_last_updated_fmt = None +# html_last_updated_fmt = None # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +html_sidebars = {"tutorial": []} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # 'ja' uses this config value. # 'zh' user can custom change `jieba` dictionary path. -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'zarrdoc' +htmlhelp_basename = "zarrdoc" + +maximum_signature_line_length = 80 # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', + # Latex figure (float) alignment + #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'zarr.tex', 'zarr Documentation', - 'Zarr Developers', 'manual'), + (main_doc, "zarr.tex", "Zarr-Python", author, "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'zarr', 'zarr Documentation', - [author], 1) -] +man_pages = [(main_doc, "zarr", "Zarr-Python", [author], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -289,26 +338,42 @@ def setup(app): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'zarr', 'zarr Documentation', - author, 'zarr', 'One line description of project.', - 'Miscellaneous'), + ( + main_doc, + "zarr", + "Zarr-Python", + author, + "zarr", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False # Example configuration for intersphinx: refer to the Python standard library. # use in refs e.g: # :ref:`comparison manual ` -intersphinx_mapping = { 'python':('https://docs.python.org/', None), - 'numpy': ('https://numpy.org/doc/stable/', None)} +intersphinx_mapping = { + "python": ("https://docs.python.org/3/", None), + "numpy": ("https://numpy.org/doc/stable/", None), + "numcodecs": ("https://numcodecs.readthedocs.io/en/stable/", None), + "obstore": ("https://developmentseed.org/obstore/latest/", None), +} + + +# sphinx-copybutton configuration +copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " +copybutton_line_continuation_character = "\\" +copybutton_prompt_is_regexp = True diff --git a/docs/contributing.rst b/docs/contributing.rst deleted file mode 100644 index da8b5b945f..0000000000 --- a/docs/contributing.rst +++ /dev/null @@ -1,356 +0,0 @@ -Contributing to Zarr -==================== - -Zarr is a community maintained project. We welcome contributions in the form of bug -reports, bug fixes, documentation, enhancement proposals and more. This page provides -information on how best to contribute. - -Asking for help ---------------- - -If you have a question about how to use Zarr, please post your question on -StackOverflow using the `"zarr" tag `_. -If you don't get a response within a day or two, feel free to raise a `GitHub issue -`_ including a link to your StackOverflow -question. We will try to respond to questions as quickly as possible, but please bear -in mind that there may be periods where we have limited time to answer questions -due to other commitments. - -Bug reports ------------ - -If you find a bug, please raise a `GitHub issue -`_. Please include the following items in -a bug report: - -1. A minimal, self-contained snippet of Python code reproducing the problem. You can - format the code nicely using markdown, e.g.:: - - - ```python - import zarr - g = zarr.group() - # etc. - ``` - -2. An explanation of why the current behaviour is wrong/not desired, and what you - expect instead. - -3. Information about the version of Zarr, along with versions of dependencies and the - Python interpreter, and installation information. The version of Zarr can be obtained - from the ``zarr.__version__`` property. Please also state how Zarr was installed, - e.g., "installed via pip into a virtual environment", or "installed using conda". - Information about other packages installed can be obtained by executing ``pip freeze`` - (if using pip to install packages) or ``conda env export`` (if using conda to install - packages) from the operating system command prompt. The version of the Python - interpreter can be obtained by running a Python interactive session, e.g.:: - - $ python - Python 3.6.1 (default, Mar 22 2017, 06:17:05) - [GCC 6.3.0 20170321] on linux - -Enhancement proposals ---------------------- - -If you have an idea about a new feature or some other improvement to Zarr, please raise a -`GitHub issue `_ first to discuss. - -We very much welcome ideas and suggestions for how to improve Zarr, but please bear in -mind that we are likely to be conservative in accepting proposals for new features. The -reasons for this are that we would like to keep the Zarr code base lean and focused on -a core set of functionalities, and available time for development, review and maintenance -of new features is limited. But if you have a great idea, please don't let that stop -you from posting it on GitHub, just please don't be offended if we respond cautiously. - -Contributing code and/or documentation --------------------------------------- - -Forking the repository -~~~~~~~~~~~~~~~~~~~~~~ - -The Zarr source code is hosted on GitHub at the following location: - -* `https://github.com/zarr-developers/zarr-python `_ - -You will need your own fork to work on the code. Go to the link above and hit -the "Fork" button. Then clone your fork to your local machine:: - - $ git clone git@github.com:your-user-name/zarr.git - $ cd zarr - $ git remote add upstream git@github.com:zarr-developers/zarr-python.git - -Creating a development environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To work with the Zarr source code, it is recommended to set up a Python virtual -environment and install all Zarr dependencies using the same versions as are used by -the core developers and continuous integration services. Assuming you have a Python -3 interpreter already installed, and have also installed the virtualenv package, and -you have cloned the Zarr source code and your current working directory is the root of -the repository, you can do something like the following:: - - $ mkdir -p ~/pyenv/zarr-dev - $ virtualenv --no-site-packages --python=/usr/bin/python3.8 ~/pyenv/zarr-dev - $ source ~/pyenv/zarr-dev/bin/activate - $ pip install -r requirements_dev_minimal.txt -r requirements_dev_numpy.txt - $ pip install -e . - -To verify that your development environment is working, you can run the unit tests:: - - $ pytest -v zarr - -Creating a branch -~~~~~~~~~~~~~~~~~ - -Before you do any new work or submit a pull request, please open an issue on GitHub to -report the bug or propose the feature you'd like to add. - -It's best to synchronize your fork with the upstream repository, then create a -new, separate branch for each piece of work you want to do. E.g.:: - - git checkout master - git fetch upstream - git rebase upstream/master - git push - git checkout -b shiny-new-feature - git push -u origin shiny-new-feature - -This changes your working directory to the 'shiny-new-feature' branch. Keep any changes in -this branch specific to one bug or feature so it is clear what the branch brings to -Zarr. - -To update this branch with latest code from Zarr, you can retrieve the changes from -the master branch and perform a rebase:: - - git fetch upstream - git rebase upstream/master - -This will replay your commits on top of the latest Zarr git master. If this leads to -merge conflicts, these need to be resolved before submitting a pull request. -Alternatively, you can merge the changes in from upstream/master instead of rebasing, -which can be simpler:: - - git fetch upstream - git merge upstream/master - -Again, any conflicts need to be resolved before submitting a pull request. - -Running the test suite -~~~~~~~~~~~~~~~~~~~~~~ - -Zarr includes a suite of unit tests, as well as doctests included in -function and class docstrings and in the tutorial and storage -spec. The simplest way to run the unit tests is to activate your -development environment (see `creating a development environment`_ above) -and invoke:: - - $ pytest -v zarr - -Some tests require optional dependencies to be installed, otherwise -the tests will be skipped. To install all optional dependencies, run:: - - $ pip install -r requirements_dev_optional.txt - -To also run the doctests within docstrings (requires optional -depencies to be installed), run:: - - $ pytest -v --doctest-plus zarr - -To run the doctests within the tutorial and storage spec (requires -optional dependencies to be installed), run:: - - $ python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst - -Note that some tests also require storage services to be running -locally. To run the Azure Blob Service storage tests, run an Azure -storage emulator (e.g., azurite) and set the environment variable -``ZARR_TEST_ABS=1``. If you're using Docker to run azurite, start the service with:: - - docker run --rm -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --loose --blobHost 0.0.0.0 - -To run the Mongo DB storage tests, run a Mongo -server locally and set the environment variable ``ZARR_TEST_MONGO=1``. -To run the Redis storage tests, run a Redis server locally on port -6379 and set the environment variable ``ZARR_TEST_REDIS=1``. - -All tests are automatically run via GitHub Actions for every pull -request and must pass before code can be accepted. Test coverage is -also collected automatically via the Codecov service, and total -coverage over all builds must be 100% (although individual builds -may be lower due to Python 2/3 or other differences). - -Code standards -~~~~~~~~~~~~~~ - -All code must conform to the PEP8 standard. Regarding line length, lines up to 100 -characters are allowed, although please try to keep under 90 wherever possible. -Conformance can be checked by running:: - - $ flake8 --max-line-length=100 zarr - -Test coverage -~~~~~~~~~~~~~ - -Zarr maintains 100% test coverage under the latest Python stable release (currently -Python 3.8). Both unit tests and docstring doctests are included when computing -coverage. Running ``tox -e py38`` will automatically run the test suite with coverage -and produce a coverage report. This should be 100% before code can be accepted into the -main code base. - -When submitting a pull request, coverage will also be collected across all supported -Python versions via the Codecov service, and will be reported back within the pull -request. Codecov coverage must also be 100% before code can be accepted. - -Documentation -~~~~~~~~~~~~~ - -Docstrings for user-facing classes and functions should follow the -`numpydoc -`_ -standard, including sections for Parameters and Examples. All examples -should run and pass as doctests under Python 3.8. To run doctests, -activate your development environment, install optional requirements, -and run:: - - $ pytest -v --doctest-plus zarr - -Zarr uses Sphinx for documentation, hosted on readthedocs.org. Documentation is -written in the RestructuredText markup language (.rst files) in the ``docs`` folder. -The documentation consists both of prose and API documentation. All user-facing classes -and functions should be included in the API documentation, under the ``docs/api`` -folder. Any new features or important usage information should be included in the -tutorial (``docs/tutorial.rst``). Any changes should also be included in the release -notes (``docs/release.rst``). - -The documentation can be built locally by running:: - - $ tox -e docs - -The resulting built documentation will be available in the ``.tox/docs/tmp/html`` folder. - -Development best practices, policies and procedures ---------------------------------------------------- - -The following information is mainly for core developers, but may also be of interest to -contributors. - -Merging pull requests -~~~~~~~~~~~~~~~~~~~~~ - -Pull requests submitted by an external contributor should be reviewed and approved by at least -one core developers before being merged. Ideally, pull requests submitted by a core developer -should be reviewed and approved by at least one other core developers before being merged. - -Pull requests should not be merged until all CI checks have passed (GitHub Actions -Codecov) against code that has had the latest master merged in. - -Compatibility and versioning policies -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Because Zarr is a data storage library, there are two types of compatibility to -consider: API compatibility and data format compatibility. - -API compatibility -""""""""""""""""" - -All functions, classes and methods that are included in the API -documentation (files under ``docs/api/*.rst``) are considered as part of the Zarr **public API**, -except if they have been documented as an experimental feature, in which case they are part of -the **experimental API**. - -Any change to the public API that does **not** break existing third party -code importing Zarr, or cause third party code to behave in a different way, is a -**backwards-compatible API change**. For example, adding a new function, class or method is usually -a backwards-compatible change. However, removing a function, class or method; removing an argument -to a function or method; adding a required argument to a function or method; or changing the -behaviour of a function or method, are examples of **backwards-incompatible API changes**. - -If a release contains no changes to the public API (e.g., contains only bug fixes or -other maintenance work), then the micro version number should be incremented (e.g., -2.2.0 -> 2.2.1). If a release contains public API changes, but all changes are -backwards-compatible, then the minor version number should be incremented -(e.g., 2.2.1 -> 2.3.0). If a release contains any backwards-incompatible public API changes, -the major version number should be incremented (e.g., 2.3.0 -> 3.0.0). - -Backwards-incompatible changes to the experimental API can be included in a minor release, -although this should be minimised if possible. I.e., it would be preferable to save up -backwards-incompatible changes to the experimental API to be included in a major release, and to -stabilise those features at the same time (i.e., move from experimental to public API), rather than -frequently tinkering with the experimental API in minor releases. - -Data format compatibility -""""""""""""""""""""""""" - -The data format used by Zarr is defined by a specification document, which should be -platform-independent and contain sufficient detail to construct an interoperable -software library to read and/or write Zarr data using any programming language. The -latest version of the specification document is available from the :ref:`spec` page. - -Here, **data format compatibility** means that all software libraries that implement a -particular version of the Zarr storage specification are interoperable, in the sense -that data written by any one library can be read by all others. It is obviously -desirable to maintain data format compatibility wherever possible. However, if a change -is needed to the storage specification, and that change would break data format -compatibility in any way, then the storage specification version number should be -incremented (e.g., 2 -> 3). - -The versioning of the Zarr software library is related to the versioning of the storage -specification as follows. A particular version of the Zarr library will -implement a particular version of the storage specification. For example, Zarr version -2.2.0 implements the Zarr storage specification version 2. If a release of the Zarr -library implements a different version of the storage specification, then the major -version number of the Zarr library should be incremented. E.g., if Zarr version 2.2.0 -implements the storage spec version 2, and the next release of the Zarr library -implements storage spec version 3, then the next library release should have version -number 3.0.0. Note however that the major version number of the Zarr library may not -always correspond to the spec version number. For example, Zarr versions 2.x, 3.x, and -4.x might all implement the same version of the storage spec and thus maintain data -format compatibility, although they will not maintain API compatibility. The version number -of the storage specification that is currently implemented is stored under the -``zarr.meta.ZARR_FORMAT`` variable. - -Note that the Zarr test suite includes a data fixture and tests to try and ensure that -data format compatibility is not accidentally broken. See the -:func:`test_format_compatibility` function in the :mod:`zarr.tests.test_storage` module -for details. - -When to make a release -~~~~~~~~~~~~~~~~~~~~~~ - -Ideally, any bug fixes that don't change the public API should be released as soon as -possible. It is fine for a micro release to contain only a single bug fix. - -When to make a minor release is at the discretion of the core developers. There are no -hard-and-fast rules, e.g., it is fine to make a minor release to make a single new -feature available; equally, it is fine to make a minor release that includes a number of -changes. - -Major releases obviously need to be given careful consideration, and should be done as -infrequently as possible, as they will break existing code and/or affect data -compatibility in some way. - -Release procedure -~~~~~~~~~~~~~~~~~ - -.. note:: - - Most of the release process is now handled by github workflow which should - automatically push a release to PyPI if a tag is pushed. - -Checkout and update the master branch:: - - $ git checkout master - $ git pull - -Verify all tests pass on all supported Python versions, and docs build:: - - $ tox - -Tag the version (where "X.X.X" stands for the version number, e.g., "2.2.0"):: - - $ version=X.X.X - $ git tag -a v$version -m v$version - $ git push origin v$version - -Create a GitHub release in order to generate the Zenodo DOI and -review the automatically generated zarr-feedstock PR. diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst new file mode 100644 index 0000000000..03388e1544 --- /dev/null +++ b/docs/developers/contributing.rst @@ -0,0 +1,374 @@ +.. _dev-guide-contributing: + +Contributing to Zarr +==================== + +Zarr is a community maintained project. We welcome contributions in the form of bug +reports, bug fixes, documentation, enhancement proposals and more. This page provides +information on how best to contribute. + +Asking for help +--------------- + +If you have a question about how to use Zarr, please post your question on +StackOverflow using the `"zarr" tag `_. +If you don't get a response within a day or two, feel free to raise a `GitHub issue +`_ including a link to your StackOverflow +question. We will try to respond to questions as quickly as possible, but please bear +in mind that there may be periods where we have limited time to answer questions +due to other commitments. + +Bug reports +----------- + +If you find a bug, please raise a `GitHub issue +`_. Please include the following items in +a bug report: + +1. A minimal, self-contained snippet of Python code reproducing the problem. You can + format the code nicely using markdown, e.g.:: + + + ```python + import zarr + g = zarr.group() + # etc. + ``` + +2. An explanation of why the current behaviour is wrong/not desired, and what you + expect instead. + +3. Information about the version of Zarr, along with versions of dependencies and the + Python interpreter, and installation information. The version of Zarr can be obtained + from the ``zarr.__version__`` property. Please also state how Zarr was installed, + e.g., "installed via pip into a virtual environment", or "installed using conda". + Information about other packages installed can be obtained by executing ``pip freeze`` + (if using pip to install packages) or ``conda env export`` (if using conda to install + packages) from the operating system command prompt. The version of the Python + interpreter can be obtained by running a Python interactive session, e.g.:: + + $ python + Python 3.12.7 | packaged by conda-forge | (main, Oct 4 2024, 15:57:01) [Clang 17.0.6 ] on darwin + +Enhancement proposals +--------------------- + +If you have an idea about a new feature or some other improvement to Zarr, please raise a +`GitHub issue `_ first to discuss. + +We very much welcome ideas and suggestions for how to improve Zarr, but please bear in +mind that we are likely to be conservative in accepting proposals for new features. The +reasons for this are that we would like to keep the Zarr code base lean and focused on +a core set of functionalities, and available time for development, review and maintenance +of new features is limited. But if you have a great idea, please don't let that stop +you from posting it on GitHub, just please don't be offended if we respond cautiously. + +Contributing code and/or documentation +-------------------------------------- + +Forking the repository +~~~~~~~~~~~~~~~~~~~~~~ + +The Zarr source code is hosted on GitHub at the following location: + +* `https://github.com/zarr-developers/zarr-python `_ + +You will need your own fork to work on the code. Go to the link above and hit +the `"Fork" `_ button. +Then clone your fork to your local machine:: + + $ git clone git@github.com:your-user-name/zarr-python.git + $ cd zarr-python + $ git remote add upstream git@github.com:zarr-developers/zarr-python.git + +Creating a development environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To work with the Zarr source code, it is recommended to use +`hatch `_ to create and manage development +environments. Hatch will automatically install all Zarr dependencies using the same +versions as are used by the core developers and continuous integration services. +Assuming you have a Python 3 interpreter already installed, and you have cloned the +Zarr source code and your current working directory is the root of the repository, +you can do something like the following:: + + $ pip install hatch + $ hatch env show # list all available environments + +To verify that your development environment is working, you can run the unit tests +for one of the test environments, e.g.:: + + $ hatch env run --env test.py3.12-2.1-optional run-pytest + +Creating a branch +~~~~~~~~~~~~~~~~~ + +Before you do any new work or submit a pull request, please open an issue on GitHub to +report the bug or propose the feature you'd like to add. + +It's best to synchronize your fork with the upstream repository, then create a +new, separate branch for each piece of work you want to do. E.g.:: + + git checkout main + git fetch upstream + git checkout -b shiny-new-feature upstream/main + git push -u origin shiny-new-feature + +This changes your working directory to the 'shiny-new-feature' branch. Keep any changes in +this branch specific to one bug or feature so it is clear what the branch brings to +Zarr. + +To update this branch with latest code from Zarr, you can retrieve the changes from +the main branch and perform a rebase:: + + git fetch upstream + git rebase upstream/main + +This will replay your commits on top of the latest Zarr git main. If this leads to +merge conflicts, these need to be resolved before submitting a pull request. +Alternatively, you can merge the changes in from upstream/main instead of rebasing, +which can be simpler:: + + git pull upstream main + +Again, any conflicts need to be resolved before submitting a pull request. + +Running the test suite +~~~~~~~~~~~~~~~~~~~~~~ + +Zarr includes a suite of unit tests. The simplest way to run the unit tests +is to activate your development environment +(see `creating a development environment`_ above) and invoke:: + + $ hatch env run --env test.py3.12-2.1-optional run-pytest + +All tests are automatically run via GitHub Actions for every pull +request and must pass before code can be accepted. Test coverage is +also collected automatically via the Codecov service. + +.. note:: + Previous versions of Zarr-Python made extensive use of doctests. These tests were + not maintained during the 3.0 refactor but may be brought back in the future. + See :issue:`2614` for more details. + +Code standards - using pre-commit +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All code must conform to the PEP8 standard. Regarding line length, lines up to 100 +characters are allowed, although please try to keep under 90 wherever possible. + +``Zarr`` uses a set of ``pre-commit`` hooks and the ``pre-commit`` bot to format, +type-check, and prettify the codebase. ``pre-commit`` can be installed locally by +running:: + + $ python -m pip install pre-commit + +The hooks can be installed locally by running:: + + $ pre-commit install + +This would run the checks every time a commit is created locally. These checks will also run +on every commit pushed to an open PR, resulting in some automatic styling fixes by the +``pre-commit`` bot. The checks will by default only run on the files modified by a commit, +but the checks can be triggered for all the files by running:: + + $ pre-commit run --all-files + +If you would like to skip the failing checks and push the code for further discussion, use +the ``--no-verify`` option with ``git commit``. + + +Test coverage +~~~~~~~~~~~~~ + +.. note:: + Test coverage for Zarr-Python 3 is currently not at 100%. This is a known issue and help + is welcome to bring test coverage back to 100%. See :issue:`2613` for more details. + +Zarr strives to maintain 100% test coverage under the latest Python stable release +Both unit tests and docstring doctests are included when computing coverage. Running:: + + $ hatch env run --env test.py3.12-2.1-optional run-coverage + +will automatically run the test suite with coverage and produce a XML coverage report. +This should be 100% before code can be accepted into the main code base. + +You can also generate an HTML coverage report by running:: + + $ hatch env run --env test.py3.12-2.1-optional run-coverage-html + +When submitting a pull request, coverage will also be collected across all supported +Python versions via the Codecov service, and will be reported back within the pull +request. Codecov coverage must also be 100% before code can be accepted. + +Documentation +~~~~~~~~~~~~~ + +Docstrings for user-facing classes and functions should follow the +`numpydoc +`_ +standard, including sections for Parameters and Examples. All examples +should run and pass as doctests under Python 3.11. + +Zarr uses Sphinx for documentation, hosted on readthedocs.org. Documentation is +written in the RestructuredText markup language (.rst files) in the ``docs`` folder. +The documentation consists both of prose and API documentation. All user-facing classes +and functions are included in the API documentation, under the ``docs/api`` folder +using the `autodoc `_ +extension to sphinx. Any new features or important usage information should be included in the +user-guide (``docs/user-guide``). Any changes should also be included as a new file in the +:file:`changes` directory. + +The documentation can be built locally by running:: + + $ hatch --env docs run build + +The resulting built documentation will be available in the ``docs/_build/html`` folder. + +Hatch can also be used to serve continuously updating version of the documentation +during development at `http://0.0.0.0:8000/ `_. This can be done by running:: + + $ hatch --env docs run serve + +.. _changelog: + +Changelog +~~~~~~~~~ + +zarr-python uses `towncrier`_ to manage release notes. Most pull requests should +include at least one news fragment describing the changes. To add a release +note, you'll need the GitHub issue or pull request number and the type of your +change (``feature``, ``bugfix``, ``doc``, ``removal``, ``misc``). With that, run +```towncrier create``` with your development environment, which will prompt you +for the issue number, change type, and the news text:: + + towncrier create + +Alternatively, you can manually create the files in the ``changes`` directory +using the naming convention ``{issue-number}.{change-type}.rst``. + +See the `towncrier`_ docs for more. + +.. _towncrier: https://towncrier.readthedocs.io/en/stable/tutorial.html + +The following information is mainly for core developers, but may also be of interest to +contributors. + +Merging pull requests +--------------------- + +Pull requests submitted by an external contributor should be reviewed and approved by at least +one core developer before being merged. Ideally, pull requests submitted by a core developer +should be reviewed and approved by at least one other core developer before being merged. + +Pull requests should not be merged until all CI checks have passed (GitHub Actions +Codecov) against code that has had the latest main merged in. + +Compatibility and versioning policies +------------------------------------- + +Versioning +~~~~~~~~~~ +Versions of this library are identified by a triplet of integers with the form +``..``, for example ``3.0.4``. A release of ``zarr-python`` is associated with a new +version identifier. That new identifier is generated by incrementing exactly one of the components of +the previous version identifier by 1. When incrementing the ``major`` component of the version identifier, +the ``minor`` and ``patch`` components is reset to 0. When incrementing the minor component, +the patch component is reset to 0. + +Releases are classified by the library changes contained in that release. This classification +determines which component of the version identifier is incremented on release. + +* ``major`` releases (for example, ``2.18.0`` -> ``3.0.0``) are for changes that will + require extensive adaptation efforts from many users and downstream projects. + For example, breaking changes to widely-used user-facing APIs should only be applied in a major release. + + + Users and downstream projects should carefully consider the impact of a major release before + adopting it. + In advance of a major release, developers should communicate the scope of the upcoming changes, + and help users prepare for them. + +* ``minor`` releases (or example, ``3.0.0`` -> ``3.1.0``) are for changes that do not require + significant effort from most users or downstream downstream projects to respond to. API changes + are possible in minor releases if the burden on users imposed by those changes is sufficiently small. + + For example, a recently released API may need fixes or refinements that are breaking, but low impact + due to the recency of the feature. Such API changes are permitted in a minor release. + + + Minor releases are safe for most users and downstream projects to adopt. + + +* ``patch`` releases (for example, ``3.1.0`` -> ``3.1.1``) are for changes that contain no breaking + or behaviour changes for downstream projects or users. Examples of changes suitable for a patch release are + bugfixes and documentation improvements. + + + Users should always feel safe upgrading to a the latest patch release. + +Note that this versioning scheme is not consistent with `Semantic Versioning `_. +Contrary to SemVer, the Zarr library may release breaking changes in ``minor`` releases, or even +``patch`` releases under exceptional circumstances. But we should strive to avoid doing so. + +A better model for our versioning scheme is `Intended Effort Versioning `_, +or "EffVer". The guiding principle off EffVer is to categorize releases based on the *expected effort +required to upgrade to that release*. + +Zarr developers should make changes as smooth as possible for users. This means making +backwards-compatible changes wherever possible. When a backwards-incompatible change is necessary, +users should be notified well in advance, e.g. via informative deprecation warnings. + +Data format compatibility +""""""""""""""""""""""""" + +The Zarr library is an implementation of a file format standard defined externally -- +see the `Zarr specifications website `_ for the list of +Zarr file format specifications. + + +If an existing Zarr format version changes, or a new version of the Zarr format is released, then +the Zarr library will generally require changes. It is very likely that a new Zarr format will +require extensive breaking changes to the Zarr library, and so support for a new Zarr format in the +Zarr library will almost certainly come in new ``major`` release. +When the Zarr library adds support for a new Zarr format, there may be a period of accelerated +changes as developers refine newly added APIs and deprecate old APIs. In such a transitional phase +breaking changes may be more frequent than usual. + + +Release procedure +----------------- + +Pre-release +~~~~~~~~~~~ +1. Make sure that all pull requests which will be included in the release + have been properly documented as changelog files in the :file:`changes/` directory. +2. Run ``towncrier build --version x.y.z`` to create the changelog, and commit the result + to the main branch. + +Releasing +~~~~~~~~~ +1. Go to https://github.com/zarr-developers/zarr-python/releases +2. Click "Draft a new release". +3. Choose a version number prefixed with a `v` (e.g. `v0.0.0`). + For pre-releases, include the appropriate suffix (e.g. `v0.0.0a1` or `v0.0.0rc2`). +4. Set the description of the release to:: + + See release notes https://zarr.readthedocs.io/en/stable/release-notes.html#release-0-0-0 + + replacing the correct version numbers. For pre-release versions, + the URL should omit the pre-release suffix, e.g. "a1" or "rc1". +5. Click on "Generate release notes" to auto-fill the description. + +After creating the release, the documentation will be built on +https://readthedocs.io. Full releases will be available under +`/stable `_ while +pre-releases will be available under +`/latest `_. + +Post-release +~~~~~~~~~~~~ + +- Review and merge the pull request on the + `conda-forge feedstock `_ that will be + automatically generated. diff --git a/docs/developers/index.rst b/docs/developers/index.rst new file mode 100644 index 0000000000..4bccb3a469 --- /dev/null +++ b/docs/developers/index.rst @@ -0,0 +1,9 @@ + +Developer's Guide +----------------- + +.. toctree:: + :maxdepth: 1 + + contributing + roadmap diff --git a/docs/developers/roadmap.rst b/docs/developers/roadmap.rst new file mode 100644 index 0000000000..d9fc32b775 --- /dev/null +++ b/docs/developers/roadmap.rst @@ -0,0 +1,696 @@ +Roadmap +======= + +- Status: active +- Author: Joe Hamman +- Created On: October 31, 2023 +- Input from: + + - Davis Bennett / @d-v-b + - Norman Rzepka / @normanrz + - Deepak Cherian @dcherian + - Brian Davis / @monodeldiablo + - Oliver McCormack / @olimcc + - Ryan Abernathey / @rabernat + - Jack Kelly / @JackKelly + - Martin Durrant / @martindurant + +.. note:: + + This document was written in the early stages of the 3.0 refactor. Some + aspects of the design have changed since this was originally written. + Questions and discussion about the contents of this document should be directed to + `this GitHub Discussion `__. + +Introduction +------------ + +This document lays out a design proposal for version 3.0 of the +`Zarr-Python `__ package. A +specific focus of the design is to bring Zarr-Python’s API up to date +with the `Zarr V3 +specification `__, +with the hope of enabling the development of the many features and +extensions that motivated the V3 Spec. The ideas presented here are +expected to result in a major release of Zarr-Python (version 3.0) +including significant a number of breaking API changes. For clarity, +“V3” will be used to describe the version of the Zarr specification and +“3.0” will be used to describe the release tag of the Zarr-Python +project. + +Current status of V3 in Zarr-Python +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +During the development of the V3 Specification, a `prototype +implementation `__ +was added to the Zarr-Python library. Since that implementation, the V3 +spec evolved in significant ways and as a result, the Zarr-Python +library is now out of sync with the approved spec. Downstream libraries +(e.g. `Xarray `__) have added support +for this implementation and will need to migrate to the accepted spec +when its available in Zarr-Python. + +Goals +----- + +- Provide a complete implementation of Zarr V3 through the Zarr-Python + API +- Clear the way for exciting extensions / ZEPs + (i.e. `sharding `__, + `variable chunking `__, + etc.) +- Provide a developer API that can be used to implement and register V3 + extensions +- Improve the performance of Zarr-Python by streamlining the interface + between the Store layer and higher level APIs (e.g. Groups and + Arrays) +- Clean up the internal and user facing APIs +- Improve code quality and robustness (e.g. achieve 100% type hint + coverage) +- Align the Zarr-Python array API with the `array API + Standard `__ + +Examples of what 3.0 will enable? +--------------------------------- + +1. Reading and writing V3 spec-compliant groups and arrays +2. V3 extensions including sharding and variable chunking. +3. Improved performance by leveraging concurrency when + creating/reading/writing to stores (imagine a + ``create_hierarchy(zarr_objects)`` function). +4. User-developed extensions (e.g. storage-transformers) can be + registered with Zarr-Python at runtime + +Non-goals (of this document) +---------------------------- + +- Implementation of any unaccepted Zarr V3 extensions +- Major revisions to the Zarr V3 spec + +Requirements +------------ + +1. Read and write spec compliant V2 and V3 data +2. Limit unnecessary traffic to/from the store +3. Cleanly define the Array/Group/Store abstractions +4. Cleanly define how V2 will be supported going forward +5. Provide a clear roadmap to help users upgrade to 3.0 +6. Developer tools / hooks for registering extensions + +Design +------ + +Async API +~~~~~~~~~ + +Zarr-Python is an IO library. As such, supporting concurrent action +against the storage layer is critical to achieving acceptable +performance. The Zarr-Python 2 was not designed with asynchronous +computation in mind and as a result has struggled to effectively +leverage the benefits of concurrency. At one point, ``getitems`` and +``setitems`` support was added to the Zarr store model but that is only +used for operating on a set of chunks in a single variable. + +With Zarr-Python 3.0, we have the opportunity to revisit this design. +The proposal here is as follows: + +1. The ``Store`` interface will be entirely async. +2. On top of the async ``Store`` interface, we will provide an + ``AsyncArray`` and ``AsyncGroup`` interface. +3. Finally, the primary user facing API will be synchronous ``Array`` + and ``Group`` classes that wrap the async equivalents. + +**Examples** + +- **Store** + + .. code:: python + + class Store: + ... + async def get(self, key: str) -> bytes: + ... + async def get_partial_values(self, key_ranges: List[Tuple[str, Tuple[int, Optional[int]]]]) -> bytes: + ... + # (no sync interface here) + +- **Array** + + .. code:: python + + class AsyncArray: + ... + + async def getitem(self, selection: Selection) -> np.ndarray: + # the core logic for getitem goes here + + class Array: + _async_array: AsyncArray + + def __getitem__(self, selection: Selection) -> np.ndarray: + return sync(self._async_array.getitem(selection)) + +- **Group** + + .. code:: python + + class AsyncGroup: + ... + + async def create_group(self, path: str, **kwargs) -> AsyncGroup: + # the core logic for create_group goes here + + class Group: + _async_group: AsyncGroup + + def create_group(self, path: str, **kwargs) -> Group: + return sync(self._async_group.create_group(path, **kwargs)) + + **Internal Synchronization API** + +With the ``Store`` and core ``AsyncArray``/ ``AsyncGroup`` classes being +predominantly async, Zarr-Python will need an internal API to provide a +synchronous API. The proposal here is to use the approach in +`fsspec `__ +to provide a high-level ``sync`` function that takes an ``awaitable`` +and runs it in its managed IO Loop / thread. + +| **FAQ** 1. Why two levels of Arrays/groups? a. First, this is an + intentional decision and departure from the current Zarrita + implementation b. The idea is that users rarely want to mix + interfaces. Either they are working within an async context (currently + quite rare) or they are in a typical synchronous context. c. Splitting + the two will allow us to clearly define behavior on the ``AsyncObj`` + and simply wrap it in the ``SyncObj``. 2. What if a store is only has + a synchronous backend? a. First off, this is expected to be a fairly + rare occurrence. Most storage backends have async interfaces. b. But + in the event a storage backend doesn’t have a async interface, there + is nothing wrong with putting synchronous code in ``async`` methods. + There are approaches to enabling concurrent action through wrappers + like AsyncIO’s ``loop.run_in_executor`` (`ref + 1 `__, + `ref 2 `__, `ref + 3 `__, + `ref + 4 `__. +| 3. Will Zarr help manage the async contexts encouraged by some + libraries + (e.g. `AioBotoCore `__)? + a. Many async IO libraries require entering an async context before + interacting with the API. We expect some experimentation to be needed + here but the initial design will follow something close to what fsspec + does (`example in + s3fs `__). + 4. Why not provide a synchronous Store interface? a. We could but this + design is simpler. It would mean supporting it in the ``AsyncGroup`` + and ``AsyncArray`` classes which, may be more trouble than its worth. + Storage backends that do not have an async API will be encouraged to + wrap blocking calls in an async wrapper + (e.g. ``loop.run_in_executor``). + +Store API +~~~~~~~~~ + +The ``Store`` API is specified directly in the V3 specification. All V3 +stores should implement this abstract API, omitting Write and List +support as needed. As described above, all stores will be expected to +expose the required methods as async methods. + +**Example** + +.. code:: python + + class ReadWriteStore: + ... + async def get(self, key: str) -> bytes: + ... + + async def get_partial_values(self, key_ranges: List[Tuple[str, int, int]) -> bytes: + ... + + async def set(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: + ... # required for writable stores + + async def set_partial_values(self, key_start_values: List[Tuple[str, int, Union[bytes, bytearray, memoryview]]]) -> None: + ... # required for writable stores + + async def list(self) -> List[str]: + ... # required for listable stores + + async def list_prefix(self, prefix: str) -> List[str]: + ... # required for listable stores + + async def list_dir(self, prefix: str) -> List[str]: + ... # required for listable stores + + # additional (optional methods) + async def getsize(self, prefix: str) -> int: + ... + + async def rename(self, src: str, dest: str) -> None + ... + + +Recognizing that there are many Zarr applications today that rely on the +``MutableMapping`` interface supported by Zarr-Python 2, a wrapper store +will be developed to allow existing stores to plug directly into this +API. + +Array API +~~~~~~~~~ + +The user facing array interface will implement a subset of the `Array +API Standard `__. Most of the +computational parts of the Array API Standard don’t fit into Zarr right +now. That’s okay. What matters most is that we ensure we can give +downstream applications a compliant API. + +*Note, Zarr already does most of this so this is more about formalizing +the relationship than a substantial change in API.* + ++------------------------+------------------------+-------------------------+-------------------------+ +| | Included | Not Included | Unknown / Maybe Possible| ++========================+========================+=========================+=========================+ +| **Attributes** | ``dtype`` | ``mT`` | ``device`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``ndim`` | ``T`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``shape`` | | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``size`` | | | ++------------------------+------------------------+-------------------------+-------------------------+ +| **Methods** | ``__getitem__`` | ``__array_namespace__`` | ``to_device`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``__setitem__`` | ``__abs__`` | ``__bool__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``__eq__`` | ``__add__`` | ``__complex__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``__bool__`` | ``__and__`` | ``__dlpack__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__floordiv__`` | ``__dlpack_device__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__ge__`` | ``__float__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__gt__`` | ``__index__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__invert__`` | ``__int__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__le__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__lshift__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__lt__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__matmul__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__mod__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__mul__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__ne__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__neg__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__or__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__pos__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__pow__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__rshift__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__sub__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__truediv__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__xor__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| **Creation functions** | ``zeros`` | | ``arange`` | +| (``zarr.creation``) | | | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``zeros_like`` | | ``asarray`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``ones`` | | ``eye`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``ones_like`` | | ``from_dlpack`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``full`` | | ``linspace`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``full_like`` | | ``meshgrid`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``empty`` | | ``tril`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``empty_like`` | | ``triu`` | ++------------------------+------------------------+-------------------------+-------------------------+ + +In addition to the core array API defined above, the Array class should +have the following Zarr specific properties: + +- ``.metadata`` (see Metadata Interface below) +- ``.attrs`` - (pulled from metadata object) +- ``.info`` - (repolicated from existing property †) + +*† In Zarr-Python 2, the info property listed the store to identify +initialized chunks. By default this will be turned off in 3.0 but will +be configurable.* + +**Indexing** + +Zarr-Python currently supports ``__getitem__`` style indexing and the +special ``oindex`` and ``vindex`` indexers. These are not part of the +current Array API standard (see +`data-apis/array-api#669 `__) +but they have been `proposed as a +NEP `__. +Zarr-Python will maintain these in 3.0. + +We are also exploring a new high-level indexing API that will enabled +optimized batch/concurrent loading of many chunks. We expect this to be +important to enable performant loading of data in the context of +sharding. See `this +discussion `__ +for more detail. + +Concurrent indexing across multiple arrays will be possible using the +AsyncArray API. + +**Async and Sync Array APIs** + +Most the logic to support Zarr Arrays will live in the ``AsyncArray`` +class. There are a few notable differences that should be called out. + +=============== ============ +Sync Method Async Method +=============== ============ +``__getitem__`` ``getitem`` +``__setitem__`` ``setitem`` +``__eq__`` ``equals`` +=============== ============ + +**Metadata interface** + +Zarr-Python 2.\* closely mirrors the V2 spec metadata schema in the +Array and Group classes. In 3.0, we plan to move the underlying metadata +representation to a separate interface (e.g. ``Array.metadata``). This +interface will return either a ``V2ArrayMetadata`` or +``V3ArrayMetadata`` object (both will inherit from a parent +``ArrayMetadataABC`` class. The ``V2ArrayMetadata`` and +``V3ArrayMetadata`` classes will be responsible for producing valid JSON +representations of their metadata, and yielding a consistent view to the +``Array`` or ``Group`` class. + +Group API +~~~~~~~~~ + +The main question is how closely we should follow the existing +Zarr-Python implementation / ``MutableMapping`` interface. The table +below shows the primary ``Group`` methods in Zarr-Python 2 and attempts +to identify if and how they would be implemented in 3.0. + ++---------------------+------------------+------------------+-----------------------+ +| V2 Group Methods | ``AsyncGroup`` | ``Group`` | ``h5py_compat.Group`` | ++=====================+==================+==================+=======================+ +| ``__len__`` | ``length`` | ``__len__`` | ``__len__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``__iter__`` | ``__aiter__`` | ``__iter__`` | ``__iter__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``__contains__`` | ``contains`` | ``__contains__`` | ``__contains__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``__getitem__`` | ``getitem`` | ``__getitem__`` | ``__getitem__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``__enter__`` | N/A | N/A | ``__enter__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``__exit__`` | N/A | N/A | ``__exit__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``group_keys`` | ``group_keys`` | ``group_keys`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``groups`` | ``groups`` | ``groups`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``array_keys`` | ``array_key`` | ``array_keys`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``arrays`` | ``arrays`` | ``arrays`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``visit`` | ? | ? | ``visit`` | ++---------------------+------------------+------------------+-----------------------+ +| ``visitkeys`` | ? | ? | ? | ++---------------------+------------------+------------------+-----------------------+ +| ``visitvalues`` | ? | ? | ? | ++---------------------+------------------+------------------+-----------------------+ +| ``visititems`` | ? | ? | ``visititems`` | ++---------------------+------------------+------------------+-----------------------+ +| ``tree`` | ``tree`` | ``tree`` | ``Both`` | ++---------------------+------------------+------------------+-----------------------+ +| ``create_group`` | ``create_group`` | ``create_group`` | ``create_group`` | ++---------------------+------------------+------------------+-----------------------+ +| ``require_group`` | N/A | N/A | ``require_group`` | ++---------------------+------------------+------------------+-----------------------+ +| ``create_groups`` | ? | ? | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``require_groups`` | ? | ? | ? | ++---------------------+------------------+------------------+-----------------------+ +| ``create_dataset`` | N/A | N/A | ``create_dataset`` | ++---------------------+------------------+------------------+-----------------------+ +| ``require_dataset`` | N/A | N/A | ``require_dataset`` | ++---------------------+------------------+------------------+-----------------------+ +| ``create`` | ``create_array`` | ``create_array`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``empty`` | ``empty`` | ``empty`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``zeros`` | ``zeros`` | ``zeros`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``ones`` | ``ones`` | ``ones`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``full`` | ``full`` | ``full`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``array`` | ``create_array`` | ``create_array`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``empty_like`` | ``empty_like`` | ``empty_like`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``zeros_like`` | ``zeros_like`` | ``zeros_like`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``ones_like`` | ``ones_like`` | ``ones_like`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``full_like`` | ``full_like`` | ``full_like`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``move`` | ``move`` | ``move`` | ``move`` | ++---------------------+------------------+------------------+-----------------------+ + +**``zarr.h5compat.Group``** +-- +Zarr-Python 2.\* made an attempt to align its API with that of +`h5py `__. With 3.0, we will +relax this alignment in favor of providing an explicit compatibility +module (``zarr.h5py_compat``). This module will expose the ``Group`` and +``Dataset`` APIs that map to Zarr-Python’s ``Group`` and ``Array`` +objects. + +Creation API +~~~~~~~~~~~~ + +Zarr-Python 2.\* bundles together the creation and serialization of Zarr +objects. Zarr-Python 3.\* will make it possible to create objects in +memory separate from serializing them. This will specifically enable +writing hierarchies of Zarr objects in a single batch step. For example: + +.. code:: python + + + arr1 = Array(shape=(10, 10), path="foo/bar", dtype="i4", store=store) + arr2 = Array(shape=(10, 10), path="foo/spam", dtype="f8", store=store) + + arr1.save() + arr2.save() + + # or equivalently + + zarr.save_many([arr1 ,arr2]) + +*Note: this batch creation API likely needs additional design effort +prior to implementation.* + +Plugin API +~~~~~~~~~~ + +Zarr V3 was designed to be extensible at multiple layers. Zarr-Python +will support these extensions through a combination of `Abstract Base +Classes `__ (ABCs) and +`Entrypoints `__. + +**ABCs** + +Zarr V3 will expose Abstract base classes for the following objects: + +- ``Store``, ``ReadStore``, ``ReadWriteStore``, ``ReadListStore``, and + ``ReadWriteListStore`` +- ``BaseArray``, ``SynchronousArray``, and ``AsynchronousArray`` +- ``BaseGroup``, ``SynchronousGroup``, and ``AsynchronousGroup`` +- ``Codec``, ``ArrayArrayCodec``, ``ArrayBytesCodec``, + ``BytesBytesCodec`` + +**Entrypoints** + +Lots more thinking here but the idea here is to provide entrypoints for +``data type``, ``chunk grid``, ``chunk key encoding``, ``codecs``, +``storage_transformers`` and ``stores``. These might look something +like: + +:: + + entry_points=""" + [zarr.codecs] + blosc_codec=codec_plugin:make_blosc_codec + zlib_codec=codec_plugin:make_zlib_codec + """ + +Python type hints and static analysis +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Target 100% Mypy coverage in 3.0 source. + +Observability +~~~~~~~~~~~~~ + +A persistent problem in Zarr-Python is diagnosing problems that span +many parts of the stack. To address this in 3.0, we will add a basic +logging framework that can be used to debug behavior at various levels +of the stack. We propose to add the separate loggers for the following +namespaces: + +- ``array`` +- ``group`` +- ``store`` +- ``codec`` + +These should be documented such that users know how to activate them and +developers know how to use them when developing extensions. + +Dependencies +~~~~~~~~~~~~ + +Today, Zarr-Python has the following required dependencies: + +.. code:: python + + dependencies = [ + 'asciitree', + 'numpy>=1.20,!=1.21.0', + 'fasteners', + 'numcodecs>=0.10.0', + ] + +What other dependencies should be considered? + +1. Attrs - Zarrita makes extensive use of the Attrs library +2. Fsspec - Zarrita has a hard dependency on Fsspec. This could be + easily relaxed though. + +Breaking changes relative to Zarr-Python 2.\* +--------------------------------------------- + +1. H5py compat moved to a stand alone module? +2. ``Group.__getitem__`` support moved to ``Group.members.__getitem__``? +3. Others? + +Open questions +-------------- + +1. How to treat V2 + + a. Note: Zarrita currently implements a separate ``V2Array`` and + ``V3Array`` classes. This feels less than ideal. + b. We could easily convert metadata from v2 to the V3 Array, but what + about writing? + c. Ideally, we don’t have completely separate code paths. But if its + too complicated to support both within one interface, its probably + better. + +2. How and when to remove the current implementation of V3. + + a. It’s hidden behind a hard-to-use feature flag so we probably don’t + need to do anything. + +3. How to model runtime configuration? +4. Which extensions belong in Zarr-Python and which belong in separate + packages? + + a. We don’t need to take a strong position on this here. It’s likely + that someone will want to put Sharding in. That will be useful to + develop in parallel because it will give us a good test case for + the plugin interface. + +Testing +------- + +Zarr-python 3.0 adds a major new dimension to Zarr: Async support. This +also comes with a compatibility risk, we will need to thoroughly test +support in key execution environments. Testing plan: - Reuse the +existing test suite for testing the ``v3`` API. - ``xfail`` tests that +expose breaking changes with ``3.0 - breaking change`` description. This +will help identify additional and/or unintentional breaking changes - +Rework tests that were only testing internal APIs. - Add a set of +functional / integration tests targeting real-world workflows in various +contexts (e.g. w/ Dask) + +Development process +------------------- + +Zarr-Python 3.0 will introduce a number of new APIs and breaking changes +to existing APIs. In order to facilitate ongoing support for Zarr-Python +2.*, we will take on the following development process: + +- Create a ``v3`` branch that can be use for developing the core + functionality apart from the ``main`` branch. This will allow us to + support ongoing work and bug fixes on the ``main`` branch. +- Put the ``3.0`` APIs inside a ``zarr.v3`` module. Imports from this + namespace will all be new APIs that users can develop and test + against once the ``v3`` branch is merged to ``main``. +- Kickstart the process by pulling in the current state of ``zarrita`` + - which has many of the features described in this design. +- Release a series of 2.\* releases with the ``v3`` namespace +- When ``v3`` is complete, move contents of ``v3`` to the package root + +**Milestones** + +Below are a set of specific milestones leading toward the completion of +this process. As work begins, we expect this list to grow in +specificity. + +1. Port current version of Zarrita to Zarr-Python +2. Formalize Async interface by splitting ``Array`` and ``Group`` + objects into Sync and Async versions +3. Implement “fancy” indexing operations on the ``AsyncArray`` +4. Implement an abstract base class for the ``Store`` interface and a + wrapper ``Store`` to make use of existing ``MutableMapping`` stores. +5. Rework the existing unit test suite to use the ``v3`` namespace. +6. Develop a plugin interface for extensions +7. Develop a set of functional and integration tests +8. Work with downstream libraries (Xarray, Dask, etc.) to test new APIs + +TODOs +----- + +The following subjects are not covered in detail above but perhaps +should be. Including them here so they are not forgotten. + +1. [Store] Should Zarr provide an API for caching objects after first + read/list/etc. Read only stores? +2. [Array] buffer protocol support +3. [Array] ``meta_array`` support +4. [Extensions] Define how Zarr-Python will consume the various plugin + types +5. [Misc] H5py compatibility requires a bit more work and a champion to + drive it forward. +6. [Misc] Define ``chunk_store`` API in 3.0 +7. [Misc] Define ``synchronizer`` API in 3.0 + +References +---------- + +1. `Zarr-Python + repository `__ +2. `Zarr core specification (version 3.0) — Zarr specs + documentation `__ +3. `Zarrita repository `__ +4. `Async-Zarr `__ +5. `Zarr-Python Discussion + Topic `__ diff --git a/docs/index.rst b/docs/index.rst index d75c159fd1..83d427e290 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,108 +1,113 @@ -.. zarr documentation master file, created by - sphinx-quickstart on Mon May 2 21:40:09 2016. +.. _zarr_docs_mainpage: -Zarr -==== +*********** +Zarr-Python +*********** -Zarr is a format for the storage of chunked, compressed, N-dimensional arrays. -These documents describe the Zarr format and its Python implementation. +.. toctree:: + :maxdepth: 1 + :hidden: -Highlights ----------- + quickstart + user-guide/index + API reference + release-notes + developers/index + about -* Create N-dimensional arrays with any NumPy dtype. -* Chunk arrays along any dimension. -* Compress and/or filter chunks using any NumCodecs_ codec. -* Store arrays in memory, on disk, inside a Zip file, on S3, ... -* Read an array concurrently from multiple threads or processes. -* Write to an array concurrently from multiple threads or processes. -* Organize arrays into hierarchies via groups. +**Version**: |version| -Status ------- +**Useful links**: +`Source Repository `_ | +`Issue Tracker `_ | +`Developer Chat `_ | +`Zarr specifications `_ -Zarr is still a young project. Feedback and bug reports are very welcome, please get in touch via -the `GitHub issue tracker `_. See -:doc:`contributing` for further information about contributing to Zarr. +Zarr-Python is a Python library for reading and writing Zarr groups and arrays. Highlights include: -Installation ------------- +* Specification support for both Zarr format 2 and 3. +* Create and read from N-dimensional arrays using NumPy-like semantics. +* Flexible storage enables reading and writing from local, cloud and in-memory stores. +* High performance: Enables fast I/O with support for asynchronous I/O and multi-threading. +* Extensible: Customizable with user-defined codecs and stores. -Zarr depends on NumPy. It is generally best to `install NumPy -`_ first using whatever method is most -appropriate for you operating system and Python distribution. Other dependencies should be -installed automatically if using one of the installation methods below. +.. grid:: 2 -Install Zarr from PyPI:: + .. grid-item-card:: + :img-top: _static/index_getting_started.svg - $ pip install zarr + Quick Start + ^^^^^^^^^^^ -Alternatively, install Zarr via conda:: + New to Zarr? Check out the quick start guide. It contains a brief + introduction to Zarr's main concepts and links to additional tutorials. - $ conda install -c conda-forge zarr + +++ -To install the latest development version of Zarr, you can use pip with the -latest GitHub master:: + .. button-ref:: quickstart + :expand: + :color: dark + :click-parent: - $ pip install git+https://github.com/zarr-developers/zarr-python.git + To the Quick Start -To work with Zarr source code in development, install from GitHub:: + .. grid-item-card:: + :img-top: _static/index_user_guide.svg - $ git clone --recursive https://github.com/zarr-developers/zarr-python.git - $ cd zarr-python - $ python setup.py install + Guide + ^^^^^ -To verify that Zarr has been fully installed, run the test suite:: + A detailed guide for how to use Zarr-Python. - $ pip install pytest - $ python -m pytest -v --pyargs zarr + +++ -Contents --------- + .. button-ref:: user-guide/index + :expand: + :color: dark + :click-parent: -.. toctree:: - :maxdepth: 2 + To the user guide + + .. grid-item-card:: + :img-top: _static/index_api.svg + + API Reference + ^^^^^^^^^^^^^ + + The reference guide contains a detailed description of the functions, + modules, and objects included in Zarr. The reference describes how the + methods work and which parameters can be used. It assumes that you have an + understanding of the key concepts. + + +++ - tutorial - api - spec - release - contributing + .. button-ref:: api/zarr/index + :expand: + :color: dark + :click-parent: -Projects using Zarr -------------------- + To the API reference -If you are using Zarr, we would `love to hear about it -`_. + .. grid-item-card:: + :img-top: _static/index_contribute.svg -Acknowledgments ---------------- + Contributor's Guide + ^^^^^^^^^^^^^^^^^^^ -The following people have contributed to the development of Zarr by contributing code, -documentation, code reviews, comments and/or ideas: + Want to contribute to Zarr? We welcome contributions in the form of bug reports, + bug fixes, documentation, enhancement proposals and more. The contributing guidelines + will guide you through the process of improving Zarr. -* :user:`Francesc Alted ` -* :user:`Martin Durant ` -* :user:`Stephan Hoyer ` -* :user:`John Kirkham ` -* :user:`Alistair Miles ` -* :user:`Mamy Ratsimbazafy ` -* :user:`Matthew Rocklin ` -* :user:`Vincent Schut ` -* :user:`Anthony Scopatz ` -* :user:`Prakhar Goel ` + +++ -Zarr is inspired by `HDF5 `_, `h5py -`_ and `bcolz `_. + .. button-ref:: developers/contributing + :expand: + :color: dark + :click-parent: -Development of Zarr is supported by the -`MRC Centre for Genomics and Global Health `_. + To the contributor's guide -Indices and tables ------------------- -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` +**Download documentation**: `PDF/Zipped HTML `_ -.. _NumCodecs: http://numcodecs.readthedocs.io/ +.. _NumCodecs: https://numcodecs.readthedocs.io diff --git a/docs/quickstart.rst b/docs/quickstart.rst new file mode 100644 index 0000000000..66bdae2a2e --- /dev/null +++ b/docs/quickstart.rst @@ -0,0 +1,209 @@ +.. only:: doctest + + >>> import shutil + >>> shutil.rmtree('data', ignore_errors=True) + >>> + >>> import numpy as np + >>> np.random.seed(0) + +Quickstart +========== + +Welcome to the Zarr-Python Quickstart guide! This page will help you get up and running with +the Zarr library in Python to efficiently manage and analyze multi-dimensional arrays. + +Zarr is a powerful library for storage of n-dimensional arrays, supporting chunking, +compression, and various backends, making it a versatile choice for scientific and +large-scale data. + +Installation +------------ + +Zarr requires Python 3.11 or higher. You can install it via `pip`: + +.. code-block:: bash + + pip install zarr + +or `conda`: + +.. code-block:: bash + + conda install --channel conda-forge zarr + +Creating an Array +----------------- + +To get started, you can create a simple Zarr array:: + + >>> import zarr + >>> import numpy as np + >>> + >>> # Create a 2D Zarr array + >>> z = zarr.create_array( + ... store="data/example-1.zarr", + ... shape=(100, 100), + ... chunks=(10, 10), + ... dtype="f4" + ... ) + >>> + >>> # Assign data to the array + >>> z[:, :] = np.random.random((100, 100)) + >>> z.info + Type : Array + Zarr format : 3 + Data type : DataType.float32 + Shape : (100, 100) + Chunk shape : (10, 10) + Order : C + Read-only : False + Store type : LocalStore + Codecs : [{'endian': }, {'level': 0, 'checksum': False}] + No. bytes : 40000 (39.1K) + +Here, we created a 2D array of shape ``(100, 100)``, chunked into blocks of +``(10, 10)``, and filled it with random floating-point data. This array was +written to a ``LocalStore`` in the ``data/example-1.zarr`` directory. + +Compression and Filters +~~~~~~~~~~~~~~~~~~~~~~~ + +Zarr supports data compression and filters. For example, to use Blosc compression:: + + >>> z = zarr.create_array( + ... "data/example-3.zarr", + ... mode="w", shape=(100, 100), + ... chunks=(10, 10), dtype="f4", + ... compressors=zarr.codecs.BloscCodec(cname="zstd", clevel=3, shuffle=zarr.codecs.BloscShuffle.shuffle) + ... ) + >>> z[:, :] = np.random.random((100, 100)) + >>> + >>> z.info + Type : Array + Zarr format : 3 + Data type : DataType.float32 + Shape : (100, 100) + Chunk shape : (10, 10) + Order : C + Read-only : False + Store type : LocalStore + Codecs : [{'endian': }, {'level': 0, 'checksum': False}] + No. bytes : 40000 (39.1K) + +This compresses the data using the Zstandard codec with shuffle enabled for better compression. + +Hierarchical Groups +------------------- + +Zarr allows you to create hierarchical groups, similar to directories:: + + >>> # Create nested groups and add arrays + >>> root = zarr.group("data/example-2.zarr") + >>> foo = root.create_group(name="foo") + >>> bar = root.create_array( + ... name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" + ... ) + >>> spam = foo.create_array(name="spam", shape=(10,), dtype="i4") + >>> + >>> # Assign values + >>> bar[:, :] = np.random.random((100, 10)) + >>> spam[:] = np.arange(10) + >>> + >>> # print the hierarchy + >>> root.tree() + / + ├── bar (100, 10) float32 + └── foo + └── spam (10,) int32 + + +This creates a group with two datasets: ``foo`` and ``bar``. + +Batch Hierarchy Creation +~~~~~~~~~~~~~~~~~~~~~~~~ + +Zarr provides tools for creating a collection of arrays and groups with a single function call. +Suppose we want to copy existing groups and arrays into a new storage backend: + + >>> # Create nested groups and add arrays + >>> root = zarr.group("data/example-3.zarr", attributes={'name': 'root'}) + >>> foo = root.create_group(name="foo") + >>> bar = root.create_array( + ... name="bar", shape=(100, 10), chunks=(10, 10), dtype="f4" + ... ) + >>> nodes = {'': root.metadata} | {k: v.metadata for k,v in root.members()} + >>> print(nodes) + >>> from zarr.storage import MemoryStore + >>> new_nodes = dict(zarr.create_hierarchy(store=MemoryStore(), nodes=nodes)) + >>> new_root = new_nodes[''] + >>> assert new_root.attrs == root.attrs + +Note that :func:`zarr.create_hierarchy` will only initialize arrays and groups -- copying array data must +be done in a separate step. + +Persistent Storage +------------------ + +Zarr supports persistent storage to disk or cloud-compatible backends. While examples above +utilized a :class:`zarr.storage.LocalStore`, a number of other storage options are available. + +Zarr integrates seamlessly with cloud object storage such as Amazon S3 and Google Cloud Storage +using external libraries like `s3fs `_ or +`gcsfs `_:: + + >>> import s3fs # doctest: +SKIP + >>> + >>> z = zarr.create_array("s3://example-bucket/foo", mode="w", shape=(100, 100), chunks=(10, 10), dtype="f4") # doctest: +SKIP + >>> z[:, :] = np.random.random((100, 100)) # doctest: +SKIP + +A single-file store can also be created using the the :class:`zarr.storage.ZipStore`:: + + >>> # Store the array in a ZIP file + >>> store = zarr.storage.ZipStore("data/example-3.zip", mode='w') + >>> + >>> z = zarr.create_array( + ... store=store, + ... mode="w", + ... shape=(100, 100), + ... chunks=(10, 10), + ... dtype="f4" + ... ) + >>> + >>> # write to the array + >>> z[:, :] = np.random.random((100, 100)) + >>> + >>> # the ZipStore must be explicitly closed + >>> store.close() + +To open an existing array from a ZIP file:: + + >>> # Open the ZipStore in read-only mode + >>> store = zarr.storage.ZipStore("data/example-3.zip", read_only=True) + >>> + >>> z = zarr.open_array(store, mode='r') + >>> + >>> # read the data as a NumPy Array + >>> z[:] + array([[0.66734236, 0.15667458, 0.98720884, ..., 0.36229587, 0.67443246, + 0.34315267], + [0.65787303, 0.9544212 , 0.4830079 , ..., 0.33097172, 0.60423803, + 0.45621237], + [0.27632037, 0.9947008 , 0.42434934, ..., 0.94860053, 0.6226942 , + 0.6386924 ], + ..., + [0.12854576, 0.934397 , 0.19524333, ..., 0.11838563, 0.4967675 , + 0.43074256], + [0.82029045, 0.4671437 , 0.8090906 , ..., 0.7814118 , 0.42650765, + 0.95929915], + [0.4335856 , 0.7565437 , 0.7828931 , ..., 0.48119593, 0.66220033, + 0.6652362 ]], shape=(100, 100), dtype=float32) + +Read more about Zarr's storage options in the :ref:`User Guide `. + +Next Steps +---------- + +Now that you're familiar with the basics, explore the following resources: + +- `User Guide `_ +- `API Reference `_ diff --git a/docs/release-notes.rst b/docs/release-notes.rst new file mode 100644 index 0000000000..a89046dd6d --- /dev/null +++ b/docs/release-notes.rst @@ -0,0 +1,269 @@ +Release notes +============= + +.. towncrier release notes start + +3.0.8 (2025-05-19) +------------------ + +.. warning:: + + In versions 3.0.0 to 3.0.7 opening arrays or groups with ``mode='a'`` (the default for many builtin functions) + would cause any existing paths in the store to be deleted. This is fixed in 3.0.8, and + we recommend all users upgrade to avoid this bug that could cause unintentional data loss. + +Features +~~~~~~~~ + +- Added a `print_debug_info` function for bug reports. (:issue:`2913`) + + +Bugfixes +~~~~~~~~ + +- Fix a bug that prevented the number of initialized chunks being counted properly. (:issue:`2862`) +- Fixed sharding with GPU buffers. (:issue:`2978`) +- Fix structured `dtype` fill value serialization for consolidated metadata (:issue:`2998`) +- It is now possible to specify no compressor when creating a zarr format 2 array. + This can be done by passing ``compressor=None`` to the various array creation routines. + + The default behaviour of automatically choosing a suitable default compressor remains if the compressor argument is not given. + To reproduce the behaviour in previous zarr-python versions when ``compressor=None`` was passed, pass ``compressor='auto'`` instead. (:issue:`3039`) +- Fixed the typing of ``dimension_names`` arguments throughout so that it now accepts iterables that contain `None` alongside `str`. (:issue:`3045`) +- Using various functions to open data with ``mode='a'`` no longer deletes existing data in the store. (:issue:`3062`) +- Internally use `typesize` constructor parameter for :class:`numcodecs.blosc.Blosc` to improve compression ratios back to the v2-package levels. (:issue:`2962`) +- Specifying the memory order of Zarr format 2 arrays using the ``order`` keyword argument has been fixed. (:issue:`2950`) + + +Misc +~~~~ + +- :issue:`2972`, :issue:`3027`, :issue:`3049` + + +3.0.7 (2025-04-22) +------------------ + +Features +~~~~~~~~ + +- Add experimental ObjectStore storage class based on obstore. (:issue:`1661`) +- Add ``zarr.from_array`` using concurrent streaming of source data (:issue:`2622`) + + +Bugfixes +~~~~~~~~ + +- 0-dimensional arrays are now returning a scalar. Therefore, the return type of ``__getitem__`` changed + to NDArrayLikeOrScalar. This change is to make the behavior of 0-dimensional arrays consistent with + ``numpy`` scalars. (:issue:`2718`) +- Fix `fill_value` serialization for `NaN` in `ArrayV2Metadata` and add property-based testing of round-trip serialization (:issue:`2802`) +- Fixes `ConsolidatedMetadata` serialization of `nan`, `inf`, and `-inf` to be + consistent with the behavior of `ArrayMetadata`. (:issue:`2996`) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Updated the 3.0 migration guide to include the removal of "." syntax for getting group members. (:issue:`2991`, :issue:`2997`) + + +Misc +~~~~ +- Define a new versioning policy based on Effective Effort Versioning. This replaces the old Semantic + Versioning-based policy. (:issue:`2924`, :issue:`2910`) +- Make warning filters in the tests more specific, so warnings emitted by tests added in the future + are more likely to be caught instead of ignored. (:issue:`2714`) +- Avoid an unnecessary memory copy when writing Zarr to a local file (:issue:`2944`) + + +3.0.6 (2025-03-20) +------------------ + +Bugfixes +~~~~~~~~ + +- Restore functionality of `del z.attrs['key']` to actually delete the key. (:issue:`2908`) + + +3.0.5 (2025-03-07) +------------------ + +Bugfixes +~~~~~~~~ + +- Fixed a bug where ``StorePath`` creation would not apply standard path normalization to the ``path`` parameter, + which led to the creation of arrays and groups with invalid keys. (:issue:`2850`) +- Prevent update_attributes calls from deleting old attributes (:issue:`2870`) + + +Misc +~~~~ + +- :issue:`2796` + +3.0.4 (2025-02-23) +------------------ + +Features +~~~~~~~~ + +- Adds functions for concurrently creating multiple arrays and groups. (:issue:`2665`) + +Bugfixes +~~~~~~~~ + +- Fixed a bug where ``ArrayV2Metadata`` could save ``filters`` as an empty array. (:issue:`2847`) +- Fix a bug when setting values of a smaller last chunk. (:issue:`2851`) + +Misc +~~~~ + +- :issue:`2828` + + +3.0.3 (2025-02-14) +------------------ + +Features +~~~~~~~~ + +- Improves performance of FsspecStore.delete_dir for remote filesystems supporting concurrent/batched deletes, e.g., s3fs. (:issue:`2661`) +- Added :meth:`zarr.config.enable_gpu` to update Zarr's configuration to use GPUs. (:issue:`2751`) +- Avoid reading chunks during writes where possible. :issue:`757` (:issue:`2784`) +- :py:class:`LocalStore` learned to ``delete_dir``. This makes array and group deletes more efficient. (:issue:`2804`) +- Add `zarr.testing.strategies.array_metadata` to generate ArrayV2Metadata and ArrayV3Metadata instances. (:issue:`2813`) +- Add arbitrary `shards` to Hypothesis strategy for generating arrays. (:issue:`2822`) + + +Bugfixes +~~~~~~~~ + +- Fixed bug with Zarr using device memory, instead of host memory, for storing metadata when using GPUs. (:issue:`2751`) +- The array returned by ``zarr.empty`` and an empty ``zarr.core.buffer.cpu.NDBuffer`` will now be filled with the + specified fill value, or with zeros if no fill value is provided. + This fixes a bug where Zarr format 2 data with no fill value was written with un-predictable chunk sizes. (:issue:`2755`) +- Fix zip-store path checking for stores with directories listed as files. (:issue:`2758`) +- Use removeprefix rather than replace when removing filename prefixes in `FsspecStore.list` (:issue:`2778`) +- Enable automatic removal of `needs release notes` with labeler action (:issue:`2781`) +- Use the proper label config (:issue:`2785`) +- Alters the behavior of ``create_array`` to ensure that any groups implied by the array's name are created if they do not already exist. Also simplifies the type signature for any function that takes an ArrayConfig-like object. (:issue:`2795`) +- Enitialise empty chunks to the default fill value during writing and add default fill values for datetime, timedelta, structured, and other (void* fixed size) data types (:issue:`2799`) +- Ensure utf8 compliant strings are used to construct numpy arrays in property-based tests (:issue:`2801`) +- Fix pickling for ZipStore (:issue:`2807`) +- Update numcodecs to not overwrite codec configuration ever. Closes :issue:`2800`. (:issue:`2811`) +- Fix fancy indexing (e.g. arr[5, [0, 1]]) with the sharding codec (:issue:`2817`) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Added new user guide on :ref:`user-guide-gpu`. (:issue:`2751`) + + +3.0.2 (2025-01-31) +------------------ + +Features +~~~~~~~~ + +- Test ``getsize()`` and ``getsize_prefix()`` in ``StoreTests``. (:issue:`2693`) +- Test that a ``ValueError`` is raised for invalid byte range syntax in ``StoreTests``. (:issue:`2693`) +- Separate instantiating and opening a store in ``StoreTests``. (:issue:`2693`) +- Add a test for using Stores as a context managers in ``StoreTests``. (:issue:`2693`) +- Implemented ``LogingStore.open()``. (:issue:`2693`) +- ``LoggingStore`` is now a generic class. (:issue:`2693`) +- Change StoreTest's ``test_store_repr``, ``test_store_supports_writes``, + ``test_store_supports_partial_writes``, and ``test_store_supports_listing`` + to to be implemented using ``@abstractmethod``, rather raising ``NotImplementedError``. (:issue:`2693`) +- Test the error raised for invalid buffer arguments in ``StoreTests``. (:issue:`2693`) +- Test that data can be written to a store that's not yet open using the store.set method in ``StoreTests``. (:issue:`2693`) +- Adds a new function ``init_array`` for initializing an array in storage, and refactors ``create_array`` + to use ``init_array``. ``create_array`` takes two new parameters: ``data``, an optional array-like object, and ``write_data``, a bool which defaults to ``True``. + If ``data`` is given to ``create_array``, then the ``dtype`` and ``shape`` attributes of ``data`` are used to define the + corresponding attributes of the resulting Zarr array. Additionally, if ``data`` given and ``write_data`` is ``True``, + then the values in ``data`` will be written to the newly created array. (:issue:`2761`) + + +Bugfixes +~~~~~~~~ + +- Wrap sync fsspec filesystems with ``AsyncFileSystemWrapper``. (:issue:`2533`) +- Added backwards compatibility for Zarr format 2 structured arrays. (:issue:`2681`) +- Update equality for ``LoggingStore`` and ``WrapperStore`` such that 'other' must also be a ``LoggingStore`` or ``WrapperStore`` respectively, rather than only checking the types of the stores they wrap. (:issue:`2693`) +- Ensure that ``ZipStore`` is open before getting or setting any values. (:issue:`2693`) +- Use stdout rather than stderr as the default stream for ``LoggingStore``. (:issue:`2693`) +- Match the errors raised by read only stores in ``StoreTests``. (:issue:`2693`) +- Fixed ``ZipStore`` to make sure the correct attributes are saved when instances are pickled. + This fixes a previous bug that prevent using ``ZipStore`` with a ``ProcessPoolExecutor``. (:issue:`2762`) +- Updated the optional test dependencies to include ``botocore`` and ``fsspec``. (:issue:`2768`) +- Fixed the fsspec tests to skip if ``botocore`` is not installed. + Previously they would have failed with an import error. (:issue:`2768`) +- Optimize full chunk writes. (:issue:`2782`) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Changed the machinery for creating changelog entries. + Now individual entries should be added as files to the `changes` directory in the `zarr-python` repository, instead of directly to the changelog file. (:issue:`2736`) + +Other +~~~~~ + +- Created a type alias ``ChunkKeyEncodingLike`` to model the union of ``ChunkKeyEncoding`` instances and the dict form of the + parameters of those instances. ``ChunkKeyEncodingLike`` should be used by high-level functions to provide a convenient + way for creating ``ChunkKeyEncoding`` objects. (:issue:`2763`) + + +3.0.1 (Jan. 17, 2025) +--------------------- + +* Implement ``zarr.from_array`` using concurrent streaming (:issue:`2622`). + +Bug fixes +~~~~~~~~~ +* Fixes ``order`` argument for Zarr format 2 arrays (:issue:`2679`). + +* Fixes a bug that prevented reading Zarr format 2 data with consolidated + metadata written using ``zarr-python`` version 2 (:issue:`2694`). + +* Ensure that compressor=None results in no compression when writing Zarr + format 2 data (:issue:`2708`). + +* Fix for empty consolidated metadata dataset: backwards compatibility with + Zarr-Python 2 (:issue:`2695`). + +Documentation +~~~~~~~~~~~~~ +* Add v3.0.0 release announcement banner (:issue:`2677`). + +* Quickstart guide alignment with V3 API (:issue:`2697`). + +* Fix doctest failures related to numcodecs 0.15 (:issue:`2727`). + +Other +~~~~~ +* Removed some unnecessary files from the source distribution + to reduce its size. (:issue:`2686`). + +* Enable codecov in GitHub actions (:issue:`2682`). + +* Speed up hypothesis tests (:issue:`2650`). + +* Remove multiple imports for an import name (:issue:`2723`). + + +.. _release_3.0.0: + +3.0.0 (Jan. 9, 2025) +-------------------- + +3.0.0 is a new major release of Zarr-Python, with many breaking changes. +See the :ref:`v3 migration guide` for a listing of what's changed. + +Normal release note service will resume with further releases in the 3.0.0 +series. + +Release notes for the zarr-python 2.x and 1.x releases can be found here: +https://zarr.readthedocs.io/en/support-v2/release.html diff --git a/docs/release.rst b/docs/release.rst deleted file mode 100644 index cd782311d1..0000000000 --- a/docs/release.rst +++ /dev/null @@ -1,1032 +0,0 @@ -Release notes -============= - -.. _unreleased: - -Unreleased ----------- - -.. _release_2.9.5: - -2.9.5 ------ - -Bug fixes -~~~~~~~~~ - -* Fix FSStore.listdir behavior for nested directories. - By :user:`Greggory Lee `; :issue:`802`. - -.. _release_2.9.4: - -2.9.4 ------ - -Bug fixes -~~~~~~~~~ - -* Fix structured arrays that contain objects - By :user: `Attila Bergou `; :issue: `806` - -.. _release_2.9.3: - -2.9.3 ------ - -Maintenance -~~~~~~~~~~~ - -* Mark the fact that some tests that require ``fsspec``, without compromising the code coverage score. - By :user:`Ben Williams `; :issue:`823`. - -* Only inspect alternate node type if desired isn't present. - By :user:`Trevor Manz `; :issue:`696`. - -.. _release_2.9.2: - -2.9.2 ------ - -Maintenance -~~~~~~~~~~~ - -* Correct conda-forge deployment of Zarr by fixing some Zarr tests. - By :user:`Ben Williams `; :issue:`821`. - -.. _release_2.9.1: - -2.9.1 ------ - -Maintenance -~~~~~~~~~~~ - -* Correct conda-forge deployment of Zarr. - By :user:`Josh Moore `; :issue:`XXX`. - -.. _release_2.9.0: - -2.9.0 ------ - -This release of Zarr Python is the first release of Zarr to not support Python 3.6. - -Enhancements -~~~~~~~~~~~~ - -* Update ABSStore for compatibility with newer `azure.storage.blob`. - By :user:`Tom Augspurger `; :issue:`759`. - -* Pathlib support. - By :user:`Chris Barnes `; :issue:`768`. - -Documentation -~~~~~~~~~~~~~ - -* Clarify that arbitrary key/value pairs are OK for attributes. - By :user:`Stephan Hoyer `; :issue:`751`. - -* Clarify how to manually convert a DirectoryStore to a ZipStore. - By :user:`pmav99 `; :issue:`763`. - -Bug fixes -~~~~~~~~~ - -* Fix dimension_separator support. - By :user:`Josh Moore `; :issue:`775`. - -* Extract ABSStore to zarr._storage.absstore. - By :user:`Josh Moore `; :issue:`781`. - -* avoid NumPy 1.21.0 due to https://github.com/numpy/numpy/issues/19325 - By :user:`Greggory Lee `; :issue:`791`. - -Maintenance -~~~~~~~~~~~ - -* Drop 3.6 builds. - By :user:`Josh Moore `; :issue:`774`, :issue:`778`. - -* Fix build with Sphinx 4. - By :user:`Elliott Sales de Andrade `; :issue:`799`. - -* TST: add missing assert in test_hexdigest. - By :user:`Greggory Lee `; :issue:`801`. - -.. _release_2.8.3: - -2.8.3 ------ - -Bug fixes -~~~~~~~~~ - -* FSStore: default to normalize_keys=False - By :user:`Josh Moore `; :issue:`755`. -* ABSStore: compatibility with ``azure.storage.python>=12`` - By :user:`Tom Augspurger `; :issue:`618` - - -.. _release_2.8.2: - -2.8.2 ------ - -Documentation -~~~~~~~~~~~~~ - -* Add section on rechunking to tutorial - By :user:`David Baddeley `; :issue:`730`. - -Bug fixes -~~~~~~~~~ - -* Expand FSStore tests and fix implementation issues - By :user:`Davis Bennett `; :issue:`709`. - -Maintenance -~~~~~~~~~~~ - -* Updated ipytree warning for jlab3 - By :user:`Ian Hunt-Isaak `; :issue:`721`. - -* b170a48a - (issue-728, copy-nested) Updated ipytree warning for jlab3 (#721) (3 weeks ago) -* Activate dependabot - By :user:`Josh Moore `; :issue:`734`. - -* Update Python classifiers (Zarr is stable!) - By :user:`Josh Moore `; :issue:`731`. - -.. _release_2.8.1: - -2.8.1 ------ - -Bug fixes -~~~~~~~~~ - -* raise an error if create_dataset's dimension_separator is inconsistent - By :user:`Gregory R. Lee `; :issue:`724`. - -.. _release_2.8.0: - -2.8.0 ------ - -V2 Specification Update -~~~~~~~~~~~~~~~~~~~~~~~ - -* Introduce optional dimension_separator .zarray key for nested chunks. - By :user:`Josh Moore `; :issue:`715`, :issue:`716`. - -.. _release_2.7.1: - -2.7.1 ------ - -Bug fixes -~~~~~~~~~ - -* Update Array to respect FSStore's key_separator (#718) - By :user:`Gregory R. Lee `; :issue:`718`. - -.. _release_2.7.0: - -2.7.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* Start stop for iterator (`islice()`) - By :user:`Sebastian Grill `; :issue:`621`. - -* Add capability to partially read and decompress chunks - By :user:`Andrew Fulton `; :issue:`667`. - -Bug fixes -~~~~~~~~~ - -* Make DirectoryStore __setitem__ resilient against antivirus file locking - By :user:`Eric Younkin `; :issue:`698`. - -* Compare test data's content generally - By :user:`John Kirkham `; :issue:`436`. - -* Fix dtype usage in zarr/meta.py - By :user:`Josh Moore `; :issue:`700`. - -* Fix FSStore key_seperator usage - By :user:`Josh Moore `; :issue:`669`. - -* Simplify text handling in DB Store - By :user:`John Kirkham `; :issue:`670`. - -* GitHub Actions migration - By :user:`Matthias Bussonnier `; - :issue:`641`, :issue:`671`, :issue:`674`, :issue:`676`, :issue:`677`, :issue:`678`, - :issue:`679`, :issue:`680`, :issue:`682`, :issue:`684`, :issue:`685`, :issue:`686`, - :issue:`687`, :issue:`695`, :issue:`706`. - -.. _release_2.6.1: - -2.6.1 ------ - -* Minor build fix - By :user:`Matthias Bussonnier `; :issue:`666`. - -.. _release_2.6.0: - -2.6.0 ------ - -This release of Zarr Python is the first release of Zarr to not support Python 3.5. - -* End Python 3.5 support. - By :user:`Chris Barnes `; :issue:`602`. - -* Fix ``open_group/open_array`` to allow opening of read-only store with - ``mode='r'`` :issue:`269` - -* Add `Array` tests for FSStore. - By :user:`Andrew Fulton `; :issue: `644`. - -* fix a bug in which ``attrs`` would not be copied on the root when using ``copy_all``; :issue:`613` - -* Fix ``FileNotFoundError`` with dask/s3fs :issue:`649` - -* Fix flaky fixture in test_storage.py :issue:`652` - -* Fix FSStore getitems fails with arrays that have a 0 length shape dimension :issue:`644` - -* Use async to fetch/write result concurrently when possible. :issue:`536`, See `this comment - `_ for some performance analysis - showing order of magnitude faster response in some benchmark. - -See `this link ` for the full list of closed and -merged PR tagged with the 2.6 milestone. - -* Add ability to partially read and decompress arrays, see :issue:`667`. It is - only available to chunks stored using fs-spec and using bloc as a compressor. - - For certain analysis case when only a small portion of chunks is needed it can - be advantageous to only access and decompress part of the chunks. Doing - partial read and decompression add high latency to many of the operation so - should be used only when the subset of the data is small compared to the full - chunks and is stored contiguously (that is to say either last dimensions for C - layout, firsts for F). Pass ``partial_decompress=True`` as argument when - creating an ``Array``, or when using ``open_array``. No option exists yet to - apply partial read and decompress on a per-operation basis. - -.. _release_2.5.0: - -2.5.0 ------ - -This release will be the last to support Python 3.5, next version of Zarr will be Python 3.6+. - -* `DirectoryStore` now uses `os.scandir`, which should make listing large store - faster, :issue:`563` - -* Remove a few remaining Python 2-isms. - By :user:`Poruri Sai Rahul `; :issue:`393`. - -* Fix minor bug in `N5Store`. - By :user:`gsakkis`, :issue:`550`. - -* Improve error message in Jupyter when trying to use the ``ipytree`` widget - without ``ipytree`` installed. - By :user:`Zain Patel `; :issue:`537` - -* Add typing informations to many of the core functions :issue:`589` - -* Explicitly close stores during testing. - By :user:`Elliott Sales de Andrade `; :issue:`442` - -* Many of the convenience functions to emit errors (``err_*`` from - ``zarr.errors`` have been replaced by ``ValueError`` subclasses. The corresponding - ``err_*`` function have been removed. :issue:`590`, :issue:`614`) - -* Improve consistency of terminology regarding arrays and datasets in the - documentation. - By :user:`Josh Moore `; :issue:`571`. - -* Added support for generic URL opening by ``fsspec``, where the URLs have the - form "protocol://[server]/path" or can be chained URls with "::" separators. - The additional argument ``storage_options`` is passed to the backend, see - the ``fsspec`` docs. - By :user:`Martin Durant `; :issue:`546` - -* Added support for fetching multiple items via ``getitems`` method of a - store, if it exists. This allows for concurrent fetching of data blocks - from stores that implement this; presently HTTP, S3, GCS. Currently only - applies to reading. - By :user:`Martin Durant `; :issue:`606` - -* Efficient iteration expanded with option to pass start and stop index via - ``array.islice``. - By :user:`Sebastian Grill `, :issue:`615`. - -.. _release_2.4.0: - -2.4.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* Add key normalization option for ``DirectoryStore``, ``NestedDirectoryStore``, - ``TempStore``, and ``N5Store``. - By :user:`James Bourbeau `; :issue:`459`. - -* Add ``recurse`` keyword to ``Group.array_keys`` and ``Group.arrays`` methods. - By :user:`James Bourbeau `; :issue:`458`. - -* Use uniform chunking for all dimensions when specifying ``chunks`` as an integer. - Also adds support for specifying ``-1`` to chunk across an entire dimension. - By :user:`James Bourbeau `; :issue:`456`. - -* Rename ``DictStore`` to ``MemoryStore``. - By :user:`James Bourbeau `; :issue:`455`. - -* Rewrite ``.tree()`` pretty representation to use ``ipytree``. - Allows it to work in both the Jupyter Notebook and JupyterLab. - By :user:`John Kirkham `; :issue:`450`. - -* Do not rename Blosc parameters in n5 backend and add `blocksize` parameter, - compatible with n5-blosc. By :user:`axtimwalde`, :issue:`485`. - -* Update ``DirectoryStore`` to create files with more permissive permissions. - By :user:`Eduardo Gonzalez ` and :user:`James Bourbeau `; :issue:`493` - -* Use ``math.ceil`` for scalars. - By :user:`John Kirkham `; :issue:`500`. - -* Ensure contiguous data using ``astype``. - By :user:`John Kirkham `; :issue:`513`. - -* Refactor out ``_tofile``/``_fromfile`` from ``DirectoryStore``. - By :user:`John Kirkham `; :issue:`503`. - -* Add ``__enter__``/``__exit__`` methods to ``Group`` for ``h5py.File`` compatibility. - By :user:`Chris Barnes `; :issue:`509`. - -Bug fixes -~~~~~~~~~ - -* Fix Sqlite Store Wrong Modification. - By :user:`Tommy Tran `; :issue:`440`. - -* Add intermediate step (using ``zipfile.ZipInfo`` object) to write - inside ``ZipStore`` to solve too restrictive permission issue. - By :user:`Raphael Dussin `; :issue:`505`. - -* Fix '/' prepend bug in ``ABSStore``. - By :user:`Shikhar Goenka `; :issue:`525`. - -Documentation -~~~~~~~~~~~~~ -* Fix hyperlink in ``README.md``. - By :user:`Anderson Banihirwe `; :issue:`531`. - -* Replace "nuimber" with "number". - By :user:`John Kirkham `; :issue:`512`. - -* Fix azure link rendering in tutorial. - By :user:`James Bourbeau `; :issue:`507`. - -* Update ``README`` file to be more detailed. - By :user:`Zain Patel `; :issue:`495`. - -* Import blosc from numcodecs in tutorial. - By :user:`James Bourbeau `; :issue:`491`. - -* Adds logo to docs. - By :user:`James Bourbeau `; :issue:`462`. - -* Fix N5 link in tutorial. - By :user:`James Bourbeau `; :issue:`480`. - -* Fix typo in code snippet. - By :user:`Joe Jevnik `; :issue:`461`. - -* Fix URLs to point to zarr-python - By :user:`John Kirkham `; :issue:`453`. - -Maintenance -~~~~~~~~~~~ - -* Add documentation build to CI. - By :user:`James Bourbeau `; :issue:`516`. - -* Use ``ensure_ndarray`` in a few more places. - By :user:`John Kirkham `; :issue:`506`. - -* Support Python 3.8. - By :user:`John Kirkham `; :issue:`499`. - -* Require Numcodecs 0.6.4+ to use text handling functionality from it. - By :user:`John Kirkham `; :issue:`497`. - -* Updates tests to use ``pytest.importorskip``. - By :user:`James Bourbeau `; :issue:`492` - -* Removed support for Python 2. - By :user:`jhamman`; :issue:`393`, :issue:`470`. - -* Upgrade dependencies in the test matrices and resolve a - compatibility issue with testing against the Azure Storage - Emulator. By :user:`alimanfoo`; :issue:`468`, :issue:`467`. - -* Use ``unittest.mock`` on Python 3. - By :user:`Elliott Sales de Andrade `; :issue:`426`. - -* Drop ``decode`` from ``ConsolidatedMetadataStore``. - By :user:`John Kirkham `; :issue:`452`. - - -.. _release_2.3.2: - -2.3.2 ------ - -Enhancements -~~~~~~~~~~~~ - -* Use ``scandir`` in ``DirectoryStore``'s ``getsize`` method. - By :user:`John Kirkham `; :issue:`431`. - -Bug fixes -~~~~~~~~~ - -* Add and use utility functions to simplify reading and writing JSON. - By :user:`John Kirkham `; :issue:`429`, :issue:`430`. - -* Fix ``collections``'s ``DeprecationWarning``\ s. - By :user:`John Kirkham `; :issue:`432`. - -* Fix tests on big endian machines. - By :user:`Elliott Sales de Andrade `; :issue:`427`. - - -.. _release_2.3.1: - -2.3.1 ------ - -Bug fixes -~~~~~~~~~ - -* Makes ``azure-storage-blob`` optional for testing. - By :user:`John Kirkham `; :issue:`419`, :issue:`420`. - - -.. _release_2.3.0: - -2.3.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* New storage backend, backed by Azure Blob Storage, class :class:`zarr.storage.ABSStore`. - All data is stored as block blobs. By :user:`Shikhar Goenka `, - :user:`Tim Crone ` and :user:`Zain Patel `; :issue:`345`. - -* Add "consolidated" metadata as an experimental feature: use - :func:`zarr.convenience.consolidate_metadata` to copy all metadata from the various - metadata keys within a dataset hierarchy under a single key, and - :func:`zarr.convenience.open_consolidated` to use this single key. This can greatly - cut down the number of calls to the storage backend, and so remove a lot of overhead - for reading remote data. - By :user:`Martin Durant `, :user:`Alistair Miles `, - :user:`Ryan Abernathey `, :issue:`268`, :issue:`332`, :issue:`338`. - -* Support has been added for structured arrays with sub-array shape and/or nested fields. By - :user:`Tarik Onalan `, :issue:`111`, :issue:`296`. - -* Adds the SQLite-backed :class:`zarr.storage.SQLiteStore` class enabling an - SQLite database to be used as the backing store for an array or group. - By :user:`John Kirkham `, :issue:`368`, :issue:`365`. - -* Efficient iteration over arrays by decompressing chunkwise. - By :user:`Jerome Kelleher `, :issue:`398`, :issue:`399`. - -* Adds the Redis-backed :class:`zarr.storage.RedisStore` class enabling a - Redis database to be used as the backing store for an array or group. - By :user:`Joe Hamman `, :issue:`299`, :issue:`372`. - -* Adds the MongoDB-backed :class:`zarr.storage.MongoDBStore` class enabling a - MongoDB database to be used as the backing store for an array or group. - By :user:`Noah D Brenowitz `, :user:`Joe Hamman `, - :issue:`299`, :issue:`372`, :issue:`401`. - -* **New storage class for N5 containers**. The :class:`zarr.n5.N5Store` has been - added, which uses :class:`zarr.storage.NestedDirectoryStore` to support - reading and writing from and to N5 containers. - By :user:`Jan Funke ` and :user:`John Kirkham `. - -Bug fixes -~~~~~~~~~ - -* The implementation of the :class:`zarr.storage.DirectoryStore` class has been modified to - ensure that writes are atomic and there are no race conditions where a chunk might appear - transiently missing during a write operation. By :user:`sbalmer `, :issue:`327`, - :issue:`263`. - -* Avoid raising in :class:`zarr.storage.DirectoryStore`'s ``__setitem__`` when file already exists. - By :user:`Justin Swaney `, :issue:`272`, :issue:`318`. - -* The required version of the `Numcodecs`_ package has been upgraded - to 0.6.2, which has enabled some code simplification and fixes a failing test involving - msgpack encoding. By :user:`John Kirkham `, :issue:`361`, :issue:`360`, :issue:`352`, - :issue:`355`, :issue:`324`. - -* Failing tests related to pickling/unpickling have been fixed. By :user:`Ryan Williams `, - :issue:`273`, :issue:`308`. - -* Corrects handling of ``NaT`` in ``datetime64`` and ``timedelta64`` in various - compressors (by :user:`John Kirkham `; :issue:`344`). - -* Ensure ``DictStore`` contains only ``bytes`` to facilitate comparisons and protect against writes. - By :user:`John Kirkham `, :issue:`350`. - -* Test and fix an issue (w.r.t. fill values) when storing complex data to ``Array``. - By :user:`John Kirkham `, :issue:`363`. - -* Always use a ``tuple`` when indexing a NumPy ``ndarray``. - By :user:`John Kirkham `, :issue:`376`. - -* Ensure when ``Array`` uses a ``dict``-based chunk store that it only contains - ``bytes`` to facilitate comparisons and protect against writes. Drop the copy - for the no filter/compressor case as this handles that case. - By :user:`John Kirkham `, :issue:`359`. - -Maintenance -~~~~~~~~~~~ - -* Simplify directory creation and removal in ``DirectoryStore.rename``. - By :user:`John Kirkham `, :issue:`249`. - -* CI and test environments have been upgraded to include Python 3.7, drop Python 3.4, and - upgrade all pinned package requirements. :user:`Alistair Miles `, :issue:`308`. - -* Start using pyup.io to maintain dependencies. - :user:`Alistair Miles `, :issue:`326`. - -* Configure flake8 line limit generally. - :user:`John Kirkham `, :issue:`335`. - -* Add missing coverage pragmas. - :user:`John Kirkham `, :issue:`343`, :issue:`355`. - -* Fix missing backslash in docs. - :user:`John Kirkham `, :issue:`254`, :issue:`353`. - -* Include tests for stores' ``popitem`` and ``pop`` methods. - By :user:`John Kirkham `, :issue:`378`, :issue:`380`. - -* Include tests for different compressors, endianness, and attributes. - By :user:`John Kirkham `, :issue:`378`, :issue:`380`. - -* Test validity of stores' contents. - By :user:`John Kirkham `, :issue:`359`, :issue:`408`. - - -.. _release_2.2.0: - -2.2.0 ------ - -Enhancements -~~~~~~~~~~~~ - -* **Advanced indexing**. The ``Array`` class has several new methods and - properties that enable a selection of items in an array to be retrieved or - updated. See the :ref:`tutorial_indexing` tutorial section for more - information. There is also a `notebook - `_ - with extended examples and performance benchmarks. :issue:`78`, :issue:`89`, - :issue:`112`, :issue:`172`. - -* **New package for compressor and filter codecs**. The classes previously - defined in the :mod:`zarr.codecs` module have been factored out into a - separate package called `Numcodecs`_. The `Numcodecs`_ package also includes - several new codec classes not previously available in Zarr, including - compressor codecs for Zstd and LZ4. This change is backwards-compatible with - existing code, as all codec classes defined by Numcodecs are imported into the - :mod:`zarr.codecs` namespace. However, it is recommended to import codecs from - the new package, see the tutorial sections on :ref:`tutorial_compress` and - :ref:`tutorial_filters` for examples. With contributions by - :user:`John Kirkham `; :issue:`74`, :issue:`102`, :issue:`120`, - :issue:`123`, :issue:`139`. - -* **New storage class for DBM-style databases**. The - :class:`zarr.storage.DBMStore` class enables any DBM-style database such as gdbm, - ndbm or Berkeley DB, to be used as the backing store for an array or group. See the - tutorial section on :ref:`tutorial_storage` for some examples. :issue:`133`, - :issue:`186`. - -* **New storage class for LMDB databases**. The :class:`zarr.storage.LMDBStore` class - enables an LMDB "Lightning" database to be used as the backing store for an array or - group. :issue:`192`. - -* **New storage class using a nested directory structure for chunk files**. The - :class:`zarr.storage.NestedDirectoryStore` has been added, which is similar to - the existing :class:`zarr.storage.DirectoryStore` class but nests chunk files - for multidimensional arrays into sub-directories. :issue:`155`, :issue:`177`. - -* **New tree() method for printing hierarchies**. The ``Group`` class has a new - :func:`zarr.hierarchy.Group.tree` method which enables a tree representation of - a group hierarchy to be printed. Also provides an interactive tree - representation when used within a Jupyter notebook. See the - :ref:`tutorial_diagnostics` tutorial section for examples. By - :user:`John Kirkham `; :issue:`82`, :issue:`140`, :issue:`184`. - -* **Visitor API**. The ``Group`` class now implements the h5py visitor API, see - docs for the :func:`zarr.hierarchy.Group.visit`, - :func:`zarr.hierarchy.Group.visititems` and - :func:`zarr.hierarchy.Group.visitvalues` methods. By - :user:`John Kirkham `, :issue:`92`, :issue:`122`. - -* **Viewing an array as a different dtype**. The ``Array`` class has a new - :func:`zarr.core.Array.astype` method, which is a convenience that enables an - array to be viewed as a different dtype. By :user:`John Kirkham `, - :issue:`94`, :issue:`96`. - -* **New open(), save(), load() convenience functions**. The function - :func:`zarr.convenience.open` provides a convenient way to open a persistent - array or group, using either a ``DirectoryStore`` or ``ZipStore`` as the backing - store. The functions :func:`zarr.convenience.save` and - :func:`zarr.convenience.load` are also available and provide a convenient way to - save an entire NumPy array to disk and load back into memory later. See the - tutorial section :ref:`tutorial_persist` for examples. :issue:`104`, - :issue:`105`, :issue:`141`, :issue:`181`. - -* **IPython completions**. The ``Group`` class now implements ``__dir__()`` and - ``_ipython_key_completions_()`` which enables tab-completion for group members - to be used in any IPython interactive environment. :issue:`170`. - -* **New info property; changes to __repr__**. The ``Group`` and - ``Array`` classes have a new ``info`` property which can be used to print - diagnostic information, including compression ratio where available. See the - tutorial section on :ref:`tutorial_diagnostics` for examples. The string - representation (``__repr__``) of these classes has been simplified to ensure - it is cheap and quick to compute in all circumstances. :issue:`83`, - :issue:`115`, :issue:`132`, :issue:`148`. - -* **Chunk options**. When creating an array, ``chunks=False`` can be specified, - which will result in an array with a single chunk only. Alternatively, - ``chunks=True`` will trigger an automatic chunk shape guess. See - :ref:`tutorial_chunks` for more on the ``chunks`` parameter. :issue:`106`, - :issue:`107`, :issue:`183`. - -* **Zero-dimensional arrays** and are now supported; by - :user:`Prakhar Goel `, :issue:`154`, :issue:`161`. - -* **Arrays with one or more zero-length dimensions** are now fully supported; by - :user:`Prakhar Goel `, :issue:`150`, :issue:`154`, :issue:`160`. - -* **The .zattrs key is now optional** and will now only be created when the first - custom attribute is set; :issue:`121`, :issue:`200`. - -* **New Group.move() method** supports moving a sub-group or array to a different - location within the same hierarchy. By :user:`John Kirkham `, - :issue:`191`, :issue:`193`, :issue:`196`. - -* **ZipStore is now thread-safe**; :issue:`194`, :issue:`192`. - -* **New Array.hexdigest() method** computes an ``Array``'s hash with ``hashlib``. - By :user:`John Kirkham `, :issue:`98`, :issue:`203`. - -* **Improved support for object arrays**. In previous versions of Zarr, - creating an array with ``dtype=object`` was possible but could under certain - circumstances lead to unexpected errors and/or segmentation faults. To make it easier - to properly configure an object array, a new ``object_codec`` parameter has been - added to array creation functions. See the tutorial section on :ref:`tutorial_objects` - for more information and examples. Also, runtime checks have been added in both Zarr - and Numcodecs so that segmentation faults are no longer possible, even with a badly - configured array. This API change is backwards compatible and previous code that created - an object array and provided an object codec via the ``filters`` parameter will - continue to work, however a warning will be raised to encourage use of the - ``object_codec`` parameter. :issue:`208`, :issue:`212`. - -* **Added support for datetime64 and timedelta64 data types**; - :issue:`85`, :issue:`215`. - -* **Array and group attributes are now cached by default** to improve performance with - slow stores, e.g., stores accessing data via the network; :issue:`220`, :issue:`218`, - :issue:`204`. - -* **New LRUStoreCache class**. The class :class:`zarr.storage.LRUStoreCache` has been - added and provides a means to locally cache data in memory from a store that may be - slow, e.g., a store that retrieves data from a remote server via the network; - :issue:`223`. - -* **New copy functions**. The new functions :func:`zarr.convenience.copy` and - :func:`zarr.convenience.copy_all` provide a way to copy groups and/or arrays - between HDF5 and Zarr, or between two Zarr groups. The - :func:`zarr.convenience.copy_store` provides a more efficient way to copy - data directly between two Zarr stores. :issue:`87`, :issue:`113`, - :issue:`137`, :issue:`217`. - -Bug fixes -~~~~~~~~~ - -* Fixed bug where ``read_only`` keyword argument was ignored when creating an - array; :issue:`151`, :issue:`179`. - -* Fixed bugs when using a ``ZipStore`` opened in 'w' mode; :issue:`158`, - :issue:`182`. - -* Fill values can now be provided for fixed-length string arrays; :issue:`165`, - :issue:`176`. - -* Fixed a bug where the number of chunks initialized could be counted - incorrectly; :issue:`97`, :issue:`174`. - -* Fixed a bug related to the use of an ellipsis (...) in indexing statements; - :issue:`93`, :issue:`168`, :issue:`172`. - -* Fixed a bug preventing use of other integer types for indexing; :issue:`143`, - :issue:`147`. - -Documentation -~~~~~~~~~~~~~ - -* Some changes have been made to the :ref:`spec_v2` document to clarify - ambiguities and add some missing information. These changes do not break compatibility - with any of the material as previously implemented, and so the changes have been made - in-place in the document without incrementing the document version number. See the - section on :ref:`spec_v2_changes` in the specification document for more information. -* A new :ref:`tutorial_indexing` section has been added to the tutorial. -* A new :ref:`tutorial_strings` section has been added to the tutorial - (:issue:`135`, :issue:`175`). -* The :ref:`tutorial_chunks` tutorial section has been reorganised and updated. -* The :ref:`tutorial_persist` and :ref:`tutorial_storage` tutorial sections have - been updated with new examples (:issue:`100`, :issue:`101`, :issue:`103`). -* A new tutorial section on :ref:`tutorial_pickle` has been added (:issue:`91`). -* A new tutorial section on :ref:`tutorial_datetime` has been added. -* A new tutorial section on :ref:`tutorial_diagnostics` has been added. -* The tutorial sections on :ref:`tutorial_sync` and :ref:`tutorial_tips_blosc` have been - updated to provide information about how to avoid program hangs when using the Blosc - compressor with multiple processes (:issue:`199`, :issue:`201`). - -Maintenance -~~~~~~~~~~~ - -* A data fixture has been included in the test suite to ensure data format - compatibility is maintained; :issue:`83`, :issue:`146`. -* The test suite has been migrated from nosetests to pytest; :issue:`189`, :issue:`225`. -* Various continuous integration updates and improvements; :issue:`118`, :issue:`124`, - :issue:`125`, :issue:`126`, :issue:`109`, :issue:`114`, :issue:`171`. -* Bump numcodecs dependency to 0.5.3, completely remove nose dependency, :issue:`237`. -* Fix compatibility issues with NumPy 1.14 regarding fill values for structured arrays, - :issue:`222`, :issue:`238`, :issue:`239`. - -Acknowledgments -~~~~~~~~~~~~~~~ - -Code was contributed to this release by :user:`Alistair Miles `, :user:`John -Kirkham ` and :user:`Prakhar Goel `. - -Documentation was contributed to this release by :user:`Mamy Ratsimbazafy ` -and :user:`Charles Noyes `. - -Thank you to :user:`John Kirkham `, :user:`Stephan Hoyer `, -:user:`Francesc Alted `, and :user:`Matthew Rocklin ` for code -reviews and/or comments on pull requests. - -.. _release_2.1.4: - -2.1.4 ------ - -* Resolved an issue where calling ``hasattr`` on a ``Group`` object erroneously - returned a ``KeyError``. By :user:`Vincent Schut `; :issue:`88`, - :issue:`95`. - -.. _release_2.1.3: - -2.1.3 ------ - -* Resolved an issue with :func:`zarr.creation.array` where dtype was given as - None (:issue:`80`). - -.. _release_2.1.2: - -2.1.2 ------ - -* Resolved an issue when no compression is used and chunks are stored in memory - (:issue:`79`). - -.. _release_2.1.1: - -2.1.1 ------ - -Various minor improvements, including: ``Group`` objects support member access -via dot notation (``__getattr__``); fixed metadata caching for ``Array.shape`` -property and derivatives; added ``Array.ndim`` property; fixed -``Array.__array__`` method arguments; fixed bug in pickling ``Array`` state; -fixed bug in pickling ``ThreadSynchronizer``. - -.. _release_2.1.0: - -2.1.0 ------ - -* Group objects now support member deletion via ``del`` statement - (:issue:`65`). -* Added :class:`zarr.storage.TempStore` class for convenience to provide - storage via a temporary directory - (:issue:`59`). -* Fixed performance issues with :class:`zarr.storage.ZipStore` class - (:issue:`66`). -* The Blosc extension has been modified to return bytes instead of array - objects from compress and decompress function calls. This should - improve compatibility and also provides a small performance increase for - compressing high compression ratio data - (:issue:`55`). -* Added ``overwrite`` keyword argument to array and group creation methods - on the :class:`zarr.hierarchy.Group` class - (:issue:`71`). -* Added ``cache_metadata`` keyword argument to array creation methods. -* The functions :func:`zarr.creation.open_array` and - :func:`zarr.hierarchy.open_group` now accept any store as first argument - (:issue:`56`). - -.. _release_2.0.1: - -2.0.1 ------ - -The bundled Blosc library has been upgraded to version 1.11.1. - -.. _release_2.0.0: - -2.0.0 ------ - -Hierarchies -~~~~~~~~~~~ - -Support has been added for organizing arrays into hierarchies via groups. See -the tutorial section on :ref:`tutorial_groups` and the :mod:`zarr.hierarchy` -API docs for more information. - -Filters -~~~~~~~ - -Support has been added for configuring filters to preprocess chunk data prior -to compression. See the tutorial section on :ref:`tutorial_filters` and the -:mod:`zarr.codecs` API docs for more information. - -Other changes -~~~~~~~~~~~~~ - -To accommodate support for hierarchies and filters, the Zarr metadata format -has been modified. See the :ref:`spec_v2` for more information. To migrate an -array stored using Zarr version 1.x, use the :func:`zarr.storage.migrate_1to2` -function. - -The bundled Blosc library has been upgraded to version 1.11.0. - -Acknowledgments -~~~~~~~~~~~~~~~ - -Thanks to :user:`Matthew Rocklin `, :user:`Stephan Hoyer ` and -:user:`Francesc Alted ` for contributions and comments. - -.. _release_1.1.0: - -1.1.0 ------ - -* The bundled Blosc library has been upgraded to version 1.10.0. The 'zstd' - internal compression library is now available within Blosc. See the tutorial - section on :ref:`tutorial_compress` for an example. -* When using the Blosc compressor, the default internal compression library - is now 'lz4'. -* The default number of internal threads for the Blosc compressor has been - increased to a maximum of 8 (previously 4). -* Added convenience functions :func:`zarr.blosc.list_compressors` and - :func:`zarr.blosc.get_nthreads`. - -.. _release_1.0.0: - -1.0.0 ------ - -This release includes a complete re-organization of the code base. The -major version number has been bumped to indicate that there have been -backwards-incompatible changes to the API and the on-disk storage -format. However, Zarr is still in an early stage of development, so -please do not take the version number as an indicator of maturity. - -Storage -~~~~~~~ - -The main motivation for re-organizing the code was to create an -abstraction layer between the core array logic and data storage (:issue:`21`). -In this release, any -object that implements the ``MutableMapping`` interface can be used as -an array store. See the tutorial sections on :ref:`tutorial_persist` -and :ref:`tutorial_storage`, the :ref:`spec_v1`, and the -:mod:`zarr.storage` module documentation for more information. - -Please note also that the file organization and file name conventions -used when storing a Zarr array in a directory on the file system have -changed. Persistent Zarr arrays created using previous versions of the -software will not be compatible with this version. See the -:mod:`zarr.storage` API docs and the :ref:`spec_v1` for more -information. - -Compression -~~~~~~~~~~~ - -An abstraction layer has also been created between the core array -logic and the code for compressing and decompressing array -chunks. This release still bundles the c-blosc library and uses Blosc -as the default compressor, however other compressors including zlib, -BZ2 and LZMA are also now supported via the Python standard -library. New compressors can also be dynamically registered for use -with Zarr. See the tutorial sections on :ref:`tutorial_compress` and -:ref:`tutorial_tips_blosc`, the :ref:`spec_v1`, and the -:mod:`zarr.compressors` module documentation for more information. - -Synchronization -~~~~~~~~~~~~~~~ - -The synchronization code has also been refactored to create a layer of -abstraction, enabling Zarr arrays to be used in parallel computations -with a number of alternative synchronization methods. For more -information see the tutorial section on :ref:`tutorial_sync` and the -:mod:`zarr.sync` module documentation. - -Changes to the Blosc extension -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -NumPy is no longer a build dependency for the :mod:`zarr.blosc` Cython -extension, so setup.py will run even if NumPy is not already -installed, and should automatically install NumPy as a runtime -dependency. Manual installation of NumPy prior to installing Zarr is -still recommended, however, as the automatic installation of NumPy may -fail or be sub-optimal on some platforms. - -Some optimizations have been made within the :mod:`zarr.blosc` -extension to avoid unnecessary memory copies, giving a ~10-20% -performance improvement for multi-threaded compression operations. - -The :mod:`zarr.blosc` extension now automatically detects whether it -is running within a single-threaded or multi-threaded program and -adapts its internal behaviour accordingly (:issue:`27`). There is no need for -the user to make any API calls to switch Blosc between contextual and -non-contextual (global lock) mode. See also the tutorial section on -:ref:`tutorial_tips_blosc`. - -Other changes -~~~~~~~~~~~~~ - -The internal code for managing chunks has been rewritten to be more -efficient. Now no state is maintained for chunks outside of the array -store, meaning that chunks do not carry any extra memory overhead not -accounted for by the store. This negates the need for the "lazy" -option present in the previous release, and this has been removed. - -The memory layout within chunks can now be set as either "C" -(row-major) or "F" (column-major), which can help to provide better -compression for some data (:issue:`7`). See the tutorial -section on :ref:`tutorial_chunks_order` for more information. - -A bug has been fixed within the ``__getitem__`` and ``__setitem__`` -machinery for slicing arrays, to properly handle getting and setting -partial slices. - -Acknowledgments -~~~~~~~~~~~~~~~ - -Thanks to :user:`Matthew Rocklin `, :user:`Stephan Hoyer `, -:user:`Francesc Alted `, :user:`Anthony Scopatz ` and -:user:`Martin Durant ` for contributions and comments. - -.. _release_0.4.0: - -0.4.0 ------ - -See `v0.4.0 release notes on GitHub -`_. - -.. _release_0.3.0: - -0.3.0 ------ - -See `v0.3.0 release notes on GitHub -`_. - -.. _Numcodecs: http://numcodecs.readthedocs.io/ diff --git a/docs/spec.rst b/docs/spec.rst deleted file mode 100644 index 765dcd782a..0000000000 --- a/docs/spec.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. _spec: - -Specifications -============== - -.. toctree:: - :maxdepth: 3 - - spec/v1 - spec/v2 diff --git a/docs/spec/v1.rst b/docs/spec/v1.rst deleted file mode 100644 index 18744b03b1..0000000000 --- a/docs/spec/v1.rst +++ /dev/null @@ -1,270 +0,0 @@ -.. _spec_v1: - -Zarr storage specification version 1 -==================================== - -This document provides a technical specification of the protocol and -format used for storing a Zarr array. The key words "MUST", "MUST -NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", -"RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be -interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is deprecated. See :ref:`spec` for the latest version. - -Storage -------- - -A Zarr array can be stored in any storage system that provides a -key/value interface, where a key is an ASCII string and a value is an -arbitrary sequence of bytes, and the supported operations are read -(get the sequence of bytes associated with a given key), write (set -the sequence of bytes associated with a given key) and delete (remove -a key/value pair). - -For example, a directory in a file system can provide this interface, -where keys are file names, values are file contents, and files can be -read, written or deleted via the operating system. Equally, an S3 -bucket can provide this interface, where keys are resource names, -values are resource contents, and resources can be read, written or -deleted via HTTP. - -Below an "array store" refers to any system implementing this -interface. - -Metadata --------- - -Each array requires essential configuration metadata to be stored, -enabling correct interpretation of the stored data. This metadata is -encoded using JSON and stored as the value of the 'meta' key within an -array store. - -The metadata resource is a JSON object. The following keys MUST be -present within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compression - A string identifying the primary compression library used to compress - each chunk of the array. -compression_opts - An integer, string or dictionary providing options to the primary - compression library. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array. -order - Either 'C' or 'F', defining the layout of bytes within each chunk of the - array. 'C' means row-major order, i.e., the last dimension varies fastest; - 'F' means column-major order, i.e., the first dimension varies fastest. - -Other keys MAY be present within the metadata object however they MUST -NOT alter the interpretation of the required fields defined above. - -For example, the JSON object below defines a 2-dimensional array of -64-bit little-endian floating point numbers with 10000 rows and 10000 -columns, divided into chunks of 1000 rows and 1000 columns (so there -will be 100 chunks in total arranged in a 10 by 10 grid). Within each -chunk the data are laid out in C contiguous order, and each chunk is -compressed using the Blosc compression library:: - - { - "chunks": [ - 1000, - 1000 - ], - "compression": "blosc", - "compression_opts": { - "clevel": 5, - "cname": "lz4", - "shuffle": 1 - }, - "dtype": "`_. The -format consists of 3 parts: a character describing the byteorder of -the data (``<``: little-endian, ``>``: big-endian, ``|``: -not-relevant), a character code giving the basic type of the array, -and an integer providing the number of bytes the type uses. The byte -order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data types. - -Structure data types (i.e., with multiple named fields) are encoded as -a list of two-element lists, following `NumPy array protocol type -descriptions (descr) -`_. -For example, the JSON list ``[["r", "|u1"], ["g", "|u1"], ["b", -"|u1"]]`` defines a data type composed of three single-byte unsigned -integers labelled 'r', 'g' and 'b'. - -Chunks ------- - -Each chunk of the array is compressed by passing the raw bytes for the -chunk through the primary compression library to obtain a new sequence -of bytes comprising the compressed chunk data. No header is added to -the compressed bytes or any other modification made. The internal -structure of the compressed bytes will depend on which primary -compressor was used. For example, the `Blosc compressor -`_ -produces a sequence of bytes that begins with a 16-byte header -followed by compressed data. - -The compressed sequence of bytes for each chunk is stored under a key -formed from the index of the chunk within the grid of chunks -representing the array. To form a string key for a chunk, the indices -are converted to strings and concatenated with the period character -('.') separating each index. For example, given an array with shape -(10000, 10000) and chunk shape (1000, 1000) there will be 100 chunks -laid out in a 10 by 10 grid. The chunk with indices (0, 0) provides -data for rows 0-1000 and columns 0-1000 and is stored under the key -'0.0'; the chunk with indices (2, 4) provides data for rows 2000-3000 -and columns 4000-5000 and is stored under the key '2.4'; etc. - -There is no need for all chunks to be present within an array -store. If a chunk is not present then it is considered to be in an -uninitialized state. An unitialized chunk MUST be treated as if it -was uniformly filled with the value of the 'fill_value' field in the -array metadata. If the 'fill_value' field is ``null`` then the -contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of -any array dimension is not exactly divisible by the length of the -corresponding chunk dimension then some chunks will overhang the edge -of the array. The contents of any chunk region falling outside the -array are undefined. - -Attributes ----------- - -Each array can also be associated with custom attributes, which are -simple key/value items with application-specific meaning. Custom -attributes are encoded as a JSON object and stored under the 'attrs' -key within an array store. Even if the attributes are empty, the -'attrs' key MUST be present within an array store. - -For example, the JSON object below encodes three attributes named -'foo', 'bar' and 'baz':: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -Example -------- - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Initialize the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('example.zarr') - >>> zarr.init_store(store, shape=(20, 20), chunks=(10, 10), - ... dtype='i4', fill_value=42, compression='zlib', - ... compression_opts=1, overwrite=True) - -No chunks are initialized yet, so only the 'meta' and 'attrs' keys -have been set:: - - >>> import os - >>> sorted(os.listdir('example.zarr')) - ['attrs', 'meta'] - -Inspect the array metadata:: - - >>> print(open('example.zarr/meta').read()) - { - "chunks": [ - 10, - 10 - ], - "compression": "zlib", - "compression_opts": 1, - "dtype": ">> print(open('example.zarr/attrs').read()) - {} - -Set some data:: - - >>> z = zarr.Array(store) - >>> z[0:10, 0:10] = 1 - >>> sorted(os.listdir('example.zarr')) - ['0.0', 'attrs', 'meta'] - -Set some more data:: - - >>> z[0:10, 10:20] = 2 - >>> z[10:20, :] = 3 - >>> sorted(os.listdir('example.zarr')) - ['0.0', '0.1', '1.0', '1.1', 'attrs', 'meta'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> b = zlib.decompress(open('example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> a = np.frombuffer(b, dtype='>> a - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> z.attrs['foo'] = 42 - >>> z.attrs['bar'] = 'apples' - >>> z.attrs['baz'] = [1, 2, 3, 4] - >>> print(open('example.zarr/attrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } diff --git a/docs/spec/v2.rst b/docs/spec/v2.rst deleted file mode 100644 index 8a1b58aeb3..0000000000 --- a/docs/spec/v2.rst +++ /dev/null @@ -1,565 +0,0 @@ -.. _spec_v2: - -Zarr storage specification version 2 -==================================== - -This document provides a technical specification of the protocol and format -used for storing Zarr arrays. The key words "MUST", "MUST NOT", "REQUIRED", -"SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and -"OPTIONAL" in this document are to be interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is the latest version. See :ref:`spec` for previous -versions. - -.. _spec_v2_storage: - -Storage -------- - -A Zarr array can be stored in any storage system that provides a key/value -interface, where a key is an ASCII string and a value is an arbitrary sequence -of bytes, and the supported operations are read (get the sequence of bytes -associated with a given key), write (set the sequence of bytes associated with -a given key) and delete (remove a key/value pair). - -For example, a directory in a file system can provide this interface, where -keys are file names, values are file contents, and files can be read, written -or deleted via the operating system. Equally, an S3 bucket can provide this -interface, where keys are resource names, values are resource contents, and -resources can be read, written or deleted via HTTP. - -Below an "array store" refers to any system implementing this interface. - -.. _spec_v2_array: - -Arrays ------- - -.. _spec_v2_array_metadata: - -Metadata -~~~~~~~~ - -Each array requires essential configuration metadata to be stored, enabling -correct interpretation of the stored data. This metadata is encoded using JSON -and stored as the value of the ".zarray" key within an array store. - -The metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compressor - A JSON object identifying the primary compression codec and providing - configuration parameters, or ``null`` if no compressor is to be used. - The object MUST contain an ``"id"`` key identifying the codec to be used. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array, or ``null`` if no fill_value is to be used. -order - Either "C" or "F", defining the layout of bytes within each chunk of the - array. "C" means row-major order, i.e., the last dimension varies fastest; - "F" means column-major order, i.e., the first dimension varies fastest. -filters - A list of JSON objects providing codec configurations, or ``null`` if no - filters are to be applied. Each codec configuration object MUST contain a - ``"id"`` key identifying the codec to be used. - -The following keys MAY be present within the object: - -dimension_separator - If present, either the string ``"."`` or ``"/""`` definining the separator placed - between the dimensions of a chunk. If the value is not set, then the - default MUST be assumed to be ``"."``, leading to chunk keys of the form "0.0". - Arrays defined with ``"/"`` as the dimension separator can be considered to have - nested, or hierarchical, keys of the form "0/0" that SHOULD where possible - produce a directory-like structure. - -Other keys SHOULD NOT be present within the metadata object and SHOULD be -ignored by implementations. - -For example, the JSON object below defines a 2-dimensional array of 64-bit -little-endian floating point numbers with 10000 rows and 10000 columns, divided -into chunks of 1000 rows and 1000 columns (so there will be 100 chunks in total -arranged in a 10 by 10 grid). Within each chunk the data are laid out in C -contiguous order. Each chunk is encoded using a delta filter and compressed -using the Blosc compression library prior to storage:: - - { - "chunks": [ - 1000, - 1000 - ], - "compressor": { - "id": "blosc", - "cname": "lz4", - "clevel": 5, - "shuffle": 1 - }, - "dtype": "`. The format -consists of 3 parts: - -* One character describing the byteorder of the data (``"<"``: little-endian; - ``">"``: big-endian; ``"|"``: not-relevant) -* One character code giving the basic type of the array (``"b"``: Boolean (integer - type where all values are only True or False); ``"i"``: integer; ``"u"``: unsigned - integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"m"``: timedelta; - ``"M"``: datetime; ``"S"``: string (fixed-length sequence of char); ``"U"``: unicode - (fixed-length sequence of Py_UNICODE); ``"V"``: other (void * – each item is a - fixed-size chunk of memory)) -* An integer specifying the number of bytes the type uses. - -The byte order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data type encodings. - -For datetime64 ("M") and timedelta64 ("m") data types, these MUST also include the -units within square brackets. A list of valid units and their definitions are given in -the :ref:`NumPy documentation on Datetimes and Timedeltas -`. -For example, ``"`. Each -sub-list has the form ``[fieldname, datatype, shape]`` where ``shape`` -is optional. ``fieldname`` is a string, ``datatype`` is a string -specifying a simple data type (see above), and ``shape`` is a list of -integers specifying subarray shape. For example, the JSON list below -defines a data type composed of three single-byte unsigned integer -fields named "r", "g" and "b":: - - [["r", "|u1"], ["g", "|u1"], ["b", "|u1"]] - -For example, the JSON list below defines a data type composed of three -fields named "x", "y" and "z", where "x" and "y" each contain 32-bit -floats, and each item in "z" is a 2 by 2 array of floats:: - - [["x", "`_ -produces a sequence of bytes that begins with a 16-byte header followed by -compressed data. - -The compressed sequence of bytes for each chunk is stored under a key formed -from the index of the chunk within the grid of chunks representing the array. -To form a string key for a chunk, the indices are converted to strings and -concatenated with the period character (".") separating each index. For -example, given an array with shape (10000, 10000) and chunk shape (1000, 1000) -there will be 100 chunks laid out in a 10 by 10 grid. The chunk with indices -(0, 0) provides data for rows 0-1000 and columns 0-1000 and is stored under the -key "0.0"; the chunk with indices (2, 4) provides data for rows 2000-3000 and -columns 4000-5000 and is stored under the key "2.4"; etc. - -There is no need for all chunks to be present within an array store. If a chunk -is not present then it is considered to be in an uninitialized state. An -unitialized chunk MUST be treated as if it was uniformly filled with the value -of the "fill_value" field in the array metadata. If the "fill_value" field is -``null`` then the contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of any -array dimension is not exactly divisible by the length of the corresponding -chunk dimension then some chunks will overhang the edge of the array. The -contents of any chunk region falling outside the array are undefined. - -.. _spec_v2_array_filters: - -Filters -~~~~~~~ - -Optionally a sequence of one or more filters can be used to transform chunk -data prior to compression. When storing data, filters are applied in the order -specified in array metadata to encode data, then the encoded data are passed to -the primary compressor. When retrieving data, stored chunk data are -decompressed by the primary compressor then decoded using filters in the -reverse order. - -.. _spec_v2_hierarchy: - -Hierarchies ------------ - -.. _spec_v2_hierarchy_paths: - -Logical storage paths -~~~~~~~~~~~~~~~~~~~~~ - -Multiple arrays can be stored in the same array store by associating each array -with a different logical path. A logical path is simply an ASCII string. The -logical path is used to form a prefix for keys used by the array. For example, -if an array is stored at logical path "foo/bar" then the array metadata will be -stored under the key "foo/bar/.zarray", the user-defined attributes will be -stored under the key "foo/bar/.zattrs", and the chunks will be stored under -keys like "foo/bar/0.0", "foo/bar/0.1", etc. - -To ensure consistent behaviour across different storage systems, logical paths -MUST be normalized as follows: - -* Replace all backward slash characters ("\\\\") with forward slash characters - ("/") -* Strip any leading "/" characters -* Strip any trailing "/" characters -* Collapse any sequence of more than one "/" character into a single "/" - character - -The key prefix is then obtained by appending a single "/" character to the -normalized logical path. - -After normalization, if splitting a logical path by the "/" character results -in any path segment equal to the string "." or the string ".." then an error -MUST be raised. - -N.B., how the underlying array store processes requests to store values under -keys containing the "/" character is entirely up to the store implementation -and is not constrained by this specification. E.g., an array store could simply -treat all keys as opaque ASCII strings; equally, an array store could map -logical paths onto some kind of hierarchical storage (e.g., directories on a -file system). - -.. _spec_v2_hierarchy_groups: - -Groups -~~~~~~ - -Arrays can be organized into groups which can also contain other groups. A -group is created by storing group metadata under the ".zgroup" key under some -logical path. E.g., a group exists at the root of an array store if the -".zgroup" key exists in the store, and a group exists at logical path "foo/bar" -if the "foo/bar/.zgroup" key exists in the store. - -If the user requests a group to be created under some logical path, then groups -MUST also be created at all ancestor paths. E.g., if the user requests group -creation at path "foo/bar" then groups MUST be created at path "foo" and the -root of the store, if they don't already exist. - -If the user requests an array to be created under some logical path, then -groups MUST also be created at all ancestor paths. E.g., if the user requests -array creation at path "foo/bar/baz" then groups must be created at path -"foo/bar", path "foo", and the root of the store, if they don't already exist. - -The group metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. - -Other keys MUST NOT be present within the metadata object. - -The members of a group are arrays and groups stored under logical paths that -are direct children of the parent group's logical path. E.g., if groups exist -under the logical paths "foo" and "foo/bar" and an array exists at logical path -"foo/baz" then the members of the group at path "foo" are the group at path -"foo/bar" and the array at path "foo/baz". - -.. _spec_v2_attrs: - -Attributes ----------- - -An array or group can be associated with custom attributes, which are arbitrary -key/value pairs with application-specific meaning. Custom attributes are encoded -as a JSON object and stored under the ".zattrs" key within an array store. The -".zattrs" key does not have to be present, and if it is absent the attributes -should be treated as empty. - -For example, the JSON object below encodes three attributes named -"foo", "bar" and "baz":: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -.. _spec_v2_examples: - -Examples --------- - -Storing a single array -~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Create an array:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/example.zarr') - >>> a = zarr.create(shape=(20, 20), chunks=(10, 10), dtype='i4', - ... fill_value=42, compressor=zarr.Zlib(level=1), - ... store=store, overwrite=True) - -No chunks are initialized yet, so only the ".zarray" and ".zattrs" keys -have been set in the store:: - - >>> import os - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray'] - -Inspect the array metadata:: - - >>> print(open('data/example.zarr/.zarray').read()) - { - "chunks": [ - 10, - 10 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": ">> a[0:10, 0:10] = 1 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0'] - -Set some more data:: - - >>> a[0:10, 10:20] = 2 - >>> a[10:20, :] = 3 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0', '0.1', '1.0', '1.1'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> buf = zlib.decompress(open('data/example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> chunk = np.frombuffer(buf, dtype='>> chunk - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> a.attrs['foo'] = 42 - >>> a.attrs['bar'] = 'apples' - >>> a.attrs['baz'] = [1, 2, 3, 4] - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - >>> print(open('data/example.zarr/.zattrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } - -Storing multiple arrays in a hierarchy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing multiple Zarr arrays organized into a group -hierarchy, using a directory on the local file system as storage. This storage -implementation maps logical paths onto directory paths on the file system, -however this is an implementation choice and is not required. - -Setup the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/group.zarr') - -Create the root group:: - - >>> root_grp = zarr.group(store, overwrite=True) - -The metadata resource for the root group has been created:: - - >>> import os - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup'] - -Inspect the group metadata:: - - >>> print(open('data/group.zarr/.zgroup').read()) - { - "zarr_format": 2 - } - -Create a sub-group:: - - >>> sub_grp = root_grp.create_group('foo') - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup'] - -Create an array within the sub-group:: - - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - -Set a custom attributes:: - - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup', 'bar'] - >>> sorted(os.listdir('data/group.zarr/foo/bar')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - -Here is the same example using a Zip file as storage:: - - >>> store = zarr.ZipStore('data/group.zip', mode='w') - >>> root_grp = zarr.group(store) - >>> sub_grp = root_grp.create_group('foo') - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - >>> store.close() - -What has been stored:: - - >>> import zipfile - >>> zf = zipfile.ZipFile('data/group.zip', mode='r') - >>> for name in sorted(zf.namelist()): - ... print(name) - .zgroup - foo/.zgroup - foo/bar/.zarray - foo/bar/.zattrs - foo/bar/0.0 - foo/bar/0.1 - foo/bar/1.0 - foo/bar/1.1 - -.. _spec_v2_changes: - -Changes -------- - -Version 2 clarifications -~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes have been made to the version 2 specification since it was -initially published to clarify ambiguities and add some missing information. - -* The specification now describes how bytes fill values should be encoded and - decoded for arrays with a fixed-length byte string data type (:issue:`165`, - :issue:`176`). - -* The specification now clarifies that units must be specified for datetime64 and - timedelta64 data types (:issue:`85`, :issue:`215`). - -* The specification now clarifies that the '.zattrs' key does not have to be present for - either arrays or groups, and if absent then custom attributes should be treated as - empty. - -* The specification now describes how structured datatypes with - subarray shapes and/or with nested structured data types are encoded - in array metadata (:issue:`111`, :issue:`296`). - -* Clarified the key/value pairs of custom attributes as "arbitrary" rather than - "simple". - -Changes from version 1 to version 2 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes were made between version 1 and version 2 of this specification: - -* Added support for storing multiple arrays in the same store and organising - arrays into hierarchies using groups. -* Array metadata is now stored under the ".zarray" key instead of the "meta" - key. -* Custom attributes are now stored under the ".zattrs" key instead of the - "attrs" key. -* Added support for filters. -* Changed encoding of "fill_value" field within array metadata. -* Changed encoding of compressor information within array metadata to be - consistent with representation of filter information. diff --git a/docs/talks/scipy2019/submission.rst b/docs/talks/scipy2019/submission.rst index a7d531c91e..57fd925b1f 100644 --- a/docs/talks/scipy2019/submission.rst +++ b/docs/talks/scipy2019/submission.rst @@ -118,7 +118,7 @@ Authors Project contributors are listed in alphabetical order by surname. -* `Ryan Abernathey `_, Columbia University +* `Ryan Abernathey `_, Columbia University * `Stephan Balmer `_, Meteotest * `Ambrose Carr `_, Chan Zuckerberg Initiative * `Tim Crone `_, Columbia University diff --git a/docs/tutorial.rst b/docs/tutorial.rst deleted file mode 100644 index a3421608cc..0000000000 --- a/docs/tutorial.rst +++ /dev/null @@ -1,1511 +0,0 @@ -.. _tutorial: - -Tutorial -======== - -Zarr provides classes and functions for working with N-dimensional arrays that -behave like NumPy arrays but whose data is divided into chunks and each chunk is -compressed. If you are already familiar with HDF5 then Zarr arrays provide -similar functionality, but with some additional flexibility. - -.. _tutorial_create: - -Creating an array ------------------ - -Zarr has several functions for creating arrays. For example:: - - >>> import zarr - >>> z = zarr.zeros((10000, 10000), chunks=(1000, 1000), dtype='i4') - >>> z - - -The code above creates a 2-dimensional array of 32-bit integers with 10000 rows -and 10000 columns, divided into chunks where each chunk has 1000 rows and 1000 -columns (and so there will be 100 chunks in total). - -For a complete list of array creation routines see the :mod:`zarr.creation` -module documentation. - -.. _tutorial_array: - -Reading and writing data ------------------------- - -Zarr arrays support a similar interface to NumPy arrays for reading and writing -data. For example, the entire array can be filled with a scalar value:: - - >>> z[:] = 42 - -Regions of the array can also be written to, e.g.:: - - >>> import numpy as np - >>> z[0, :] = np.arange(10000) - >>> z[:, 0] = np.arange(10000) - -The contents of the array can be retrieved by slicing, which will load the -requested region into memory as a NumPy array, e.g.:: - - >>> z[0, 0] - 0 - >>> z[-1, -1] - 42 - >>> z[0, :] - array([ 0, 1, 2, ..., 9997, 9998, 9999], dtype=int32) - >>> z[:, 0] - array([ 0, 1, 2, ..., 9997, 9998, 9999], dtype=int32) - >>> z[:] - array([[ 0, 1, 2, ..., 9997, 9998, 9999], - [ 1, 42, 42, ..., 42, 42, 42], - [ 2, 42, 42, ..., 42, 42, 42], - ..., - [9997, 42, 42, ..., 42, 42, 42], - [9998, 42, 42, ..., 42, 42, 42], - [9999, 42, 42, ..., 42, 42, 42]], dtype=int32) - -.. _tutorial_persist: - -Persistent arrays ------------------ - -In the examples above, compressed data for each chunk of the array was stored in -main memory. Zarr arrays can also be stored on a file system, enabling -persistence of data between sessions. For example:: - - >>> z1 = zarr.open('data/example.zarr', mode='w', shape=(10000, 10000), - ... chunks=(1000, 1000), dtype='i4') - -The array above will store its configuration metadata and all compressed chunk -data in a directory called 'data/example.zarr' relative to the current working -directory. The :func:`zarr.convenience.open` function provides a convenient way -to create a new persistent array or continue working with an existing -array. Note that although the function is called "open", there is no need to -close an array: data are automatically flushed to disk, and files are -automatically closed whenever an array is modified. - -Persistent arrays support the same interface for reading and writing data, -e.g.:: - - >>> z1[:] = 42 - >>> z1[0, :] = np.arange(10000) - >>> z1[:, 0] = np.arange(10000) - -Check that the data have been written and can be read again:: - - >>> z2 = zarr.open('data/example.zarr', mode='r') - >>> np.all(z1[:] == z2[:]) - True - -If you are just looking for a fast and convenient way to save NumPy arrays to -disk then load back into memory later, the functions -:func:`zarr.convenience.save` and :func:`zarr.convenience.load` may be -useful. E.g.:: - - >>> a = np.arange(10) - >>> zarr.save('data/example.zarr', a) - >>> zarr.load('data/example.zarr') - array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - -Please note that there are a number of other options for persistent array -storage, see the section on :ref:`tutorial_storage` below. - -.. _tutorial_resize: - -Resizing and appending ----------------------- - -A Zarr array can be resized, which means that any of its dimensions can be -increased or decreased in length. For example:: - - >>> z = zarr.zeros(shape=(10000, 10000), chunks=(1000, 1000)) - >>> z[:] = 42 - >>> z.resize(20000, 10000) - >>> z.shape - (20000, 10000) - -Note that when an array is resized, the underlying data are not rearranged in -any way. If one or more dimensions are shrunk, any chunks falling outside the -new array shape will be deleted from the underlying store. - -For convenience, Zarr arrays also provide an ``append()`` method, which can be -used to append data to any axis. E.g.:: - - >>> a = np.arange(10000000, dtype='i4').reshape(10000, 1000) - >>> z = zarr.array(a, chunks=(1000, 100)) - >>> z.shape - (10000, 1000) - >>> z.append(a) - (20000, 1000) - >>> z.append(np.vstack([a, a]), axis=1) - (20000, 2000) - >>> z.shape - (20000, 2000) - -.. _tutorial_compress: - -Compressors ------------ - -A number of different compressors can be used with Zarr. A separate package -called NumCodecs_ is available which provides a common interface to various -compressor libraries including Blosc, Zstandard, LZ4, Zlib, BZ2 and -LZMA. Different compressors can be provided via the ``compressor`` keyword -argument accepted by all array creation functions. For example:: - - >>> from numcodecs import Blosc - >>> compressor = Blosc(cname='zstd', clevel=3, shuffle=Blosc.BITSHUFFLE) - >>> data = np.arange(100000000, dtype='i4').reshape(10000, 10000) - >>> z = zarr.array(data, chunks=(1000, 1000), compressor=compressor) - >>> z.compressor - Blosc(cname='zstd', clevel=3, shuffle=BITSHUFFLE, blocksize=0) - -This array above will use Blosc as the primary compressor, using the Zstandard -algorithm (compression level 3) internally within Blosc, and with the -bit-shuffle filter applied. - -When using a compressor, it can be useful to get some diagnostics on the -compression ratio. Zarr arrays provide a ``info`` property which can be used to -print some diagnostics, e.g.:: - - >>> z.info - Type : zarr.core.Array - Data type : int32 - Shape : (10000, 10000) - Chunk shape : (1000, 1000) - Order : C - Read-only : False - Compressor : Blosc(cname='zstd', clevel=3, shuffle=BITSHUFFLE, - : blocksize=0) - Store type : builtins.dict - No. bytes : 400000000 (381.5M) - No. bytes stored : 3379344 (3.2M) - Storage ratio : 118.4 - Chunks initialized : 100/100 - -If you don't specify a compressor, by default Zarr uses the Blosc -compressor. Blosc is generally very fast and can be configured in a variety of -ways to improve the compression ratio for different types of data. Blosc is in -fact a "meta-compressor", which means that it can use a number of different -compression algorithms internally to compress the data. Blosc also provides -highly optimized implementations of byte- and bit-shuffle filters, which can -improve compression ratios for some data. A list of the internal compression -libraries available within Blosc can be obtained via:: - - >>> from numcodecs import blosc - >>> blosc.list_compressors() - ['blosclz', 'lz4', 'lz4hc', 'snappy', 'zlib', 'zstd'] - -In addition to Blosc, other compression libraries can also be used. For example, -here is an array using Zstandard compression, level 1:: - - >>> from numcodecs import Zstd - >>> z = zarr.array(np.arange(100000000, dtype='i4').reshape(10000, 10000), - ... chunks=(1000, 1000), compressor=Zstd(level=1)) - >>> z.compressor - Zstd(level=1) - -Here is an example using LZMA with a custom filter pipeline including LZMA's -built-in delta filter:: - - >>> import lzma - >>> lzma_filters = [dict(id=lzma.FILTER_DELTA, dist=4), - ... dict(id=lzma.FILTER_LZMA2, preset=1)] - >>> from numcodecs import LZMA - >>> compressor = LZMA(filters=lzma_filters) - >>> z = zarr.array(np.arange(100000000, dtype='i4').reshape(10000, 10000), - ... chunks=(1000, 1000), compressor=compressor) - >>> z.compressor - LZMA(format=1, check=-1, preset=None, filters=[{'dist': 4, 'id': 3}, {'id': 33, 'preset': 1}]) - -The default compressor can be changed by setting the value of the -``zarr.storage.default_compressor`` variable, e.g.:: - - >>> import zarr.storage - >>> from numcodecs import Zstd, Blosc - >>> # switch to using Zstandard - ... zarr.storage.default_compressor = Zstd(level=1) - >>> z = zarr.zeros(100000000, chunks=1000000) - >>> z.compressor - Zstd(level=1) - >>> # switch back to Blosc defaults - ... zarr.storage.default_compressor = Blosc() - -To disable compression, set ``compressor=None`` when creating an array, e.g.:: - - >>> z = zarr.zeros(100000000, chunks=1000000, compressor=None) - >>> z.compressor is None - True - -.. _tutorial_filters: - -Filters -------- - -In some cases, compression can be improved by transforming the data in some -way. For example, if nearby values tend to be correlated, then shuffling the -bytes within each numerical value or storing the difference between adjacent -values may increase compression ratio. Some compressors provide built-in filters -that apply transformations to the data prior to compression. For example, the -Blosc compressor has built-in implementations of byte- and bit-shuffle filters, -and the LZMA compressor has a built-in implementation of a delta -filter. However, to provide additional flexibility for implementing and using -filters in combination with different compressors, Zarr also provides a -mechanism for configuring filters outside of the primary compressor. - -Here is an example using a delta filter with the Blosc compressor:: - - >>> from numcodecs import Blosc, Delta - >>> filters = [Delta(dtype='i4')] - >>> compressor = Blosc(cname='zstd', clevel=1, shuffle=Blosc.SHUFFLE) - >>> data = np.arange(100000000, dtype='i4').reshape(10000, 10000) - >>> z = zarr.array(data, chunks=(1000, 1000), filters=filters, compressor=compressor) - >>> z.info - Type : zarr.core.Array - Data type : int32 - Shape : (10000, 10000) - Chunk shape : (1000, 1000) - Order : C - Read-only : False - Filter [0] : Delta(dtype='`_ documentation. - -.. _tutorial_groups: - -Groups ------- - -Zarr supports hierarchical organization of arrays via groups. As with arrays, -groups can be stored in memory, on disk, or via other storage systems that -support a similar interface. - -To create a group, use the :func:`zarr.group` function:: - - >>> root = zarr.group() - >>> root - - -Groups have a similar API to the Group class from `h5py -`_. For example, groups can contain other groups:: - - >>> foo = root.create_group('foo') - >>> bar = foo.create_group('bar') - -Groups can also contain arrays, e.g.:: - - >>> z1 = bar.zeros('baz', shape=(10000, 10000), chunks=(1000, 1000), dtype='i4') - >>> z1 - - -Arrays are known as "datasets" in HDF5 terminology. For compatibility with h5py, -Zarr groups also implement the ``create_dataset()`` and ``require_dataset()`` -methods, e.g.:: - - >>> z = bar.create_dataset('quux', shape=(10000, 10000), chunks=(1000, 1000), dtype='i4') - >>> z - - -Members of a group can be accessed via the suffix notation, e.g.:: - - >>> root['foo'] - - -The '/' character can be used to access multiple levels of the hierarchy in one -call, e.g.:: - - >>> root['foo/bar'] - - >>> root['foo/bar/baz'] - - -The :func:`zarr.hierarchy.Group.tree` method can be used to print a tree -representation of the hierarchy, e.g.:: - - >>> root.tree() - / - └── foo - └── bar - ├── baz (10000, 10000) int32 - └── quux (10000, 10000) int32 - -The :func:`zarr.convenience.open` function provides a convenient way to create or -re-open a group stored in a directory on the file-system, with sub-groups stored in -sub-directories, e.g.:: - - >>> root = zarr.open('data/group.zarr', mode='w') - >>> root - - >>> z = root.zeros('foo/bar/baz', shape=(10000, 10000), chunks=(1000, 1000), dtype='i4') - >>> z - - -Groups can be used as context managers (in a ``with`` statement). -If the underlying store has a ``close`` method, it will be called on exit. - -For more information on groups see the :mod:`zarr.hierarchy` and -:mod:`zarr.convenience` API docs. - -.. _tutorial_diagnostics: - -Array and group diagnostics ---------------------------- - -Diagnostic information about arrays and groups is available via the ``info`` -property. E.g.:: - - >>> root = zarr.group() - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=1000000, chunks=100000, dtype='i8') - >>> bar[:] = 42 - >>> baz = foo.zeros('baz', shape=(1000, 1000), chunks=(100, 100), dtype='f4') - >>> baz[:] = 4.2 - >>> root.info - Name : / - Type : zarr.hierarchy.Group - Read-only : False - Store type : zarr.storage.MemoryStore - No. members : 1 - No. arrays : 0 - No. groups : 1 - Groups : foo - - >>> foo.info - Name : /foo - Type : zarr.hierarchy.Group - Read-only : False - Store type : zarr.storage.MemoryStore - No. members : 2 - No. arrays : 2 - No. groups : 0 - Arrays : bar, baz - - >>> bar.info - Name : /foo/bar - Type : zarr.core.Array - Data type : int64 - Shape : (1000000,) - Chunk shape : (100000,) - Order : C - Read-only : False - Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0) - Store type : zarr.storage.MemoryStore - No. bytes : 8000000 (7.6M) - No. bytes stored : 33240 (32.5K) - Storage ratio : 240.7 - Chunks initialized : 10/10 - - >>> baz.info - Name : /foo/baz - Type : zarr.core.Array - Data type : float32 - Shape : (1000, 1000) - Chunk shape : (100, 100) - Order : C - Read-only : False - Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0) - Store type : zarr.storage.MemoryStore - No. bytes : 4000000 (3.8M) - No. bytes stored : 23943 (23.4K) - Storage ratio : 167.1 - Chunks initialized : 100/100 - -Groups also have the :func:`zarr.hierarchy.Group.tree` method, e.g.:: - - >>> root.tree() - / - └── foo - ├── bar (1000000,) int64 - └── baz (1000, 1000) float32 - -If you're using Zarr within a Jupyter notebook (requires -`ipytree `_), calling ``tree()`` will generate an -interactive tree representation, see the `repr_tree.ipynb notebook -`_ -for more examples. - -.. _tutorial_attrs: - -User attributes ---------------- - -Zarr arrays and groups support custom key/value attributes, which can be useful for -storing application-specific metadata. For example:: - - >>> root = zarr.group() - >>> root.attrs['foo'] = 'bar' - >>> z = root.zeros('zzz', shape=(10000, 10000)) - >>> z.attrs['baz'] = 42 - >>> z.attrs['qux'] = [1, 4, 7, 12] - >>> sorted(root.attrs) - ['foo'] - >>> 'foo' in root.attrs - True - >>> root.attrs['foo'] - 'bar' - >>> sorted(z.attrs) - ['baz', 'qux'] - >>> z.attrs['baz'] - 42 - >>> z.attrs['qux'] - [1, 4, 7, 12] - -Internally Zarr uses JSON to store array attributes, so attribute values must be -JSON serializable. - -.. _tutorial_indexing: - -Advanced indexing ------------------ - -As of version 2.2, Zarr arrays support several methods for advanced or "fancy" -indexing, which enable a subset of data items to be extracted or updated in an -array without loading the entire array into memory. - -Note that although this functionality is similar to some of the advanced -indexing capabilities available on NumPy arrays and on h5py datasets, **the Zarr -API for advanced indexing is different from both NumPy and h5py**, so please -read this section carefully. For a complete description of the indexing API, -see the documentation for the :class:`zarr.core.Array` class. - -Indexing with coordinate arrays -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Items from a Zarr array can be extracted by providing an integer array of -coordinates. E.g.:: - - >>> z = zarr.array(np.arange(10)) - >>> z[:] - array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - >>> z.get_coordinate_selection([1, 4]) - array([1, 4]) - -Coordinate arrays can also be used to update data, e.g.:: - - >>> z.set_coordinate_selection([1, 4], [-1, -2]) - >>> z[:] - array([ 0, -1, 2, 3, -2, 5, 6, 7, 8, 9]) - -For multidimensional arrays, coordinates must be provided for each dimension, -e.g.:: - - >>> z = zarr.array(np.arange(15).reshape(3, 5)) - >>> z[:] - array([[ 0, 1, 2, 3, 4], - [ 5, 6, 7, 8, 9], - [10, 11, 12, 13, 14]]) - >>> z.get_coordinate_selection(([0, 2], [1, 3])) - array([ 1, 13]) - >>> z.set_coordinate_selection(([0, 2], [1, 3]), [-1, -2]) - >>> z[:] - array([[ 0, -1, 2, 3, 4], - [ 5, 6, 7, 8, 9], - [10, 11, 12, -2, 14]]) - -For convenience, coordinate indexing is also available via the ``vindex`` -property, e.g.:: - - >>> z.vindex[[0, 2], [1, 3]] - array([-1, -2]) - >>> z.vindex[[0, 2], [1, 3]] = [-3, -4] - >>> z[:] - array([[ 0, -3, 2, 3, 4], - [ 5, 6, 7, 8, 9], - [10, 11, 12, -4, 14]]) - -Indexing with a mask array -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Items can also be extracted by providing a Boolean mask. E.g.:: - - >>> z = zarr.array(np.arange(10)) - >>> z[:] - array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - >>> sel = np.zeros_like(z, dtype=bool) - >>> sel[1] = True - >>> sel[4] = True - >>> z.get_mask_selection(sel) - array([1, 4]) - >>> z.set_mask_selection(sel, [-1, -2]) - >>> z[:] - array([ 0, -1, 2, 3, -2, 5, 6, 7, 8, 9]) - -Here's a multidimensional example:: - - >>> z = zarr.array(np.arange(15).reshape(3, 5)) - >>> z[:] - array([[ 0, 1, 2, 3, 4], - [ 5, 6, 7, 8, 9], - [10, 11, 12, 13, 14]]) - >>> sel = np.zeros_like(z, dtype=bool) - >>> sel[0, 1] = True - >>> sel[2, 3] = True - >>> z.get_mask_selection(sel) - array([ 1, 13]) - >>> z.set_mask_selection(sel, [-1, -2]) - >>> z[:] - array([[ 0, -1, 2, 3, 4], - [ 5, 6, 7, 8, 9], - [10, 11, 12, -2, 14]]) - -For convenience, mask indexing is also available via the ``vindex`` property, -e.g.:: - - >>> z.vindex[sel] - array([-1, -2]) - >>> z.vindex[sel] = [-3, -4] - >>> z[:] - array([[ 0, -3, 2, 3, 4], - [ 5, 6, 7, 8, 9], - [10, 11, 12, -4, 14]]) - -Mask indexing is conceptually the same as coordinate indexing, and is -implemented internally via the same machinery. Both styles of indexing allow -selecting arbitrary items from an array, also known as point selection. - -Orthogonal indexing -~~~~~~~~~~~~~~~~~~~ - -Zarr arrays also support methods for orthogonal indexing, which allows -selections to be made along each dimension of an array independently. For -example, this allows selecting a subset of rows and/or columns from a -2-dimensional array. E.g.:: - - >>> z = zarr.array(np.arange(15).reshape(3, 5)) - >>> z[:] - array([[ 0, 1, 2, 3, 4], - [ 5, 6, 7, 8, 9], - [10, 11, 12, 13, 14]]) - >>> z.get_orthogonal_selection(([0, 2], slice(None))) # select first and third rows - array([[ 0, 1, 2, 3, 4], - [10, 11, 12, 13, 14]]) - >>> z.get_orthogonal_selection((slice(None), [1, 3])) # select second and fourth columns - array([[ 1, 3], - [ 6, 8], - [11, 13]]) - >>> z.get_orthogonal_selection(([0, 2], [1, 3])) # select rows [0, 2] and columns [1, 4] - array([[ 1, 3], - [11, 13]]) - -Data can also be modified, e.g.:: - - >>> z.set_orthogonal_selection(([0, 2], [1, 3]), [[-1, -2], [-3, -4]]) - >>> z[:] - array([[ 0, -1, 2, -2, 4], - [ 5, 6, 7, 8, 9], - [10, -3, 12, -4, 14]]) - -For convenience, the orthogonal indexing functionality is also available via the -``oindex`` property, e.g.:: - - >>> z = zarr.array(np.arange(15).reshape(3, 5)) - >>> z.oindex[[0, 2], :] # select first and third rows - array([[ 0, 1, 2, 3, 4], - [10, 11, 12, 13, 14]]) - >>> z.oindex[:, [1, 3]] # select second and fourth columns - array([[ 1, 3], - [ 6, 8], - [11, 13]]) - >>> z.oindex[[0, 2], [1, 3]] # select rows [0, 2] and columns [1, 4] - array([[ 1, 3], - [11, 13]]) - >>> z.oindex[[0, 2], [1, 3]] = [[-1, -2], [-3, -4]] - >>> z[:] - array([[ 0, -1, 2, -2, 4], - [ 5, 6, 7, 8, 9], - [10, -3, 12, -4, 14]]) - -Any combination of integer, slice, 1D integer array and/or 1D Boolean array can -be used for orthogonal indexing. - -Indexing fields in structured arrays -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -All selection methods support a ``fields`` parameter which allows retrieving or -replacing data for a specific field in an array with a structured dtype. E.g.:: - - >>> a = np.array([(b'aaa', 1, 4.2), - ... (b'bbb', 2, 8.4), - ... (b'ccc', 3, 12.6)], - ... dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - >>> z = zarr.array(a) - >>> z['foo'] - array([b'aaa', b'bbb', b'ccc'], - dtype='|S3') - >>> z['baz'] - array([ 4.2, 8.4, 12.6]) - >>> z.get_basic_selection(slice(0, 2), fields='bar') - array([1, 2], dtype=int32) - >>> z.get_coordinate_selection([0, 2], fields=['foo', 'baz']) - array([(b'aaa', 4.2), (b'ccc', 12.6)], - dtype=[('foo', 'S3'), ('baz', '>> z = zarr.open('data/example.zarr', mode='w', shape=1000000, dtype='i4') - -...is short-hand for:: - - >>> store = zarr.DirectoryStore('data/example.zarr') - >>> z = zarr.create(store=store, overwrite=True, shape=1000000, dtype='i4') - -...and the following code:: - - >>> root = zarr.open('data/example.zarr', mode='w') - -...is short-hand for:: - - >>> store = zarr.DirectoryStore('data/example.zarr') - >>> root = zarr.group(store=store, overwrite=True) - -Any other compatible storage class could be used in place of -:class:`zarr.storage.DirectoryStore` in the code examples above. For example, -here is an array stored directly into a Zip file, via the -:class:`zarr.storage.ZipStore` class:: - - >>> store = zarr.ZipStore('data/example.zip', mode='w') - >>> root = zarr.group(store=store) - >>> z = root.zeros('foo/bar', shape=(1000, 1000), chunks=(100, 100), dtype='i4') - >>> z[:] = 42 - >>> store.close() - -Re-open and check that data have been written:: - - >>> store = zarr.ZipStore('data/example.zip', mode='r') - >>> root = zarr.group(store=store) - >>> z = root['foo/bar'] - >>> z[:] - array([[42, 42, 42, ..., 42, 42, 42], - [42, 42, 42, ..., 42, 42, 42], - [42, 42, 42, ..., 42, 42, 42], - ..., - [42, 42, 42, ..., 42, 42, 42], - [42, 42, 42, ..., 42, 42, 42], - [42, 42, 42, ..., 42, 42, 42]], dtype=int32) - >>> store.close() - -Note that there are some limitations on how Zip files can be used, because items -within a Zip file cannot be updated in place. This means that data in the array -should only be written once and write operations should be aligned with chunk -boundaries. Note also that the ``close()`` method must be called after writing -any data to the store, otherwise essential records will not be written to the -underlying zip file. - -Another storage alternative is the :class:`zarr.storage.DBMStore` class, added -in Zarr version 2.2. This class allows any DBM-style database to be used for -storing an array or group. Here is an example using a Berkeley DB B-tree -database for storage (requires `bsddb3 -`_ to be installed):: - - >>> import bsddb3 - >>> store = zarr.DBMStore('data/example.bdb', open=bsddb3.btopen) - >>> root = zarr.group(store=store, overwrite=True) - >>> z = root.zeros('foo/bar', shape=(1000, 1000), chunks=(100, 100), dtype='i4') - >>> z[:] = 42 - >>> store.close() - -Also added in Zarr version 2.2 is the :class:`zarr.storage.LMDBStore` class which -enables the lightning memory-mapped database (LMDB) to be used for storing an array or -group (requires `lmdb `_ to be installed):: - - >>> store = zarr.LMDBStore('data/example.lmdb') - >>> root = zarr.group(store=store, overwrite=True) - >>> z = root.zeros('foo/bar', shape=(1000, 1000), chunks=(100, 100), dtype='i4') - >>> z[:] = 42 - >>> store.close() - -In Zarr version 2.3 is the :class:`zarr.storage.SQLiteStore` class which -enables the SQLite database to be used for storing an array or group (requires -Python is built with SQLite support):: - - >>> store = zarr.SQLiteStore('data/example.sqldb') - >>> root = zarr.group(store=store, overwrite=True) - >>> z = root.zeros('foo/bar', shape=(1000, 1000), chunks=(100, 100), dtype='i4') - >>> z[:] = 42 - >>> store.close() - -Also added in Zarr version 2.3 are two storage classes for interfacing with server-client -databases. The :class:`zarr.storage.RedisStore` class interfaces `Redis `_ -(an in memory data structure store), and the :class:`zarr.storage.MongoDB` class interfaces -with `MongoDB `_ (an object oriented NoSQL database). These stores -respectively require the `redis-py `_ and -`pymongo `_ packages to be installed. - -For compatibility with the `N5 `_ data format, Zarr also provides -an N5 backend (this is currently an experimental feature). Similar to the zip storage class, an -:class:`zarr.n5.N5Store` can be instantiated directly:: - - >>> store = zarr.N5Store('data/example.n5') - >>> root = zarr.group(store=store) - >>> z = root.zeros('foo/bar', shape=(1000, 1000), chunks=(100, 100), dtype='i4') - >>> z[:] = 42 - -For convenience, the N5 backend will automatically be chosen when the filename -ends with `.n5`:: - - >>> root = zarr.open('data/example.n5', mode='w') - -Distributed/cloud storage -~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is also possible to use distributed storage systems. The Dask project has -implementations of the ``MutableMapping`` interface for Amazon S3 (`S3Map -`_), Hadoop -Distributed File System (`HDFSMap -`_) and -Google Cloud Storage (`GCSMap -`_), which -can be used with Zarr. - -Here is an example using S3Map to read an array created previously:: - - >>> import s3fs - >>> import zarr - >>> s3 = s3fs.S3FileSystem(anon=True, client_kwargs=dict(region_name='eu-west-2')) - >>> store = s3fs.S3Map(root='zarr-demo/store', s3=s3, check=False) - >>> root = zarr.group(store=store) - >>> z = root['foo/bar/baz'] - >>> z - - >>> z.info - Name : /foo/bar/baz - Type : zarr.core.Array - Data type : |S1 - Shape : (21,) - Chunk shape : (7,) - Order : C - Read-only : False - Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0) - Store type : fsspec.mapping.FSMap - No. bytes : 21 - Chunks initialized : 3/3 - >>> z[:] - array([b'H', b'e', b'l', b'l', b'o', b' ', b'f', b'r', b'o', b'm', b' ', - b't', b'h', b'e', b' ', b'c', b'l', b'o', b'u', b'd', b'!'], - dtype='|S1') - >>> z[:].tostring() - b'Hello from the cloud!' - -Zarr now also has a builtin storage backend for Azure Blob Storage. -The class is :class:`zarr.storage.ABSStore` (requires -`azure-storage-blob `_ -to be installed):: - - >>> import azure.storage.blob - >>> container_client = azure.storage.blob.ContainerClient(...) # doctest: +SKIP - >>> store = zarr.ABSStore(client=container_client, prefix='zarr-testing') # doctest: +SKIP - >>> root = zarr.group(store=store, overwrite=True) # doctest: +SKIP - >>> z = root.zeros('foo/bar', shape=(1000, 1000), chunks=(100, 100), dtype='i4') # doctest: +SKIP - >>> z[:] = 42 # doctest: +SKIP - -When using an actual storage account, provide ``account_name`` and -``account_key`` arguments to :class:`zarr.storage.ABSStore`, the -above client is just testing against the emulator. Please also note -that this is an experimental feature. - -Note that retrieving data from a remote service via the network can be significantly -slower than retrieving data from a local file system, and will depend on network latency -and bandwidth between the client and server systems. If you are experiencing poor -performance, there are several things you can try. One option is to increase the array -chunk size, which will reduce the number of chunks and thus reduce the number of network -round-trips required to retrieve data for an array (and thus reduce the impact of network -latency). Another option is to try to increase the compression ratio by changing -compression options or trying a different compressor (which will reduce the impact of -limited network bandwidth). - -As of version 2.2, Zarr also provides the :class:`zarr.storage.LRUStoreCache` -which can be used to implement a local in-memory cache layer over a remote -store. E.g.:: - - >>> s3 = s3fs.S3FileSystem(anon=True, client_kwargs=dict(region_name='eu-west-2')) - >>> store = s3fs.S3Map(root='zarr-demo/store', s3=s3, check=False) - >>> cache = zarr.LRUStoreCache(store, max_size=2**28) - >>> root = zarr.group(store=cache) - >>> z = root['foo/bar/baz'] - >>> from timeit import timeit - >>> # first data access is relatively slow, retrieved from store - ... timeit('print(z[:].tostring())', number=1, globals=globals()) # doctest: +SKIP - b'Hello from the cloud!' - 0.1081731989979744 - >>> # second data access is faster, uses cache - ... timeit('print(z[:].tostring())', number=1, globals=globals()) # doctest: +SKIP - b'Hello from the cloud!' - 0.0009490990014455747 - -If you are still experiencing poor performance with distributed/cloud storage, -please raise an issue on the GitHub issue tracker with any profiling data you -can provide, as there may be opportunities to optimise further either within -Zarr or within the mapping interface to the storage. - -IO with ``fsspec`` -~~~~~~~~~~~~~~~~~~ - -As of version 2.5, zarr supports passing URLs directly to `fsspec`_, -and having it create the "mapping" instance automatically. This means, that -for all of the backend storage implementations `supported by fsspec`_, -you can skip importing and configuring the storage explicitly. -For example:: - - >>> g = zarr.open_group("s3://zarr-demo/store", storage_options={'anon': True}) # doctest: +SKIP - >>> g['foo/bar/baz'][:].tobytes() # doctest: +SKIP - b'Hello from the cloud!' - -The provision of the protocol specifier "s3://" will select the correct backend. -Notice the kwargs ``storage_options``, used to pass parameters to that backend. - -As of version 2.6, write mode and complex URLs are also supported, such as:: - - >>> g = zarr.open_group("simplecache::s3://zarr-demo/store", - ... storage_options={"s3": {'anon': True}}) # doctest: +SKIP - >>> g['foo/bar/baz'][:].tobytes() # downloads target file # doctest: +SKIP - b'Hello from the cloud!' - >>> g['foo/bar/baz'][:].tobytes() # uses cached file # doctest: +SKIP - b'Hello from the cloud!' - -The second invocation here will be much faster. Note that the ``storage_options`` -have become more complex here, to account for the two parts of the supplied -URL. - -.. _fsspec: https://filesystem-spec.readthedocs.io/en/latest/ - -.. _supported by fsspec: https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - -.. _tutorial_copy: - -Consolidating metadata -~~~~~~~~~~~~~~~~~~~~~~ - -Since there is a significant overhead for every connection to a cloud object -store such as S3, the pattern described in the previous section may incur -significant latency while scanning the metadata of the array hierarchy, even -though each individual metadata object is small. For cases such as these, once -the data are static and can be regarded as read-only, at least for the -metadata/structure of the array hierarchy, the many metadata objects can be -consolidated into a single one via -:func:`zarr.convenience.consolidate_metadata`. Doing this can greatly increase -the speed of reading the array metadata, e.g.:: - - >>> zarr.consolidate_metadata(store) # doctest: +SKIP - -This creates a special key with a copy of all of the metadata from all of the -metadata objects in the store. - -Later, to open a Zarr store with consolidated metadata, use -:func:`zarr.convenience.open_consolidated`, e.g.:: - - >>> root = zarr.open_consolidated(store) # doctest: +SKIP - -This uses the special key to read all of the metadata in a single call to the -backend storage. - -Note that, the hierarchy could still be opened in the normal way and altered, -causing the consolidated metadata to become out of sync with the real state of -the array hierarchy. In this case, -:func:`zarr.convenience.consolidate_metadata` would need to be called again. - -To protect against consolidated metadata accidentally getting out of sync, the -root group returned by :func:`zarr.convenience.open_consolidated` is read-only -for the metadata, meaning that no new groups or arrays can be created, and -arrays cannot be resized. However, data values with arrays can still be updated. - -Copying/migrating data ----------------------- - -If you have some data in an HDF5 file and would like to copy some or all of it -into a Zarr group, or vice-versa, the :func:`zarr.convenience.copy` and -:func:`zarr.convenience.copy_all` functions can be used. Here's an example -copying a group named 'foo' from an HDF5 file to a Zarr group:: - - >>> import h5py - >>> import zarr - >>> import numpy as np - >>> source = h5py.File('data/example.h5', mode='w') - >>> foo = source.create_group('foo') - >>> baz = foo.create_dataset('bar/baz', data=np.arange(100), chunks=(50,)) - >>> spam = source.create_dataset('spam', data=np.arange(100, 200), chunks=(30,)) - >>> zarr.tree(source) - / - ├── foo - │ └── bar - │ └── baz (100,) int64 - └── spam (100,) int64 - >>> dest = zarr.open_group('data/example.zarr', mode='w') - >>> from sys import stdout - >>> zarr.copy(source['foo'], dest, log=stdout) - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - all done: 3 copied, 0 skipped, 800 bytes copied - (3, 0, 800) - >>> dest.tree() # N.B., no spam - / - └── foo - └── bar - └── baz (100,) int64 - >>> source.close() - -If rather than copying a single group or array you would like to copy all -groups and arrays, use :func:`zarr.convenience.copy_all`, e.g.:: - - >>> source = h5py.File('data/example.h5', mode='r') - >>> dest = zarr.open_group('data/example2.zarr', mode='w') - >>> zarr.copy_all(source, dest, log=stdout) - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - copy /spam (100,) int64 - all done: 4 copied, 0 skipped, 1,600 bytes copied - (4, 0, 1600) - >>> dest.tree() - / - ├── foo - │ └── bar - │ └── baz (100,) int64 - └── spam (100,) int64 - -If you need to copy data between two Zarr groups, the -:func:`zarr.convenience.copy` and :func:`zarr.convenience.copy_all` functions can -be used and provide the most flexibility. However, if you want to copy data -in the most efficient way possible, without changing any configuration options, -the :func:`zarr.convenience.copy_store` function can be used. This function -copies data directly between the underlying stores, without any decompression or -re-compression, and so should be faster. E.g.:: - - >>> import zarr - >>> import numpy as np - >>> store1 = zarr.DirectoryStore('data/example.zarr') - >>> root = zarr.group(store1, overwrite=True) - >>> baz = root.create_dataset('foo/bar/baz', data=np.arange(100), chunks=(50,)) - >>> spam = root.create_dataset('spam', data=np.arange(100, 200), chunks=(30,)) - >>> root.tree() - / - ├── foo - │ └── bar - │ └── baz (100,) int64 - └── spam (100,) int64 - >>> from sys import stdout - >>> store2 = zarr.ZipStore('data/example.zip', mode='w') - >>> zarr.copy_store(store1, store2, log=stdout) - copy .zgroup - copy foo/.zgroup - copy foo/bar/.zgroup - copy foo/bar/baz/.zarray - copy foo/bar/baz/0 - copy foo/bar/baz/1 - copy spam/.zarray - copy spam/0 - copy spam/1 - copy spam/2 - copy spam/3 - all done: 11 copied, 0 skipped, 1,138 bytes copied - (11, 0, 1138) - >>> new_root = zarr.group(store2) - >>> new_root.tree() - / - ├── foo - │ └── bar - │ └── baz (100,) int64 - └── spam (100,) int64 - >>> new_root['foo/bar/baz'][:] - array([ 0, 1, 2, ..., 97, 98, 99]) - >>> store2.close() # zip stores need to be closed - -.. _tutorial_strings: - -String arrays -------------- - -There are several options for storing arrays of strings. - -If your strings are all ASCII strings, and you know the maximum length of the string in -your array, then you can use an array with a fixed-length bytes dtype. E.g.:: - - >>> z = zarr.zeros(10, dtype='S6') - >>> z - - >>> z[0] = b'Hello' - >>> z[1] = b'world!' - >>> z[:] - array([b'Hello', b'world!', b'', b'', b'', b'', b'', b'', b'', b''], - dtype='|S6') - -A fixed-length unicode dtype is also available, e.g.:: - - >>> greetings = ['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', 'Hei maailma!', - ... 'Xin chào thế giới', 'Njatjeta Botë!', 'Γεια σου κόσμε!', - ... 'こんにちは世界', '世界，你好！', 'Helló, világ!', 'Zdravo svete!', - ... 'เฮลโลเวิลด์'] - >>> text_data = greetings * 10000 - >>> z = zarr.array(text_data, dtype='U20') - >>> z - - >>> z[:] - array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., - 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], - dtype='>> import numcodecs - >>> z = zarr.array(text_data, dtype=object, object_codec=numcodecs.VLenUTF8()) - >>> z - - >>> z.filters - [VLenUTF8()] - >>> z[:] - array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., - 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object) - -As a convenience, ``dtype=str`` (or ``dtype=unicode`` on Python 2.7) can be used, which -is a short-hand for ``dtype=object, object_codec=numcodecs.VLenUTF8()``, e.g.:: - - >>> z = zarr.array(text_data, dtype=str) - >>> z - - >>> z.filters - [VLenUTF8()] - >>> z[:] - array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., - 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object) - -Variable-length byte strings are also supported via ``dtype=object``. Again an -``object_codec`` is required, which can be one of :class:`numcodecs.VLenBytes` or -:class:`numcodecs.Pickle`. For convenience, ``dtype=bytes`` (or ``dtype=str`` on Python -2.7) can be used as a short-hand for ``dtype=object, object_codec=numcodecs.VLenBytes()``, -e.g.:: - - >>> bytes_data = [g.encode('utf-8') for g in greetings] * 10000 - >>> z = zarr.array(bytes_data, dtype=bytes) - >>> z - - >>> z.filters - [VLenBytes()] - >>> z[:] - array([b'\xc2\xa1Hola mundo!', b'Hej V\xc3\xa4rlden!', b'Servus Woid!', - ..., b'Hell\xc3\xb3, vil\xc3\xa1g!', b'Zdravo svete!', - b'\xe0\xb9\x80\xe0\xb8\xae\xe0\xb8\xa5\xe0\xb9\x82\xe0\xb8\xa5\xe0\xb9\x80\xe0\xb8\xa7\xe0\xb8\xb4\xe0\xb8\xa5\xe0\xb8\x94\xe0\xb9\x8c'], dtype=object) - -If you know ahead of time all the possible string values that can occur, you could -also use the :class:`numcodecs.Categorize` codec to encode each unique string value as an -integer. E.g.:: - - >>> categorize = numcodecs.Categorize(greetings, dtype=object) - >>> z = zarr.array(text_data, dtype=object, object_codec=categorize) - >>> z - - >>> z.filters - [Categorize(dtype='|O', astype='|u1', labels=['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ...])] - >>> z[:] - array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', ..., - 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object) - - -.. _tutorial_objects: - -Object arrays -------------- - -Zarr supports arrays with an "object" dtype. This allows arrays to contain any type of -object, such as variable length unicode strings, or variable length arrays of numbers, or -other possibilities. When creating an object array, a codec must be provided via the -``object_codec`` argument. This codec handles encoding (serialization) of Python objects. -The best codec to use will depend on what type of objects are present in the array. - -At the time of writing there are three codecs available that can serve as a general -purpose object codec and support encoding of a mixture of object types: -:class:`numcodecs.JSON`, :class:`numcodecs.MsgPack`. and :class:`numcodecs.Pickle`. - -For example, using the JSON codec:: - - >>> z = zarr.empty(5, dtype=object, object_codec=numcodecs.JSON()) - >>> z[0] = 42 - >>> z[1] = 'foo' - >>> z[2] = ['bar', 'baz', 'qux'] - >>> z[3] = {'a': 1, 'b': 2.2} - >>> z[:] - array([42, 'foo', list(['bar', 'baz', 'qux']), {'a': 1, 'b': 2.2}, None], dtype=object) - -Not all codecs support encoding of all object types. The -:class:`numcodecs.Pickle` codec is the most flexible, supporting encoding any type -of Python object. However, if you are sharing data with anyone other than yourself, then -Pickle is not recommended as it is a potential security risk. This is because malicious -code can be embedded within pickled data. The JSON and MsgPack codecs do not have any -security issues and support encoding of unicode strings, lists and dictionaries. -MsgPack is usually faster for both encoding and decoding. - -Ragged arrays -~~~~~~~~~~~~~ - -If you need to store an array of arrays, where each member array can be of any length -and stores the same primitive type (a.k.a. a ragged array), the -:class:`numcodecs.VLenArray` codec can be used, e.g.:: - - >>> z = zarr.empty(4, dtype=object, object_codec=numcodecs.VLenArray(int)) - >>> z - - >>> z.filters - [VLenArray(dtype='>> z[0] = np.array([1, 3, 5]) - >>> z[1] = np.array([4]) - >>> z[2] = np.array([7, 9, 14]) - >>> z[:] - array([array([1, 3, 5]), array([4]), array([ 7, 9, 14]), - array([], dtype=int64)], dtype=object) - -As a convenience, ``dtype='array:T'`` can be used as a short-hand for -``dtype=object, object_codec=numcodecs.VLenArray('T')``, where 'T' can be any NumPy -primitive dtype such as 'i4' or 'f8'. E.g.:: - - >>> z = zarr.empty(4, dtype='array:i8') - >>> z - - >>> z.filters - [VLenArray(dtype='>> z[0] = np.array([1, 3, 5]) - >>> z[1] = np.array([4]) - >>> z[2] = np.array([7, 9, 14]) - >>> z[:] - array([array([1, 3, 5]), array([4]), array([ 7, 9, 14]), - array([], dtype=int64)], dtype=object) - -.. _tutorial_chunks: - -Chunk optimizations -------------------- - -.. _tutorial_chunks_shape: - -Chunk size and shape -~~~~~~~~~~~~~~~~~~~~ - -In general, chunks of at least 1 megabyte (1M) uncompressed size seem to provide -better performance, at least when using the Blosc compression library. - -The optimal chunk shape will depend on how you want to access the data. E.g., -for a 2-dimensional array, if you only ever take slices along the first -dimension, then chunk across the second dimenson. If you know you want to chunk -across an entire dimension you can use ``None`` or ``-1`` within the ``chunks`` -argument, e.g.:: - - >>> z1 = zarr.zeros((10000, 10000), chunks=(100, None), dtype='i4') - >>> z1.chunks - (100, 10000) - -Alternatively, if you only ever take slices along the second dimension, then -chunk across the first dimension, e.g.:: - - >>> z2 = zarr.zeros((10000, 10000), chunks=(None, 100), dtype='i4') - >>> z2.chunks - (10000, 100) - -If you require reasonable performance for both access patterns then you need to -find a compromise, e.g.:: - - >>> z3 = zarr.zeros((10000, 10000), chunks=(1000, 1000), dtype='i4') - >>> z3.chunks - (1000, 1000) - -If you are feeling lazy, you can let Zarr guess a chunk shape for your data by -providing ``chunks=True``, although please note that the algorithm for guessing -a chunk shape is based on simple heuristics and may be far from optimal. E.g.:: - - >>> z4 = zarr.zeros((10000, 10000), chunks=True, dtype='i4') - >>> z4.chunks - (625, 625) - -If you know you are always going to be loading the entire array into memory, you -can turn off chunks by providing ``chunks=False``, in which case there will be -one single chunk for the array:: - - >>> z5 = zarr.zeros((10000, 10000), chunks=False, dtype='i4') - >>> z5.chunks - (10000, 10000) - -.. _tutorial_chunks_order: - -Chunk memory layout -~~~~~~~~~~~~~~~~~~~ - -The order of bytes **within each chunk** of an array can be changed via the -``order`` keyword argument, to use either C or Fortran layout. For -multi-dimensional arrays, these two layouts may provide different compression -ratios, depending on the correlation structure within the data. E.g.:: - - >>> a = np.arange(100000000, dtype='i4').reshape(10000, 10000).T - >>> c = zarr.array(a, chunks=(1000, 1000)) - >>> c.info - Type : zarr.core.Array - Data type : int32 - Shape : (10000, 10000) - Chunk shape : (1000, 1000) - Order : C - Read-only : False - Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0) - Store type : builtins.dict - No. bytes : 400000000 (381.5M) - No. bytes stored : 6696010 (6.4M) - Storage ratio : 59.7 - Chunks initialized : 100/100 - >>> f = zarr.array(a, chunks=(1000, 1000), order='F') - >>> f.info - Type : zarr.core.Array - Data type : int32 - Shape : (10000, 10000) - Chunk shape : (1000, 1000) - Order : F - Read-only : False - Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0) - Store type : builtins.dict - No. bytes : 400000000 (381.5M) - No. bytes stored : 4684636 (4.5M) - Storage ratio : 85.4 - Chunks initialized : 100/100 - -In the above example, Fortran order gives a better compression ratio. This is an -artifical example but illustrates the general point that changing the order of -bytes within chunks of an array may improve the compression ratio, depending on -the structure of the data, the compression algorithm used, and which compression -filters (e.g., byte-shuffle) have been applied. - -.. _tutorial_rechunking: - -Changing chunk shapes (rechunking) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Sometimes you are not free to choose the initial chunking of your input data, or -you might have data saved with chunking which is not optimal for the analysis you -have planned. In such cases it can be advantageous to re-chunk the data. For small -datasets, or when the mismatch between input and output chunks is small -such that only a few chunks of the input dataset need to be read to create each -chunk in the output array, it is sufficient to simply copy the data to a new array -with the desired chunking, e.g. :: - - >>> a = zarr.zeros((10000, 10000), chunks=(100,100), dtype='uint16', store='a.zarr') - >>> b = zarr.array(a, chunks=(100, 200), store='b.zarr') - -If the chunk shapes mismatch, however, a simple copy can lead to non-optimal data -access patterns and incur a substantial performance hit when using -file based stores. One of the most pathological examples is -switching from column-based chunking to row-based chunking e.g. :: - - >>> a = zarr.zeros((10000,10000), chunks=(10000, 1), dtype='uint16, store='a.zarr') - >>> b = zarr.array(a, chunks=(1,10000), store='b.zarr') - -which will require every chunk in the input data set to be repeatedly read when creating -each output chunk. If the entire array will fit within memory, this is simply resolved -by forcing the entire input array into memory as a numpy array before converting -back to zarr with the desired chunking. :: - - >>> a = zarr.zeros((10000,10000), chunks=(10000, 1), dtype='uint16, store='a.zarr') - >>> b = a[...] - >>> c = zarr.array(b, chunks=(1,10000), store='c.zarr') - -For data sets which have mismatched chunks and which do not fit in memory, a -more sophisticated approach to rechunking, such as offered by the -`rechunker `_ package and discussed -`here `_ -may offer a substantial improvement in performance. - -.. _tutorial_sync: - -Parallel computing and synchronization --------------------------------------- - -Zarr arrays have been designed for use as the source or sink for data in -parallel computations. By data source we mean that multiple concurrent read -operations may occur. By data sink we mean that multiple concurrent write -operations may occur, with each writer updating a different region of the -array. Zarr arrays have **not** been designed for situations where multiple -readers and writers are concurrently operating on the same array. - -Both multi-threaded and multi-process parallelism are possible. The bottleneck -for most storage and retrieval operations is compression/decompression, and the -Python global interpreter lock (GIL) is released wherever possible during these -operations, so Zarr will generally not block other Python threads from running. - -When using a Zarr array as a data sink, some synchronization (locking) may be -required to avoid data loss, depending on how data are being updated. If each -worker in a parallel computation is writing to a separate region of the array, -and if region boundaries are perfectly aligned with chunk boundaries, then no -synchronization is required. However, if region and chunk boundaries are not -perfectly aligned, then synchronization is required to avoid two workers -attempting to modify the same chunk at the same time, which could result in data -loss. - -To give a simple example, consider a 1-dimensional array of length 60, ``z``, -divided into three chunks of 20 elements each. If three workers are running and -each attempts to write to a 20 element region (i.e., ``z[0:20]``, ``z[20:40]`` -and ``z[40:60]``) then each worker will be writing to a separate chunk and no -synchronization is required. However, if two workers are running and each -attempts to write to a 30 element region (i.e., ``z[0:30]`` and ``z[30:60]``) -then it is possible both workers will attempt to modify the middle chunk at the -same time, and synchronization is required to prevent data loss. - -Zarr provides support for chunk-level synchronization. E.g., create an array -with thread synchronization:: - - >>> z = zarr.zeros((10000, 10000), chunks=(1000, 1000), dtype='i4', - ... synchronizer=zarr.ThreadSynchronizer()) - >>> z - - -This array is safe to read or write within a multi-threaded program. - -Zarr also provides support for process synchronization via file locking, -provided that all processes have access to a shared file system, and provided -that the underlying file system supports file locking (which is not the case for -some networked file systems). E.g.:: - - >>> synchronizer = zarr.ProcessSynchronizer('data/example.sync') - >>> z = zarr.open_array('data/example', mode='w', shape=(10000, 10000), - ... chunks=(1000, 1000), dtype='i4', - ... synchronizer=synchronizer) - >>> z - - -This array is safe to read or write from multiple processes. - -When using multiple processes to parallelize reads or writes on arrays using the Blosc -compression library, it may be necessary to set ``numcodecs.blosc.use_threads = False``, -as otherwise Blosc may share incorrect global state amongst processes causing programs -to hang. See also the section on :ref:`tutorial_tips_blosc` below. - -Please note that support for parallel computing is an area of ongoing research -and development. If you are using Zarr for parallel computing, we welcome -feedback, experience, discussion, ideas and advice, particularly about issues -related to data integrity and performance. - -.. _tutorial_pickle: - -Pickle support --------------- - -Zarr arrays and groups can be pickled, as long as the underlying store object can be -pickled. Instances of any of the storage classes provided in the :mod:`zarr.storage` -module can be pickled, as can the built-in ``dict`` class which can also be used for -storage. - -Note that if an array or group is backed by an in-memory store like a ``dict`` or -:class:`zarr.storage.MemoryStore`, then when it is pickled all of the store data will be -included in the pickled data. However, if an array or group is backed by a persistent -store like a :class:`zarr.storage.DirectoryStore`, :class:`zarr.storage.ZipStore` or -:class:`zarr.storage.DBMStore` then the store data **are not** pickled. The only thing -that is pickled is the necessary parameters to allow the store to re-open any -underlying files or databases upon being unpickled. - -E.g., pickle/unpickle an in-memory array:: - - >>> import pickle - >>> z1 = zarr.array(np.arange(100000)) - >>> s = pickle.dumps(z1) - >>> len(s) > 5000 # relatively large because data have been pickled - True - >>> z2 = pickle.loads(s) - >>> z1 == z2 - True - >>> np.all(z1[:] == z2[:]) - True - -E.g., pickle/unpickle an array stored on disk:: - - >>> z3 = zarr.open('data/walnuts.zarr', mode='w', shape=100000, dtype='i8') - >>> z3[:] = np.arange(100000) - >>> s = pickle.dumps(z3) - >>> len(s) < 200 # small because no data have been pickled - True - >>> z4 = pickle.loads(s) - >>> z3 == z4 - True - >>> np.all(z3[:] == z4[:]) - True - -.. _tutorial_datetime: - -Datetimes and timedeltas ------------------------- - -NumPy's ``datetime64`` ('M8') and ``timedelta64`` ('m8') dtypes are supported for Zarr -arrays, as long as the units are specified. E.g.:: - - >>> z = zarr.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='M8[D]') - >>> z - - >>> z[:] - array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64[D]') - >>> z[0] - numpy.datetime64('2007-07-13') - >>> z[0] = '1999-12-31' - >>> z[:] - array(['1999-12-31', '2006-01-13', '2010-08-13'], dtype='datetime64[D]') - -.. _tutorial_tips: - -Usage tips ----------- - -.. _tutorial_tips_copy: - -Copying large arrays -~~~~~~~~~~~~~~~~~~~~ - -Data can be copied between large arrays without needing much memory, e.g.:: - - >>> z1 = zarr.empty((10000, 10000), chunks=(1000, 1000), dtype='i4') - >>> z1[:] = 42 - >>> z2 = zarr.empty_like(z1) - >>> z2[:] = z1 - -Internally the example above works chunk-by-chunk, extracting only the data from -``z1`` required to fill each chunk in ``z2``. The source of the data (``z1``) -could equally be an h5py Dataset. - -.. _tutorial_tips_blosc: - -Configuring Blosc -~~~~~~~~~~~~~~~~~ - -The Blosc compressor is able to use multiple threads internally to accelerate -compression and decompression. By default, Blosc uses up to 8 -internal threads. The number of Blosc threads can be changed to increase or -decrease this number, e.g.:: - - >>> from numcodecs import blosc - >>> blosc.set_nthreads(2) # doctest: +SKIP - 8 - -When a Zarr array is being used within a multi-threaded program, Zarr -automatically switches to using Blosc in a single-threaded -"contextual" mode. This is generally better as it allows multiple -program threads to use Blosc simultaneously and prevents CPU thrashing -from too many active threads. If you want to manually override this -behaviour, set the value of the ``blosc.use_threads`` variable to -``True`` (Blosc always uses multiple internal threads) or ``False`` -(Blosc always runs in single-threaded contextual mode). To re-enable -automatic switching, set ``blosc.use_threads`` to ``None``. - -Please note that if Zarr is being used within a multi-process program, Blosc may not -be safe to use in multi-threaded mode and may cause the program to hang. If using Blosc -in a multi-process program then it is recommended to set ``blosc.use_threads = False``. diff --git a/docs/user-guide/arrays.rst b/docs/user-guide/arrays.rst new file mode 100644 index 0000000000..c27f1296b9 --- /dev/null +++ b/docs/user-guide/arrays.rst @@ -0,0 +1,658 @@ +.. only:: doctest + + >>> import shutil + >>> shutil.rmtree('data', ignore_errors=True) + +.. _user-guide-arrays: + +Working with arrays +=================== + +Creating an array +----------------- + +Zarr has several functions for creating arrays. For example:: + + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> z = zarr.create_array(store=store, shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') + >>> z + + +The code above creates a 2-dimensional array of 32-bit integers with 10000 rows +and 10000 columns, divided into chunks where each chunk has 1000 rows and 1000 +columns (and so there will be 100 chunks in total). The data is written to a +:class:`zarr.storage.MemoryStore` (e.g. an in-memory dict). See +:ref:`user-guide-persist` for details on storing arrays in other stores. + +For a complete list of array creation routines see the :mod:`zarr` +module documentation. + +.. _user-guide-array: + +Reading and writing data +------------------------ + +Zarr arrays support a similar interface to `NumPy `_ +arrays for reading and writing data. For example, the entire array can be filled +with a scalar value:: + + >>> z[:] = 42 + +Regions of the array can also be written to, e.g.:: + + >>> import numpy as np + >>> + >>> z[0, :] = np.arange(10000) + >>> z[:, 0] = np.arange(10000) + +The contents of the array can be retrieved by slicing, which will load the +requested region into memory as a NumPy array, e.g.:: + + >>> z[0, 0] + array(0, dtype=int32) + >>> z[-1, -1] + array(42, dtype=int32) + >>> z[0, :] + array([ 0, 1, 2, ..., 9997, 9998, 9999], + shape=(10000,), dtype=int32) + >>> z[:, 0] + array([ 0, 1, 2, ..., 9997, 9998, 9999], + shape=(10000,), dtype=int32) + >>> z[:] + array([[ 0, 1, 2, ..., 9997, 9998, 9999], + [ 1, 42, 42, ..., 42, 42, 42], + [ 2, 42, 42, ..., 42, 42, 42], + ..., + [9997, 42, 42, ..., 42, 42, 42], + [9998, 42, 42, ..., 42, 42, 42], + [9999, 42, 42, ..., 42, 42, 42]], + shape=(10000, 10000), dtype=int32) + +Read more about NumPy-style indexing can be found in the +`NumPy documentation `_. + +.. _user-guide-persist: + +Persistent arrays +----------------- + +In the examples above, compressed data for each chunk of the array was stored in +main memory. Zarr arrays can also be stored on a file system, enabling +persistence of data between sessions. To do this, we can change the store +argument to point to a filesystem path:: + + >>> z1 = zarr.create_array(store='data/example-1.zarr', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') + +The array above will store its configuration metadata and all compressed chunk +data in a directory called ``'data/example-1.zarr'`` relative to the current working +directory. The :func:`zarr.create_array` function provides a convenient way +to create a new persistent array or continue working with an existing +array. Note, there is no need to close an array: data are automatically +flushed to disk, and files are automatically closed whenever an array is modified. + +Persistent arrays support the same interface for reading and writing data, +e.g.:: + + >>> z1[:] = 42 + >>> z1[0, :] = np.arange(10000) + >>> z1[:, 0] = np.arange(10000) + +Check that the data have been written and can be read again:: + + >>> z2 = zarr.open_array('data/example-1.zarr', mode='r') + >>> np.all(z1[:] == z2[:]) + np.True_ + +If you are just looking for a fast and convenient way to save NumPy arrays to +disk then load back into memory later, the functions +:func:`zarr.save` and :func:`zarr.load` may be +useful. E.g.:: + + >>> a = np.arange(10) + >>> zarr.save('data/example-2.zarr', a) + >>> zarr.load('data/example-2.zarr') + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + +Please note that there are a number of other options for persistent array +storage, see the :ref:`Storage Guide ` guide for more details. + +.. _user-guide-resize: + +Resizing and appending +---------------------- + +A Zarr array can be resized, which means that any of its dimensions can be +increased or decreased in length. For example:: + + >>> z = zarr.create_array(store='data/example-3.zarr', shape=(10000, 10000), dtype='int32',chunks=(1000, 1000)) + >>> z[:] = 42 + >>> z.shape + (10000, 10000) + >>> z.resize((20000, 10000)) + >>> z.shape + (20000, 10000) + +Note that when an array is resized, the underlying data are not rearranged in +any way. If one or more dimensions are shrunk, any chunks falling outside the +new array shape will be deleted from the underlying store. + +:func:`zarr.Array.append` is provided as a convenience function, which can be +used to append data to any axis. E.g.:: + + >>> a = np.arange(10000000, dtype='int32').reshape(10000, 1000) + >>> z = zarr.create_array(store='data/example-4.zarr', shape=a.shape, dtype=a.dtype, chunks=(1000, 100)) + >>> z[:] = a + >>> z.shape + (10000, 1000) + >>> z.append(a) + (20000, 1000) + >>> z.append(np.vstack([a, a]), axis=1) + (20000, 2000) + >>> z.shape + (20000, 2000) + +.. _user-guide-compress: + +Compressors +----------- + +A number of different compressors can be used with Zarr. Zarr includes Blosc, +Zstandard and Gzip compressors. Additional compressors are available through +a separate package called NumCodecs_ which provides various +compressor libraries including LZ4, Zlib, BZ2 and LZMA. +Different compressors can be provided via the ``compressors`` keyword +argument accepted by all array creation functions. For example:: + + >>> compressors = zarr.codecs.BloscCodec(cname='zstd', clevel=3, shuffle=zarr.codecs.BloscShuffle.bitshuffle) + >>> data = np.arange(100000000, dtype='int32').reshape(10000, 10000) + >>> z = zarr.create_array(store='data/example-5.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors) + >>> z[:] = data + >>> z.compressors + (BloscCodec(typesize=4, cname=, clevel=3, shuffle=, blocksize=0),) + +This array above will use Blosc as the primary compressor, using the Zstandard +algorithm (compression level 3) internally within Blosc, and with the +bit-shuffle filter applied. + +When using a compressor, it can be useful to get some diagnostics on the +compression ratio. Zarr arrays provide the :attr:`zarr.Array.info` property +which can be used to print useful diagnostics, e.g.:: + + >>> z.info + Type : Array + Zarr format : 3 + Data type : Int32(endianness='little') + Fill value : 0 + Shape : (10000, 10000) + Chunk shape : (1000, 1000) + Order : C + Read-only : False + Store type : LocalStore + Filters : () + Serializer : BytesCodec(endian=) + Compressors : (BloscCodec(typesize=4, cname=, clevel=3, shuffle=, blocksize=0),) + No. bytes : 400000000 (381.5M) + +The :func:`zarr.Array.info_complete` method inspects the underlying store and +prints additional diagnostics, e.g.:: + + >>> z.info_complete() + Type : Array + Zarr format : 3 + Data type : Int32(endianness='little') + Fill value : 0 + Shape : (10000, 10000) + Chunk shape : (1000, 1000) + Order : C + Read-only : False + Store type : LocalStore + Filters : () + Serializer : BytesCodec(endian=) + Compressors : (BloscCodec(typesize=4, cname=, clevel=3, shuffle=, blocksize=0),) + No. bytes : 400000000 (381.5M) + No. bytes stored : 3558573 + Storage ratio : 112.4 + Chunks Initialized : 100 + +.. note:: + :func:`zarr.Array.info_complete` will inspect the underlying store and may + be slow for large arrays. Use :attr:`zarr.Array.info` if detailed storage + statistics are not needed. + +If you don't specify a compressor, by default Zarr uses the Zstandard +compressor. + +In addition to Blosc and Zstandard, other compression libraries can also be used. For example, +here is an array using Gzip compression, level 1:: + + >>> data = np.arange(100000000, dtype='int32').reshape(10000, 10000) + >>> z = zarr.create_array(store='data/example-6.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=zarr.codecs.GzipCodec(level=1)) + >>> z[:] = data + >>> z.compressors + (GzipCodec(level=1),) + +Here is an example using LZMA from NumCodecs_ with a custom filter pipeline including LZMA's +built-in delta filter:: + + >>> import lzma + >>> from numcodecs.zarr3 import LZMA + >>> + >>> lzma_filters = [dict(id=lzma.FILTER_DELTA, dist=4), dict(id=lzma.FILTER_LZMA2, preset=1)] + >>> compressors = LZMA(filters=lzma_filters) + >>> data = np.arange(100000000, dtype='int32').reshape(10000, 10000) + >>> z = zarr.create_array(store='data/example-7.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), compressors=compressors) + >>> z.compressors + (LZMA(codec_name='numcodecs.lzma', codec_config={'filters': [{'id': 3, 'dist': 4}, {'id': 33, 'preset': 1}]}),) + +The default compressor can be changed by setting the value of the using Zarr's +:ref:`user-guide-config`, e.g.:: + + >>> with zarr.config.set({'array.v2_default_compressor.default': {'id': 'blosc'}}): + ... z = zarr.create_array(store={}, shape=(100000000,), chunks=(1000000,), dtype='int32', zarr_format=2) + >>> z.filters + () + >>> z.compressors + (Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0),) + +To disable compression, set ``compressors=None`` when creating an array, e.g.:: + + >>> z = zarr.create_array(store='data/example-8.zarr', shape=(100000000,), chunks=(1000000,), dtype='int32', compressors=None) + >>> z.compressors + () + +.. _user-guide-filters: + +Filters +------- + +In some cases, compression can be improved by transforming the data in some +way. For example, if nearby values tend to be correlated, then shuffling the +bytes within each numerical value or storing the difference between adjacent +values may increase compression ratio. Some compressors provide built-in filters +that apply transformations to the data prior to compression. For example, the +Blosc compressor has built-in implementations of byte- and bit-shuffle filters, +and the LZMA compressor has a built-in implementation of a delta +filter. However, to provide additional flexibility for implementing and using +filters in combination with different compressors, Zarr also provides a +mechanism for configuring filters outside of the primary compressor. + +Here is an example using a delta filter with the Blosc compressor:: + + >>> from numcodecs.zarr3 import Delta + >>> + >>> filters = [Delta(dtype='int32')] + >>> compressors = zarr.codecs.BloscCodec(cname='zstd', clevel=1, shuffle=zarr.codecs.BloscShuffle.shuffle) + >>> data = np.arange(100000000, dtype='int32').reshape(10000, 10000) + >>> z = zarr.create_array(store='data/example-9.zarr', shape=data.shape, dtype=data.dtype, chunks=(1000, 1000), filters=filters, compressors=compressors) + >>> z.info + Type : Array + Zarr format : 3 + Data type : Int32(endianness='little') + Fill value : 0 + Shape : (10000, 10000) + Chunk shape : (1000, 1000) + Order : C + Read-only : False + Store type : LocalStore + Filters : (Delta(codec_name='numcodecs.delta', codec_config={'dtype': 'int32'}),) + Serializer : BytesCodec(endian=) + Compressors : (BloscCodec(typesize=4, cname=, clevel=1, shuffle=, blocksize=0),) + No. bytes : 400000000 (381.5M) + +For more information about available filter codecs, see the `Numcodecs +`_ documentation. + +.. _user-guide-indexing: + +Advanced indexing +----------------- + +Zarr arrays support several methods for advanced or "fancy" +indexing, which enable a subset of data items to be extracted or updated in an +array without loading the entire array into memory. + +Note that although this functionality is similar to some of the advanced +indexing capabilities available on NumPy arrays and on h5py datasets, **the Zarr +API for advanced indexing is different from both NumPy and h5py**, so please +read this section carefully. For a complete description of the indexing API, +see the documentation for the :class:`zarr.Array` class. + +Indexing with coordinate arrays +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Items from a Zarr array can be extracted by providing an integer array of +coordinates. E.g.:: + + >>> data = np.arange(10) ** 2 + >>> z = zarr.create_array(store='data/example-10.zarr', shape=data.shape, dtype=data.dtype) + >>> z[:] = data + >>> z[:] + array([ 0, 1, 4, 9, 16, 25, 36, 49, 64, 81]) + >>> z.get_coordinate_selection([2, 5]) + array([ 4, 25]) + +Coordinate arrays can also be used to update data, e.g.:: + + >>> z.set_coordinate_selection([2, 5], [-1, -2]) + >>> z[:] + array([ 0, 1, -1, 9, 16, -2, 36, 49, 64, 81]) + +For multidimensional arrays, coordinates must be provided for each dimension, +e.g.:: + + >>> data = np.arange(15).reshape(3, 5) + >>> z = zarr.create_array(store='data/example-11.zarr', shape=data.shape, dtype=data.dtype) + >>> z[:] = data + >>> z[:] + array([[ 0, 1, 2, 3, 4], + [ 5, 6, 7, 8, 9], + [10, 11, 12, 13, 14]]) + >>> z.get_coordinate_selection(([0, 2], [1, 3])) + array([ 1, 13]) + >>> z.set_coordinate_selection(([0, 2], [1, 3]), [-1, -2]) + >>> z[:] + array([[ 0, -1, 2, 3, 4], + [ 5, 6, 7, 8, 9], + [10, 11, 12, -2, 14]]) + +For convenience, coordinate indexing is also available via the ``vindex`` +property, as well as the square bracket operator, e.g.:: + + >>> z.vindex[[0, 2], [1, 3]] + array([-1, -2]) + >>> z.vindex[[0, 2], [1, 3]] = [-3, -4] + >>> z[:] + array([[ 0, -3, 2, 3, 4], + [ 5, 6, 7, 8, 9], + [10, 11, 12, -4, 14]]) + >>> z[[0, 2], [1, 3]] + array([-3, -4]) + +When the indexing arrays have different shapes, they are broadcast together. +That is, the following two calls are equivalent:: + + >>> z[1, [1, 3]] + array([6, 8]) + >>> z[[1, 1], [1, 3]] + array([6, 8]) + +Indexing with a mask array +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Items can also be extracted by providing a Boolean mask. E.g.:: + + >>> data = np.arange(10) ** 2 + >>> z = zarr.create_array(store='data/example-12.zarr', shape=data.shape, dtype=data.dtype) + >>> z[:] = data + >>> z[:] + array([ 0, 1, 4, 9, 16, 25, 36, 49, 64, 81]) + >>> sel = np.zeros_like(z, dtype=bool) + >>> sel[2] = True + >>> sel[5] = True + >>> z.get_mask_selection(sel) + array([ 4, 25]) + >>> z.set_mask_selection(sel, [-1, -2]) + >>> z[:] + array([ 0, 1, -1, 9, 16, -2, 36, 49, 64, 81]) + +Here's a multidimensional example:: + + >>> data = np.arange(15).reshape(3, 5) + >>> z = zarr.create_array(store='data/example-13.zarr', shape=data.shape, dtype=data.dtype) + >>> z[:] = data + >>> z[:] + array([[ 0, 1, 2, 3, 4], + [ 5, 6, 7, 8, 9], + [10, 11, 12, 13, 14]]) + >>> sel = np.zeros_like(z, dtype=bool) + >>> sel[0, 1] = True + >>> sel[2, 3] = True + >>> z.get_mask_selection(sel) + array([ 1, 13]) + >>> z.set_mask_selection(sel, [-1, -2]) + >>> z[:] + array([[ 0, -1, 2, 3, 4], + [ 5, 6, 7, 8, 9], + [10, 11, 12, -2, 14]]) + +For convenience, mask indexing is also available via the ``vindex`` property, +e.g.:: + + >>> z.vindex[sel] + array([-1, -2]) + >>> z.vindex[sel] = [-3, -4] + >>> z[:] + array([[ 0, -3, 2, 3, 4], + [ 5, 6, 7, 8, 9], + [10, 11, 12, -4, 14]]) + +Mask indexing is conceptually the same as coordinate indexing, and is +implemented internally via the same machinery. Both styles of indexing allow +selecting arbitrary items from an array, also known as point selection. + +Orthogonal indexing +~~~~~~~~~~~~~~~~~~~ + +Zarr arrays also support methods for orthogonal indexing, which allows +selections to be made along each dimension of an array independently. For +example, this allows selecting a subset of rows and/or columns from a +2-dimensional array. E.g.:: + + >>> data = np.arange(15).reshape(3, 5) + >>> z = zarr.create_array(store='data/example-14.zarr', shape=data.shape, dtype=data.dtype) + >>> z[:] = data + >>> z[:] + array([[ 0, 1, 2, 3, 4], + [ 5, 6, 7, 8, 9], + [10, 11, 12, 13, 14]]) + >>> z.get_orthogonal_selection(([0, 2], slice(None))) # select first and third rows + array([[ 0, 1, 2, 3, 4], + [10, 11, 12, 13, 14]]) + >>> z.get_orthogonal_selection((slice(None), [1, 3])) # select second and fourth columns + array([[ 1, 3], + [ 6, 8], + [11, 13]]) + >>> z.get_orthogonal_selection(([0, 2], [1, 3])) # select rows [0, 2] and columns [1, 4] + array([[ 1, 3], + [11, 13]]) + +Data can also be modified, e.g.:: + + >>> z.set_orthogonal_selection(([0, 2], [1, 3]), [[-1, -2], [-3, -4]]) + +For convenience, the orthogonal indexing functionality is also available via the +``oindex`` property, e.g.:: + + >>> data = np.arange(15).reshape(3, 5) + >>> z = zarr.create_array(store='data/example-15.zarr', shape=data.shape, dtype=data.dtype) + >>> z[:] = data + >>> z.oindex[[0, 2], :] # select first and third rows + array([[ 0, 1, 2, 3, 4], + [10, 11, 12, 13, 14]]) + >>> z.oindex[:, [1, 3]] # select second and fourth columns + array([[ 1, 3], + [ 6, 8], + [11, 13]]) + >>> z.oindex[[0, 2], [1, 3]] # select rows [0, 2] and columns [1, 4] + array([[ 1, 3], + [11, 13]]) + >>> z.oindex[[0, 2], [1, 3]] = [[-1, -2], [-3, -4]] + >>> z[:] + array([[ 0, -1, 2, -2, 4], + [ 5, 6, 7, 8, 9], + [10, -3, 12, -4, 14]]) + +Any combination of integer, slice, 1D integer array and/or 1D Boolean array can +be used for orthogonal indexing. + +If the index contains at most one iterable, and otherwise contains only slices and integers, +orthogonal indexing is also available directly on the array:: + + >>> data = np.arange(15).reshape(3, 5) + >>> z = zarr.create_array(store='data/example-16.zarr', shape=data.shape, dtype=data.dtype) + >>> z[:] = data + >>> np.all(z.oindex[[0, 2], :] == z[[0, 2], :]) + np.True_ + +Block Indexing +~~~~~~~~~~~~~~ + +Zarr also support block indexing, which allows selections of whole chunks based on their +logical indices along each dimension of an array. For example, this allows selecting +a subset of chunk aligned rows and/or columns from a 2-dimensional array. E.g.:: + + >>> data = np.arange(100).reshape(10, 10) + >>> z = zarr.create_array(store='data/example-17.zarr', shape=data.shape, dtype=data.dtype, chunks=(3, 3)) + >>> z[:] = data + +Retrieve items by specifying their block coordinates:: + + >>> z.get_block_selection(1) + array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], + [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], + [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]]) + +Equivalent slicing:: + + >>> z[3:6] + array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], + [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], + [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]]) + +For convenience, the block selection functionality is also available via the +`blocks` property, e.g.:: + + >>> z.blocks[1] + array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], + [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], + [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]]) + +Block index arrays may be multidimensional to index multidimensional arrays. +For example:: + + >>> z.blocks[0, 1:3] + array([[ 3, 4, 5, 6, 7, 8], + [13, 14, 15, 16, 17, 18], + [23, 24, 25, 26, 27, 28]]) + +Data can also be modified. Let's start by a simple 2D array:: + + >>> z = zarr.create_array(store='data/example-18.zarr', shape=(6, 6), dtype=int, chunks=(2, 2)) + +Set data for a selection of items:: + + >>> z.set_block_selection((1, 0), 1) + >>> z[...] + array([[0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0]]) + +For convenience, this functionality is also available via the ``blocks`` property. +E.g.:: + + >>> z.blocks[:, 2] = 7 + >>> z[...] + array([[0, 0, 0, 0, 7, 7], + [0, 0, 0, 0, 7, 7], + [1, 1, 0, 0, 7, 7], + [1, 1, 0, 0, 7, 7], + [0, 0, 0, 0, 7, 7], + [0, 0, 0, 0, 7, 7]]) + +Any combination of integer and slice can be used for block indexing:: + + >>> z.blocks[2, 1:3] + array([[0, 0, 7, 7], + [0, 0, 7, 7]]) + >>> + >>> root = zarr.create_group('data/example-19.zarr') + >>> foo = root.create_array(name='foo', shape=(1000, 100), chunks=(10, 10), dtype='float32') + >>> bar = root.create_array(name='foo/bar', shape=(100,), dtype='int32') + >>> foo[:, :] = np.random.random((1000, 100)) + >>> bar[:] = np.arange(100) + >>> root.tree() + / + └── foo (1000, 100) float32 + + +.. _user-guide-sharding: + +Sharding +-------- + +Using small chunk shapes in very large arrays can lead to a very large number of chunks. +This can become a performance issue for file systems and object storage. +With Zarr format 3, a new sharding feature has been added to address this issue. + +With sharding, multiple chunks can be stored in a single storage object (e.g. a file). +Within a shard, chunks are compressed and serialized separately. +This allows individual chunks to be read independently. +However, when writing data, a full shard must be written in one go for optimal +performance and to avoid concurrency issues. +That means that shards are the units of writing and chunks are the units of reading. +Users need to configure the chunk and shard shapes accordingly. + +Sharded arrays can be created by providing the ``shards`` parameter to :func:`zarr.create_array`. + + >>> a = zarr.create_array('data/example-20.zarr', shape=(10000, 10000), shards=(1000, 1000), chunks=(100, 100), dtype='uint8') + >>> a[:] = (np.arange(10000 * 10000) % 256).astype('uint8').reshape(10000, 10000) + >>> a.info_complete() + Type : Array + Zarr format : 3 + Data type : UInt8() + Fill value : 0 + Shape : (10000, 10000) + Shard shape : (1000, 1000) + Chunk shape : (100, 100) + Order : C + Read-only : False + Store type : LocalStore + Filters : () + Serializer : BytesCodec(endian=None) + Compressors : (ZstdCodec(level=0, checksum=False),) + No. bytes : 100000000 (95.4M) + No. bytes stored : 3981473 + Storage ratio : 25.1 + Shards Initialized : 100 + +In this example a shard shape of (1000, 1000) and a chunk shape of (100, 100) is used. +This means that 10*10 chunks are stored in each shard, and there are 10*10 shards in total. +Without the ``shards`` argument, there would be 10,000 chunks stored as individual files. + +Missing features in 3.0 +----------------------- + + +The following features have not been ported to 3.0 yet. + +.. _user-guide-objects: + +Object arrays +~~~~~~~~~~~~~ + +See the Zarr-Python 2 documentation on `Object arrays `_ for more details. + +.. _user-guide-strings: + +Fixed-length string arrays +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See the Zarr-Python 2 documentation on `Fixed-length string arrays `_ for more details. + +.. _user-guide-datetime: + +Datetime and Timedelta arrays +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See the Zarr-Python 2 documentation on `Datetime and Timedelta `_ for more details. + +.. _user-guide-copy: + +Copying and migrating data +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See the Zarr-Python 2 documentation on `Copying and migrating data `_ for more details. diff --git a/docs/user-guide/attributes.rst b/docs/user-guide/attributes.rst new file mode 100644 index 0000000000..ed48623e29 --- /dev/null +++ b/docs/user-guide/attributes.rst @@ -0,0 +1,30 @@ +.. _user-guide-attrs: + +Working with attributes +======================= + +Zarr arrays and groups support custom key/value attributes, which can be useful for +storing application-specific metadata. For example:: + + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> root = zarr.create_group(store=store) + >>> root.attrs['foo'] = 'bar' + >>> z = root.create_array(name='zzz', shape=(10000, 10000), dtype='int32') + >>> z.attrs['baz'] = 42 + >>> z.attrs['qux'] = [1, 4, 7, 12] + >>> sorted(root.attrs) + ['foo'] + >>> 'foo' in root.attrs + True + >>> root.attrs['foo'] + 'bar' + >>> sorted(z.attrs) + ['baz', 'qux'] + >>> z.attrs['baz'] + 42 + >>> z.attrs['qux'] + [1, 4, 7, 12] + +Internally Zarr uses JSON to store array attributes, so attribute values must be +JSON serializable. diff --git a/docs/user-guide/config.rst b/docs/user-guide/config.rst new file mode 100644 index 0000000000..5a9d26f2b9 --- /dev/null +++ b/docs/user-guide/config.rst @@ -0,0 +1,82 @@ +.. _user-guide-config: + +Runtime configuration +===================== + +``zarr.config`` is responsible for managing the configuration of zarr and +is based on the `donfig `_ Python library. + +Configuration values can be set using code like the following:: + + >>> import zarr + >>> + >>> zarr.config.set({'array.order': 'F'}) + + >>> + >>> # revert this change so it doesn't impact the rest of the docs + >>> zarr.config.set({'array.order': 'C'}) + + +Alternatively, configuration values can be set using environment variables, e.g. +``ZARR_ARRAY__ORDER=F``. + +The configuration can also be read from a YAML file in standard locations. +For more information, see the +`donfig documentation `_. + +Configuration options include the following: + +- Default Zarr format ``default_zarr_version`` +- Default array order in memory ``array.order`` +- Default filters, serializers and compressors, e.g. ``array.v3_default_filters``, ``array.v3_default_serializer``, ``array.v3_default_compressors``, ``array.v2_default_filters`` and ``array.v2_default_compressor`` +- Whether empty chunks are written to storage ``array.write_empty_chunks`` +- Async and threading options, e.g. ``async.concurrency`` and ``threading.max_workers`` +- Selections of implementations of codecs, codec pipelines and buffers +- Enabling GPU support with ``zarr.config.enable_gpu()``. See :ref:`user-guide-gpu` for more. + +For selecting custom implementations of codecs, pipelines, buffers and ndbuffers, +first register the implementations in the registry and then select them in the config. +For example, an implementation of the bytes codec in a class ``'custompackage.NewBytesCodec'``, +requires the value of ``codecs.bytes.name`` to be ``'custompackage.NewBytesCodec'``. + +This is the current default configuration:: + + >>> zarr.config.pprint() + {'array': {'order': 'C', + 'v2_default_compressor': {'default': {'checksum': False, + 'id': 'zstd', + 'level': 0}, + 'variable-length-string': {'checksum': False, + 'id': 'zstd', + 'level': 0}}, + 'v2_default_filters': {'default': None, + 'variable-length-string': [{'id': 'vlen-utf8'}]}, + 'v3_default_compressors': {'default': [{'configuration': {'checksum': False, + 'level': 0}, + 'name': 'zstd'}], + 'variable-length-string': [{'configuration': {'checksum': False, + 'level': 0}, + 'name': 'zstd'}]}, + 'v3_default_filters': {'default': [], 'variable-length-string': []}, + 'v3_default_serializer': {'default': {'configuration': {'endian': 'little'}, + 'name': 'bytes'}, + 'variable-length-string': {'name': 'vlen-utf8'}}, + 'write_empty_chunks': False}, + 'async': {'concurrency': 10, 'timeout': None}, + 'buffer': 'zarr.buffer.cpu.Buffer', + 'codec_pipeline': {'batch_size': 1, + 'path': 'zarr.core.codec_pipeline.BatchedCodecPipeline'}, + 'codecs': {'blosc': 'zarr.codecs.blosc.BloscCodec', + 'bytes': 'zarr.codecs.bytes.BytesCodec', + 'crc32c': 'zarr.codecs.crc32c_.Crc32cCodec', + 'endian': 'zarr.codecs.bytes.BytesCodec', + 'gzip': 'zarr.codecs.gzip.GzipCodec', + 'sharding_indexed': 'zarr.codecs.sharding.ShardingCodec', + 'transpose': 'zarr.codecs.transpose.TransposeCodec', + 'vlen-bytes': 'zarr.codecs.vlen_utf8.VLenBytesCodec', + 'vlen-utf8': 'zarr.codecs.vlen_utf8.VLenUTF8Codec', + 'zstd': 'zarr.codecs.zstd.ZstdCodec'}, + 'default_zarr_format': 3, + 'json_indent': 2, + 'ndbuffer': 'zarr.buffer.cpu.NDBuffer', + 'threading': {'max_workers': None}} diff --git a/docs/user-guide/consolidated_metadata.rst b/docs/user-guide/consolidated_metadata.rst new file mode 100644 index 0000000000..4cd72dbc74 --- /dev/null +++ b/docs/user-guide/consolidated_metadata.rst @@ -0,0 +1,136 @@ +.. _user-guide-consolidated-metadata: + +Consolidated metadata +===================== + +.. warning:: + The Consolidated Metadata feature in Zarr-Python is considered experimental for v3 + stores. `zarr-specs#309 `_ + has proposed a formal extension to the v3 specification to support consolidated metadata. + +Zarr-Python implements the `Consolidated Metadata`_ for v2 and v3 stores. +Consolidated metadata can reduce the time needed to load the metadata for an +entire hierarchy, especially when the metadata is being served over a network. +Consolidated metadata essentially stores all the metadata for a hierarchy in the +metadata of the root Group. + +Usage +----- + +If consolidated metadata is present in a Zarr Group's metadata then it is used +by default. The initial read to open the group will need to communicate with +the store (reading from a file for a :class:`zarr.storage.LocalStore`, making a +network request for a :class:`zarr.storage.FsspecStore`). After that, any subsequent +metadata reads get child Group or Array nodes will *not* require reads from the store. + +In Python, the consolidated metadata is available on the ``.consolidated_metadata`` +attribute of the ``GroupMetadata`` object. + + >>> import zarr + >>> + >>> store = zarr.storage.MemoryStore() + >>> group = zarr.create_group(store=store) + >>> group.create_array(shape=(1,), name='a', dtype='float64') + + >>> group.create_array(shape=(2, 2), name='b', dtype='float64') + + >>> group.create_array(shape=(3, 3, 3), name='c', dtype='float64') + + >>> zarr.consolidate_metadata(store) + + +If we open that group, the Group's metadata has a :class:`zarr.core.group.ConsolidatedMetadata` +that can be used.: + + >>> consolidated = zarr.open_group(store=store) + >>> consolidated_metadata = consolidated.metadata.consolidated_metadata.metadata + >>> from pprint import pprint + >>> pprint(dict(sorted(consolidated_metadata.items()))) + {'a': ArrayV3Metadata(shape=(1,), + data_type=Float64(endianness='little'), + chunk_grid=RegularChunkGrid(chunk_shape=(1,)), + chunk_key_encoding=DefaultChunkKeyEncoding(name='default', + separator='/'), + fill_value=np.float64(0.0), + codecs=(BytesCodec(endian=), + ZstdCodec(level=0, checksum=False)), + attributes={}, + dimension_names=None, + zarr_format=3, + node_type='array', + storage_transformers=()), + 'b': ArrayV3Metadata(shape=(2, 2), + data_type=Float64(endianness='little'), + chunk_grid=RegularChunkGrid(chunk_shape=(2, 2)), + chunk_key_encoding=DefaultChunkKeyEncoding(name='default', + separator='/'), + fill_value=np.float64(0.0), + codecs=(BytesCodec(endian=), + ZstdCodec(level=0, checksum=False)), + attributes={}, + dimension_names=None, + zarr_format=3, + node_type='array', + storage_transformers=()), + 'c': ArrayV3Metadata(shape=(3, 3, 3), + data_type=Float64(endianness='little'), + chunk_grid=RegularChunkGrid(chunk_shape=(3, 3, 3)), + chunk_key_encoding=DefaultChunkKeyEncoding(name='default', + separator='/'), + fill_value=np.float64(0.0), + codecs=(BytesCodec(endian=), + ZstdCodec(level=0, checksum=False)), + attributes={}, + dimension_names=None, + zarr_format=3, + node_type='array', + storage_transformers=())} + +Operations on the group to get children automatically use the consolidated metadata.: + + >>> consolidated['a'] # no read / HTTP request to the Store is required + + +With nested groups, the consolidated metadata is available on the children, recursively.: + + >>> child = group.create_group('child', attributes={'kind': 'child'}) + >>> grandchild = child.create_group('child', attributes={'kind': 'grandchild'}) + >>> consolidated = zarr.consolidate_metadata(store) + >>> + >>> consolidated['child'].metadata.consolidated_metadata + ConsolidatedMetadata(metadata={'child': GroupMetadata(attributes={'kind': 'grandchild'}, zarr_format=3, consolidated_metadata=ConsolidatedMetadata(metadata={}, kind='inline', must_understand=False), node_type='group')}, kind='inline', must_understand=False) + +Synchronization and Concurrency +------------------------------- + +Consolidated metadata is intended for read-heavy use cases on slowly changing +hierarchies. For hierarchies where new nodes are constantly being added, +removed, or modified, consolidated metadata may not be desirable. + +1. It will add some overhead to each update operation, since the metadata + would need to be re-consolidated to keep it in sync with the store. +2. Readers using consolidated metadata will regularly see a "past" version + of the metadata, at the time they read the root node with its consolidated + metadata. + +.. _Consolidated Metadata: https://github.com/zarr-developers/zarr-specs/pull/309 + +Stores Without Support for Consolidated Metadata +------------------------------------------------ + +Some stores may want to opt out of the consolidated metadata mechanism. This +may be for several reasons like: + +* They want to maintain read-write consistency, which is challenging with + consolidated metadata. +* They have their own consolidated metadata mechanism. +* They offer good enough performance without need for consolidation. + +This type of store can declare it doesn't want consolidation by implementing +`Store.supports_consolidated_metadata` and returning `False`. For stores that don't support +consolidation, Zarr will: + +* Raise an error on `consolidate_metadata` calls, maintaining the store in + its unconsolidated state. +* Raise an error in `AsyncGroup.open(..., use_consolidated=True)` +* Not use consolidated metadata in `AsyncGroup.open(..., use_consolidated=None)` diff --git a/docs/user-guide/data_types.rst b/docs/user-guide/data_types.rst new file mode 100644 index 0000000000..87c8efc1f5 --- /dev/null +++ b/docs/user-guide/data_types.rst @@ -0,0 +1,172 @@ +Data types +========== + +Zarr's data type model +---------------------- + +Every Zarr array has a "data type", which defines the meaning and physical layout of the +array's elements. As Zarr Python is tightly integrated with `NumPy `_, +it's easy to create arrays with NumPy data types: + +.. code-block:: python + + >>> import zarr + >>> import numpy as np + >>> z = zarr.create_array(store={}, shape=(10,), dtype=np.dtype('uint8')) + >>> z + + +Unlike NumPy arrays, Zarr arrays are designed to accessed by Zarr +implementations in different programming languages. This means Zarr data types must be interpreted +correctly when clients read an array. Each Zarr data type defines procedures for +encoding and decoding both the data type itself, and scalars from that data type to and from Zarr array metadata. And these serialization procedures +depend on the Zarr format. + +Data types in Zarr version 2 +----------------------------- + +Version 2 of the Zarr format defined its data types relative to +`NumPy's data types `_, +and added a few non-NumPy data types as well. Thus the JSON identifier for a NumPy-compatible data +type is just the NumPy ``str`` attribute of that data type: + +.. code-block:: python + + >>> import zarr + >>> import numpy as np + >>> import json + >>> + >>> store = {} + >>> np_dtype = np.dtype('int64') + >>> z = zarr.create_array(store=store, shape=(1,), dtype=np_dtype, zarr_format=2) + >>> dtype_meta = json.loads(store['.zarray'].to_bytes())["dtype"] + >>> dtype_meta + '>> assert dtype_meta == np_dtype.str + +.. note:: + The ``<`` character in the data type metadata encodes the + `endianness `_, + or "byte order", of the data type. Following NumPy's example, + in Zarr version 2 each data type has an endianness where applicable. + However, Zarr version 3 data types do not store endianness information. + +In addition to defining a representation of the data type itself (which in the example above was +just a simple string ``"M[10s]"`` in + Zarr V2. This is more compact, but can be harder to parse. + +For more about data types in Zarr V3, see the +`V3 specification `_. + +Data types in Zarr Python +------------------------- + +The two Zarr formats that Zarr Python supports specify data types in two different ways: +data types in Zarr version 2 are encoded as NumPy-compatible strings, while data types in Zarr version +3 are encoded as either strings or ``JSON`` objects, +and the Zarr V3 data types don't have any associated endianness information, unlike Zarr V2 data types. + +To abstract over these syntactical and semantic differences, Zarr Python uses a class called +`ZDType <../api/zarr/dtype/index.html#zarr.dtype.ZDType>`_ provide Zarr V2 and Zarr V3 compatibility +routines for ""native" data types. In this context, a "native" data type is a Python class, +typically defined in another library, that models an array's data type. For example, ``np.uint8`` is a native +data type defined in NumPy, which Zarr Python wraps with a ``ZDType`` instance called +`UInt8 <../api/zarr/dtype/index.html#zarr.dtype.ZDType>`_. + +Each data type supported by Zarr Python is modeled by ``ZDType`` subclass, which provides an +API for the following operations: + +- Wrapping / unwrapping a native data type +- Encoding / decoding a data type to / from Zarr V2 and Zarr V3 array metadata. +- Encoding / decoding a scalar value to / from Zarr V2 and Zarr V3 array metadata. + + +Example Usage +~~~~~~~~~~~~~ + +Create a ``ZDType`` from a native data type: + +.. code-block:: python + + >>> from zarr.core.dtype import Int8 + >>> import numpy as np + >>> int8 = Int8.from_native_dtype(np.dtype('int8')) + +Convert back to native data type: + +.. code-block:: python + + >>> native_dtype = int8.to_native_dtype() + >>> assert native_dtype == np.dtype('int8') + +Get the default scalar value for the data type: + +.. code-block:: python + + >>> default_value = int8.default_scalar() + >>> assert default_value == np.int8(0) + + +Serialize to JSON for Zarr V2 and V3 + +.. code-block:: python + + >>> json_v2 = int8.to_json(zarr_format=2) + >>> json_v2 + {'name': '|i1', 'object_codec_id': None} + >>> json_v3 = int8.to_json(zarr_format=3) + >>> json_v3 + 'int8' + +Serialize a scalar value to JSON: + +.. code-block:: python + + >>> json_value = int8.to_json_scalar(42, zarr_format=3) + >>> json_value + 42 + +Deserialize a scalar value from JSON: + +.. code-block:: python + + >>> scalar_value = int8.from_json_scalar(42, zarr_format=3) + >>> assert scalar_value == np.int8(42) diff --git a/docs/user-guide/extending.rst b/docs/user-guide/extending.rst new file mode 100644 index 0000000000..4487e07ddf --- /dev/null +++ b/docs/user-guide/extending.rst @@ -0,0 +1,94 @@ + +Extending Zarr +============== + +Zarr-Python 3 was designed to be extensible. This means that you can extend +the library by writing custom classes and plugins. Currently, Zarr can be extended +in the following ways: + +Custom codecs +------------- + +.. note:: + This section explains how custom codecs can be created for Zarr format 3 arrays. For Zarr + format 2, codecs should subclass the + `numcodecs.abc.Codec `_ + base class and register through + `numcodecs.registry.register_codec `_. + +There are three types of codecs in Zarr: +- array-to-array +- array-to-bytes +- bytes-to-bytes + +Array-to-array codecs are used to transform the array data before serializing +to bytes. Examples include delta encoding or scaling codecs. Array-to-bytes codecs are used +for serializing the array data to bytes. In Zarr, the main codec to use for numeric arrays +is the :class:`zarr.codecs.BytesCodec`. Bytes-to-bytes codecs transform the serialized bytestreams +of the array data. Examples include compression codecs, such as +:class:`zarr.codecs.GzipCodec`, :class:`zarr.codecs.BloscCodec` or +:class:`zarr.codecs.ZstdCodec`, and codecs that add a checksum to the bytestream, such as +:class:`zarr.codecs.Crc32cCodec`. + +Custom codecs for Zarr are implemented by subclassing the relevant base class, see +:class:`zarr.abc.codec.ArrayArrayCodec`, :class:`zarr.abc.codec.ArrayBytesCodec` and +:class:`zarr.abc.codec.BytesBytesCodec`. Most custom codecs should implemented the +``_encode_single`` and ``_decode_single`` methods. These methods operate on single chunks +of the array data. Alternatively, custom codecs can implement the ``encode`` and ``decode`` +methods, which operate on batches of chunks, in case the codec is intended to implement +its own batch processing. + +Custom codecs should also implement the following methods: + +- ``compute_encoded_size``, which returns the byte size of the encoded data given the byte + size of the original data. It should raise ``NotImplementedError`` for codecs with + variable-sized outputs, such as compression codecs. +- ``validate`` (optional), which can be used to check that the codec metadata is compatible with the + array metadata. It should raise errors if not. +- ``resolve_metadata`` (optional), which is important for codecs that change the shape, + dtype or fill value of a chunk. +- ``evolve_from_array_spec`` (optional), which can be useful for automatically filling in + codec configuration metadata from the array metadata. + +To use custom codecs in Zarr, they need to be registered using the +`entrypoint mechanism `_. +Commonly, entrypoints are declared in the ``pyproject.toml`` of your package under the +``[project.entry-points."zarr.codecs"]`` section. Zarr will automatically discover and +load all codecs registered with the entrypoint mechanism from imported modules. + +.. code-block:: toml + + [project.entry-points."zarr.codecs"] + "custompackage.fancy_codec" = "custompackage:FancyCodec" + +New codecs need to have their own unique identifier. To avoid naming collisions, it is +strongly recommended to prefix the codec identifier with a unique name. For example, +the codecs from ``numcodecs`` are prefixed with ``numcodecs.``, e.g. ``numcodecs.delta``. + +.. note:: + Note that the extension mechanism for the Zarr format 3 is still under development. + Requirements for custom codecs including the choice of codec identifiers might + change in the future. + +It is also possible to register codecs as replacements for existing codecs. This might be +useful for providing specialized implementations, such as GPU-based codecs. In case of +multiple codecs, the :mod:`zarr.core.config` mechanism can be used to select the preferred +implementation. + +Custom stores +------------- + +Coming soon. + +Custom array buffers +-------------------- + +Zarr-python provides control over where and how arrays stored in memory through +:mod:`zarr.buffer`. Currently both CPU (the default) and GPU implementations are +provided (see :ref:`user-guide-gpu` for more). You can implement your own buffer +classes by implementing the interface defined in :mod:`zarr.abc.buffer`. + +Other extensions +---------------- + +In the future, Zarr will support writing custom custom data types and chunk grids. diff --git a/docs/user-guide/gpu.rst b/docs/user-guide/gpu.rst new file mode 100644 index 0000000000..4d3492f8bd --- /dev/null +++ b/docs/user-guide/gpu.rst @@ -0,0 +1,37 @@ +.. _user-guide-gpu: + +Using GPUs with Zarr +==================== + +Zarr can use GPUs to accelerate your workload by running +:meth:`zarr.config.enable_gpu`. + +.. note:: + + `zarr-python` currently supports reading the ndarray data into device (GPU) + memory as the final stage of the codec pipeline. Data will still be read into + or copied to host (CPU) memory for encoding and decoding. + + In the future, codecs will be available compressing and decompressing data on + the GPU, avoiding the need to move data between the host and device for + compression and decompression. + +Reading data into device memory +------------------------------- + +:meth:`zarr.config.enable_gpu` configures Zarr to use GPU memory for the data +buffers used internally by Zarr. + +.. code-block:: python + + >>> import zarr + >>> import cupy as cp # doctest: +SKIP + >>> zarr.config.enable_gpu() # doctest: +SKIP + >>> store = zarr.storage.MemoryStore() # doctest: +SKIP + >>> z = zarr.create_array( # doctest: +SKIP + ... store=store, shape=(100, 100), chunks=(10, 10), dtype="float32", + ... ) + >>> type(z[:10, :10]) # doctest: +SKIP + cupy.ndarray + +Note that the output type is a ``cupy.ndarray`` rather than a NumPy array. diff --git a/docs/user-guide/groups.rst b/docs/user-guide/groups.rst new file mode 100644 index 0000000000..4237a9df50 --- /dev/null +++ b/docs/user-guide/groups.rst @@ -0,0 +1,172 @@ +.. only:: doctest + + >>> import shutil + >>> shutil.rmtree('data', ignore_errors=True) + +.. _user-guide-groups: + +Working with groups +=================== + +Zarr supports hierarchical organization of arrays via groups. As with arrays, +groups can be stored in memory, on disk, or via other storage systems that +support a similar interface. + +To create a group, use the :func:`zarr.group` function:: + + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> root = zarr.create_group(store=store) + >>> root + + +Groups have a similar API to the Group class from `h5py +`_. For example, groups can contain other groups:: + + >>> foo = root.create_group('foo') + >>> bar = foo.create_group('bar') + +Groups can also contain arrays, e.g.:: + + >>> z1 = bar.create_array(name='baz', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') + >>> z1 + + +Members of a group can be accessed via the suffix notation, e.g.:: + + >>> root['foo'] + + +The '/' character can be used to access multiple levels of the hierarchy in one +call, e.g.:: + + >>> root['foo/bar'] + + >>> root['foo/bar/baz'] + + +The :func:`zarr.Group.tree` method can be used to print a tree +representation of the hierarchy, e.g.:: + + >>> root.tree() + / + └── foo + └── bar + └── baz (10000, 10000) int32 + + +The :func:`zarr.open_group` function provides a convenient way to create or +re-open a group stored in a directory on the file-system, with sub-groups stored in +sub-directories, e.g.:: + + >>> root = zarr.open_group('data/group.zarr', mode='w') + >>> root + + >>> + >>> z = root.create_array(name='foo/bar/baz', shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') + >>> z + + +.. TODO: uncomment after __enter__ and __exit__ are implemented +.. Groups can be used as context managers (in a ``with`` statement). +.. If the underlying store has a ``close`` method, it will be called on exit. + +For more information on groups see the :class:`zarr.Group` API docs. + +.. _user-guide-diagnostics: + +Batch Group Creation +-------------------- + +You can also create multiple groups concurrently with a single function call. :func:`zarr.create_hierarchy` takes +a :class:`zarr.storage.Store` instance and a dict of ``key : metadata`` pairs, parses that dict, and +writes metadata documents to storage: + + >>> from zarr import create_hierarchy + >>> from zarr.core.group import GroupMetadata + >>> from zarr.storage import LocalStore + >>> node_spec = {'a/b/c': GroupMetadata()} + >>> nodes_created = dict(create_hierarchy(store=LocalStore(root='data'), nodes=node_spec)) + >>> print(sorted(nodes_created.items(), key=lambda kv: len(kv[0]))) + [('', ), ('a', ), ('a/b', ), ('a/b/c', )] + +Note that we only specified a single group named ``a/b/c``, but 4 groups were created. These additional groups +were created to ensure that the desired node ``a/b/c`` is connected to the root group ``''`` by a sequence +of intermediate groups. :func:`zarr.create_hierarchy` normalizes the ``nodes`` keyword argument to +ensure that the resulting hierarchy is complete, i.e. all groups or arrays are connected to the root +of the hierarchy via intermediate groups. + +Because :func:`zarr.create_hierarchy` concurrently creates metadata documents, it's more efficient +than repeated calls to :func:`create_group` or :func:`create_array`, provided you can statically define +the metadata for the groups and arrays you want to create. + +Array and group diagnostics +--------------------------- + +Diagnostic information about arrays and groups is available via the ``info`` +property. E.g.:: + + >>> store = zarr.storage.MemoryStore() + >>> root = zarr.group(store=store) + >>> foo = root.create_group('foo') + >>> bar = foo.create_array(name='bar', shape=1000000, chunks=100000, dtype='int64') + >>> bar[:] = 42 + >>> baz = foo.create_array(name='baz', shape=(1000, 1000), chunks=(100, 100), dtype='float32') + >>> baz[:] = 4.2 + >>> root.info + Name : + Type : Group + Zarr format : 3 + Read-only : False + Store type : MemoryStore + >>> foo.info + Name : foo + Type : Group + Zarr format : 3 + Read-only : False + Store type : MemoryStore + >>> bar.info_complete() + Type : Array + Zarr format : 3 + Data type : Int64(endianness='little') + Fill value : 0 + Shape : (1000000,) + Chunk shape : (100000,) + Order : C + Read-only : False + Store type : MemoryStore + Filters : () + Serializer : BytesCodec(endian=) + Compressors : (ZstdCodec(level=0, checksum=False),) + No. bytes : 8000000 (7.6M) + No. bytes stored : 1614 + Storage ratio : 4956.6 + Chunks Initialized : 10 + >>> baz.info + Type : Array + Zarr format : 3 + Data type : Float32(endianness='little') + Fill value : 0.0 + Shape : (1000, 1000) + Chunk shape : (100, 100) + Order : C + Read-only : False + Store type : MemoryStore + Filters : () + Serializer : BytesCodec(endian=) + Compressors : (ZstdCodec(level=0, checksum=False),) + No. bytes : 4000000 (3.8M) + +Groups also have the :func:`zarr.Group.tree` method, e.g.:: + + >>> root.tree() + / + └── foo + ├── bar (1000000,) int64 + └── baz (1000, 1000) float32 + + +.. note:: + + :func:`zarr.Group.tree` requires the optional `rich `_ + dependency. It can be installed with the ``[tree]`` extra. diff --git a/docs/user-guide/index.rst b/docs/user-guide/index.rst new file mode 100644 index 0000000000..ea34ac2561 --- /dev/null +++ b/docs/user-guide/index.rst @@ -0,0 +1,31 @@ +.. _user-guide: + +User guide +========== + +.. toctree:: + :maxdepth: 1 + + installation + arrays + data_types + groups + attributes + storage + config + v3_migration + +Advanced Topics +--------------- + +.. toctree:: + :maxdepth: 1 + + performance + consolidated_metadata + extending + gpu + + +.. Coming soon + async diff --git a/docs/user-guide/installation.rst b/docs/user-guide/installation.rst new file mode 100644 index 0000000000..a79f0763cb --- /dev/null +++ b/docs/user-guide/installation.rst @@ -0,0 +1,54 @@ +Installation +============ + +Required dependencies +--------------------- + +Required dependencies include: + +- `Python `_ (3.11 or later) +- `packaging `_ (22.0 or later) +- `numpy `_ (1.25 or later) +- `numcodecs[crc32c] `_ (0.14 or later) +- `typing_extensions `_ (4.9 or later) +- `donfig `_ (0.8 or later) + +pip +--- + +Zarr is available on `PyPI `_. Install it using ``pip``: + +.. code-block:: console + + $ pip install zarr + +There are a number of optional dependency groups you can install for extra functionality. +These can be installed using ``pip install "zarr[]"``, e.g. ``pip install "zarr[gpu]"`` + +- ``gpu``: support for GPUs +- ``remote``: support for reading/writing to remote data stores + +Additional optional dependencies include ``rich``, ``universal_pathlib``. These must be installed separately. + +conda +----- + +Zarr is also published to `conda-forge `_. Install it using ``conda``: + +.. code-block:: console + + $ conda install -c conda-forge zarr + +Conda does not support optional dependencies, so you will have to manually install any packages +needed to enable extra functionality. + +Dependency support +------------------ +Zarr has endorsed `Scientific-Python SPEC 0 `_ and now follows the version support window as outlined below: + +- Python: 36 months after initial release +- Core package dependencies (e.g. NumPy): 24 months after initial release + +Development +----------- +To install the latest development version of Zarr, see the :ref:`contributing guide `. diff --git a/docs/user-guide/performance.rst b/docs/user-guide/performance.rst new file mode 100644 index 0000000000..7d24c87373 --- /dev/null +++ b/docs/user-guide/performance.rst @@ -0,0 +1,278 @@ +.. only:: doctest + + >>> import shutil + >>> shutil.rmtree('data', ignore_errors=True) + +.. _user-guide-performance: + +Optimizing performance +====================== + +.. _user-guide-chunks: + +Chunk optimizations +------------------- + +.. _user-guide-chunks-shape: + +Chunk size and shape +~~~~~~~~~~~~~~~~~~~~ + +In general, chunks of at least 1 megabyte (1M) uncompressed size seem to provide +better performance, at least when using the Blosc compression library. + +The optimal chunk shape will depend on how you want to access the data. E.g., +for a 2-dimensional array, if you only ever take slices along the first +dimension, then chunk across the second dimension. If you know you want to chunk +across an entire dimension you can use the full size of that dimension within the +``chunks`` argument, e.g.:: + + >>> import zarr + >>> z1 = zarr.create_array(store={}, shape=(10000, 10000), chunks=(100, 10000), dtype='int32') + >>> z1.chunks + (100, 10000) + +Alternatively, if you only ever take slices along the second dimension, then +chunk across the first dimension, e.g.:: + + >>> z2 = zarr.create_array(store={}, shape=(10000, 10000), chunks=(10000, 100), dtype='int32') + >>> z2.chunks + (10000, 100) + +If you require reasonable performance for both access patterns then you need to +find a compromise, e.g.:: + + >>> z3 = zarr.create_array(store={}, shape=(10000, 10000), chunks=(1000, 1000), dtype='int32') + >>> z3.chunks + (1000, 1000) + +If you are feeling lazy, you can let Zarr guess a chunk shape for your data by +providing ``chunks='auto'``, although please note that the algorithm for guessing +a chunk shape is based on simple heuristics and may be far from optimal. E.g.:: + + >>> z4 = zarr.create_array(store={}, shape=(10000, 10000), chunks='auto', dtype='int32') + >>> z4.chunks + (625, 625) + +If you know you are always going to be loading the entire array into memory, you +can turn off chunks by providing ``chunks`` equal to ``shape``, in which case there +will be one single chunk for the array:: + + >>> z5 = zarr.create_array(store={}, shape=(10000, 10000), chunks=(10000, 10000), dtype='int32') + >>> z5.chunks + (10000, 10000) + + +Sharding +~~~~~~~~ + +If you have large arrays but need small chunks to efficiently access the data, you can +use sharding. Sharding provides a mechanism to store multiple chunks in a single +storage object or file. This can be useful because traditional file systems and object +storage systems may have performance issues storing and accessing many files. +Additionally, small files can be inefficient to store if they are smaller than the +block size of the file system. + +Picking a good combination of chunk shape and shard shape is important for performance. +The chunk shape determines what unit of your data can be read independently, while the +shard shape determines what unit of your data can be written efficiently. + +For an example, consider you have a 100 GB array and need to read small chunks of 1 MB. +Without sharding, each chunk would be one file resulting in 100,000 files. That can +already cause performance issues on some file systems. +With sharding, you could use a shard size of 1 GB. This would result in 1000 chunks per +file and 100 files in total, which seems manageable for most storage systems. +You would still be able to read each 1 MB chunk independently, but you would need to +write your data in 1 GB increments. + +To use sharding, you need to specify the ``shards`` parameter when creating the array. + + >>> z6 = zarr.create_array(store={}, shape=(10000, 10000, 1000), shards=(1000, 1000, 1000), chunks=(100, 100, 100), dtype='uint8') + >>> z6.info + Type : Array + Zarr format : 3 + Data type : UInt8() + Fill value : 0 + Shape : (10000, 10000, 1000) + Shard shape : (1000, 1000, 1000) + Chunk shape : (100, 100, 100) + Order : C + Read-only : False + Store type : MemoryStore + Filters : () + Serializer : BytesCodec(endian=None) + Compressors : (ZstdCodec(level=0, checksum=False),) + No. bytes : 100000000000 (93.1G) + +.. _user-guide-chunks-order: + +Chunk memory layout +~~~~~~~~~~~~~~~~~~~ + +The order of bytes **within each chunk** of an array can be changed via the +``order`` config option, to use either C or Fortran layout. For +multi-dimensional arrays, these two layouts may provide different compression +ratios, depending on the correlation structure within the data. E.g.:: + + >>> import numpy as np + >>> + >>> a = np.arange(100000000, dtype='int32').reshape(10000, 10000).T + >>> c = zarr.create_array(store={}, shape=a.shape, chunks=(1000, 1000), dtype=a.dtype, config={'order': 'C'}) + >>> c[:] = a + >>> c.info_complete() + Type : Array + Zarr format : 3 + Data type : Int32(endianness='little') + Fill value : 0 + Shape : (10000, 10000) + Chunk shape : (1000, 1000) + Order : C + Read-only : False + Store type : MemoryStore + Filters : () + Serializer : BytesCodec(endian=) + Compressors : (ZstdCodec(level=0, checksum=False),) + No. bytes : 400000000 (381.5M) + No. bytes stored : 342588911 + Storage ratio : 1.2 + Chunks Initialized : 100 + >>> with zarr.config.set({'array.order': 'F'}): + ... f = zarr.create_array(store={}, shape=a.shape, chunks=(1000, 1000), dtype=a.dtype) + ... f[:] = a + >>> f.info_complete() + Type : Array + Zarr format : 3 + Data type : Int32(endianness='little') + Fill value : 0 + Shape : (10000, 10000) + Chunk shape : (1000, 1000) + Order : F + Read-only : False + Store type : MemoryStore + Filters : () + Serializer : BytesCodec(endian=) + Compressors : (ZstdCodec(level=0, checksum=False),) + No. bytes : 400000000 (381.5M) + No. bytes stored : 342588911 + Storage ratio : 1.2 + Chunks Initialized : 100 + +In the above example, Fortran order gives a better compression ratio. This is an +artificial example but illustrates the general point that changing the order of +bytes within chunks of an array may improve the compression ratio, depending on +the structure of the data, the compression algorithm used, and which compression +filters (e.g., byte-shuffle) have been applied. + +.. _user-guide-chunks-empty-chunks: + +Empty chunks +~~~~~~~~~~~~ + +It is possible to configure how Zarr handles the storage of chunks that are "empty" +(i.e., every element in the chunk is equal to the array's fill value). When creating +an array with ``write_empty_chunks=False``, Zarr will check whether a chunk is empty before compression and storage. If a chunk is empty, +then Zarr does not store it, and instead deletes the chunk from storage +if the chunk had been previously stored. + +This optimization prevents storing redundant objects and can speed up reads, but the cost is +added computation during array writes, since the contents of +each chunk must be compared to the fill value, and these advantages are contingent on the content of the array. +If you know that your data will form chunks that are almost always non-empty, then there is no advantage to the optimization described above. +In this case, creating an array with ``write_empty_chunks=True`` (the default) will instruct Zarr to write every chunk without checking for emptiness. + +The following example illustrates the effect of the ``write_empty_chunks`` flag on +the time required to write an array with different values.:: + + >>> import zarr + >>> import numpy as np + >>> import time + >>> + >>> def timed_write(write_empty_chunks): + ... """ + ... Measure the time required and number of objects created when writing + ... to a Zarr array with random ints or fill value. + ... """ + ... chunks = (8192,) + ... shape = (chunks[0] * 1024,) + ... data = np.random.randint(0, 255, shape) + ... dtype = 'uint8' + ... arr = zarr.create_array( + ... f'data/example-{write_empty_chunks}.zarr', + ... shape=shape, + ... chunks=chunks, + ... dtype=dtype, + ... fill_value=0, + ... config={'write_empty_chunks': write_empty_chunks} + ... ) + ... # initialize all chunks + ... arr[:] = 100 + ... result = [] + ... for value in (data, arr.fill_value): + ... start = time.time() + ... arr[:] = value + ... elapsed = time.time() - start + ... result.append((elapsed, arr.nchunks_initialized)) + ... return result + ... # log results + >>> for write_empty_chunks in (True, False): + ... full, empty = timed_write(write_empty_chunks) + ... print(f'\nwrite_empty_chunks={write_empty_chunks}:\n\tRandom Data: {full[0]:.4f}s, {full[1]} objects stored\n\t Empty Data: {empty[0]:.4f}s, {empty[1]} objects stored\n') + write_empty_chunks=True: + Random Data: ..., 1024 objects stored + Empty Data: ...s, 1024 objects stored + + write_empty_chunks=False: + Random Data: ...s, 1024 objects stored + Empty Data: ...s, 0 objects stored + + +In this example, writing random data is slightly slower with ``write_empty_chunks=True``, +but writing empty data is substantially faster and generates far fewer objects in storage. + +.. _user-guide-rechunking: + +Changing chunk shapes (rechunking) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Coming soon. + +.. _user-guide-sync: + +Parallel computing and synchronization +-------------------------------------- + +Coming soon. + +.. _user-guide-pickle: + +Pickle support +-------------- + +Zarr arrays and groups can be pickled, as long as the underlying store object can be +pickled. With the exception of the :class:`zarr.storage.MemoryStore`, any of the +storage classes provided in the :mod:`zarr.storage` module can be pickled. + +If an array or group is backed by a persistent store such as the a :class:`zarr.storage.LocalStore`, +:class:`zarr.storage.ZipStore` or :class:`zarr.storage.FsspecStore` then the store data +**are not** pickled. The only thing that is pickled is the necessary parameters to allow the store +to re-open any underlying files or databases upon being unpickled. + +E.g., pickle/unpickle an local store array:: + + >>> import pickle + >>> data = np.arange(100000) + >>> z1 = zarr.create_array(store='data/example-2.zarr', shape=data.shape, chunks=data.shape, dtype=data.dtype) + >>> z1[:] = data + >>> s = pickle.dumps(z1) + >>> z2 = pickle.loads(s) + >>> z1 == z2 + True + >>> np.all(z1[:] == z2[:]) + np.True_ + +.. _user-guide-tips-blosc: + +Configuring Blosc +----------------- + +Coming soon. diff --git a/docs/user-guide/storage.rst b/docs/user-guide/storage.rst new file mode 100644 index 0000000000..4215cbaf20 --- /dev/null +++ b/docs/user-guide/storage.rst @@ -0,0 +1,148 @@ +.. only:: doctest + + >>> import shutil + >>> shutil.rmtree('data', ignore_errors=True) + +.. _user-guide-storage: + +Storage guide +============= + +Zarr-Python supports multiple storage backends, including: local file systems, +Zip files, remote stores via fsspec_ (S3, HTTP, etc.), and in-memory stores. In +Zarr-Python 3, stores must implement the abstract store API from +:class:`zarr.abc.store.Store`. + +.. note:: + Unlike Zarr-Python 2 where the store interface was built around a generic ``MutableMapping`` + API, Zarr-Python 3 utilizes a custom store API that utilizes Python's AsyncIO library. + +Implicit Store Creation +----------------------- + +In most cases, it is not required to create a ``Store`` object explicitly. Passing a string +to Zarr's top level API will result in the store being created automatically.: + + >>> import zarr + >>> + >>> # Implicitly create a writable LocalStore + >>> zarr.create_group(store='data/foo/bar') + + >>> + >>> # Implicitly create a read-only FsspecStore + >>> zarr.open_group( + ... store='s3://noaa-nwm-retro-v2-zarr-pds', + ... mode='r', + ... storage_options={'anon': True} + ... ) + > + >>> + >>> # Implicitly creates a MemoryStore + >>> data = {} + >>> zarr.create_group(store=data) + + +Explicit Store Creation +----------------------- + +In some cases, it may be helpful to create a store instance directly. Zarr-Python offers four +built-in store: :class:`zarr.storage.LocalStore`, :class:`zarr.storage.FsspecStore`, +:class:`zarr.storage.ZipStore`, :class:`zarr.storage.MemoryStore`, and :class:`zarr.storage.ObjectStore`. + +Local Store +~~~~~~~~~~~ + +The :class:`zarr.storage.LocalStore` stores data in a nested set of directories on a local +filesystem.: + + >>> store = zarr.storage.LocalStore('data/foo/bar', read_only=True) + >>> zarr.open_group(store=store, mode='r') + + +Zip Store +~~~~~~~~~ + +The :class:`zarr.storage.ZipStore` stores the contents of a Zarr hierarchy in a single +Zip file. The `Zip Store specification`_ is currently in draft form.: + + >>> store = zarr.storage.ZipStore('data.zip', mode='w') + >>> zarr.create_array(store=store, shape=(2,), dtype='float64') + + +Remote Store +~~~~~~~~~~~~ + +The :class:`zarr.storage.FsspecStore` stores the contents of a Zarr hierarchy in following the same +logical layout as the ``LocalStore``, except the store is assumed to be on a remote storage system +such as cloud object storage (e.g. AWS S3, Google Cloud Storage, Azure Blob Store). The +:class:`zarr.storage.FsspecStore` is backed by `fsspec`_ and can support any backend +that implements the `AbstractFileSystem `_ +API. ``storage_options`` can be used to configure the fsspec backend.: + + >>> store = zarr.storage.FsspecStore.from_url( + ... 's3://noaa-nwm-retro-v2-zarr-pds', + ... read_only=True, + ... storage_options={'anon': True} + ... ) + >>> zarr.open_group(store=store, mode='r') + > + +Memory Store +~~~~~~~~~~~~ + +The :class:`zarr.storage.MemoryStore` a in-memory store that allows for serialization of +Zarr data (metadata and chunks) to a dictionary.: + + >>> data = {} + >>> store = zarr.storage.MemoryStore(data) + >>> # TODO: replace with create_array after #2463 + >>> zarr.create_array(store=store, shape=(2,), dtype='float64') + + +Object Store +~~~~~~~~~~~~ + +:class:`zarr.storage.ObjectStore` stores the contents of the Zarr hierarchy using any ObjectStore +`storage implementation `_, including AWS S3 (:class:`obstore.store.S3Store`), Google Cloud Storage (:class:`obstore.store.GCSStore`), and Azure Blob Storage (:class:`obstore.store.AzureStore`). This store is backed by `obstore `_, which +builds on the production quality Rust library `object_store `_. + + + >>> from zarr.storage import ObjectStore + >>> from obstore.store import MemoryStore + >>> + >>> store = ObjectStore(MemoryStore()) + >>> zarr.create_array(store=store, shape=(2,), dtype='float64') + + +Here's an example of using ObjectStore for accessing remote data: + + >>> from zarr.storage import ObjectStore + >>> from obstore.store import S3Store + >>> + >>> s3_store = S3Store('noaa-nwm-retro-v2-zarr-pds', skip_signature=True, region="us-west-2") + >>> store = zarr.storage.ObjectStore(store=s3_store, read_only=True) + >>> group = zarr.open_group(store=store, mode='r') + >>> group.info + Name : + Type : Group + Zarr format : 2 + Read-only : True + Store type : ObjectStore + No. members : 12 + No. arrays : 12 + No. groups : 0 + +.. warning:: + The :class:`zarr.storage.ObjectStore` class is experimental. + +.. _user-guide-custom-stores: + +Developing custom stores +------------------------ + +Zarr-Python :class:`zarr.abc.store.Store` API is meant to be extended. The Store Abstract Base +Class includes all of the methods needed to be a fully operational store in Zarr Python. +Zarr also provides a test harness for custom stores: :class:`zarr.testing.store.StoreTests`. + +.. _Zip Store Specification: https://github.com/zarr-developers/zarr-specs/pull/311 +.. _fsspec: https://filesystem-spec.readthedocs.io diff --git a/docs/user-guide/v3_migration.rst b/docs/user-guide/v3_migration.rst new file mode 100644 index 0000000000..a6258534e4 --- /dev/null +++ b/docs/user-guide/v3_migration.rst @@ -0,0 +1,238 @@ +.. _v3 migration guide: + +3.0 Migration Guide +=================== + +Zarr-Python 3 represents a major refactor of the Zarr-Python codebase. Some of the +goals motivating this refactor included: + +* adding support for the Zarr format 3 specification (along with the Zarr format 2 specification) +* cleaning up internal and user facing APIs +* improving performance (particularly in high latency storage environments like + cloud object stores) + +To accommodate this, Zarr-Python 3 introduces a number of changes to the API, including a number +of significant breaking changes and deprecations. + +This page provides a guide explaining breaking changes and deprecations to help you +migrate your code from version 2 to version 3. If we have missed anything, please +open a `GitHub issue `_ +so we can improve this guide. + +Compatibility target +-------------------- + +The goals described above necessitated some breaking changes to the API (hence the +major version update), but where possible we have maintained backwards compatibility +in the most widely used parts of the API. This in the :class:`zarr.Array` and +:class:`zarr.Group` classes and the "top-level API" (e.g. :func:`zarr.open_array` and +:func:`zarr.open_group`). + +Getting ready for 3.0 +--------------------- + +Before migrating to Zarr-Python 3, we suggest projects that depend on Zarr-Python take +the following actions in order: + +1. Pin the supported Zarr-Python version to ``zarr>=2,<3``. This is a best practice + and will protect your users from any incompatibilities that may arise during the + release of Zarr-Python 3. This pin can be removed after migrating to Zarr-Python 3. +2. Limit your imports from the Zarr-Python package. Most of the primary API ``zarr.*`` + will be compatible in Zarr-Python 3. However, the following breaking API changes are + planned: + + - ``numcodecs.*`` will no longer be available in ``zarr.*``. To migrate, import codecs + directly from ``numcodecs``: + + .. code-block:: python + + from numcodecs import Blosc + # instead of: + # from zarr import Blosc + + - The ``zarr.v3_api_available`` feature flag is being removed. In Zarr-Python 3 + the v3 API is always available, so you shouldn't need to use this flag. + - The following internal modules are being removed or significantly changed. If + your application relies on imports from any of the below modules, you will need + to either a) modify your application to no longer rely on these imports or b) + vendor the parts of the specific modules that you need. + + * ``zarr.attrs`` has gone, with no replacement + * ``zarr.codecs`` has gone, use ``numcodecs`` instead + * ``zarr.context`` has gone, with no replacement + * ``zarr.core`` remains but should be considered private API + * ``zarr.hierarchy`` has gone, with no replacement (use ``zarr.Group`` inplace of ``zarr.hierarchy.Group``) + * ``zarr.indexing`` has gone, with no replacement + * ``zarr.meta`` has gone, with no replacement + * ``zarr.meta_v1`` has gone, with no replacement + * ``zarr.sync`` has gone, with no replacement + * ``zarr.types`` has gone, with no replacement + * ``zarr.util`` has gone, with no replacement + * ``zarr.n5`` has gone, see below for an alternative N5 options + +3. Test that your package works with version 3. +4. Update the pin to include ``zarr>=3,<4``. + +Zarr-Python 2 support window +---------------------------- + +Zarr-Python 2.x is still available, though we recommend migrating to Zarr-Python 3 for +its performance improvements and new features. Security and bug fixes will be made to +the 2.x series for at least six months following the first Zarr-Python 3 release. +If you need to use the latest Zarr-Python 2 release, you can install it with: + +.. code-block:: console + + $ pip install "zarr==2.*" + +.. note:: + Development and maintenance of the 2.x release series has moved to the + `support/v2 `_ branch. + Issues and pull requests related to this branch are tagged with the + `V2 `_ label. + +Migrating to Zarr-Python 3 +-------------------------- + +The following sections provide details on breaking changes in Zarr-Python 3. + +The Array class +~~~~~~~~~~~~~~~ + +1. Disallow direct construction - the signature for initializing the ``Array`` class has changed + significantly. Please use :func:`zarr.create_array` or :func:`zarr.open_array` instead of + directly constructing the :class:`zarr.Array` class. + +2. Defaulting to ``zarr_format=3`` - newly created arrays will use the version 3 of the + Zarr specification. To continue using version 2, set ``zarr_format=2`` when creating arrays + or set ``default_zarr_version=2`` in Zarr's :ref:`runtime configuration `. + +The Group class +~~~~~~~~~~~~~~~ + +1. Disallow direct construction - use :func:`zarr.open_group` or :func:`zarr.create_group` + instead of directly constructing the :class:`zarr.Group` class. +2. Most of the h5py compatibility methods are deprecated and will issue warnings if used. + The following functions are drop in replacements that have the same signature and functionality: + + - Use :func:`zarr.Group.create_array` in place of :func:`zarr.Group.create_dataset` + - Use :func:`zarr.Group.require_array` in place of :func:`zarr.Group.require_dataset` +3. Disallow "." syntax for getting group members. To get a member of a group named ``foo``, + use ``group["foo"]`` in place of ``group.foo``. + +The Store class +~~~~~~~~~~~~~~~ + +The Store API has changed significant in Zarr-Python 3. The most notable changes to the +Store API are: + +Store Import Paths +^^^^^^^^^^^^^^^^^^ +Several store implementations have moved from the top-level module to ``zarr.storage``: + +.. code-block:: diff + :caption: Store import changes from v2 to v3 + + # Before (v2) + - from zarr import MemoryStore, DirectoryStore + + from zarr.storage import MemoryStore, LocalStore # LocalStore replaces DirectoryStore + +Common replacements: + ++-------------------------+------------------------------------+ +| v2 Import | v3 Import | ++=========================+====================================+ +| ``zarr.MemoryStore`` | ``zarr.storage.MemoryStore`` | ++-------------------------+------------------------------------+ +| ``zarr.DirectoryStore`` | ``zarr.storage.LocalStore`` | ++-------------------------+------------------------------------+ +| ``zarr.TempStore`` | Use ``tempfile.TemporaryDirectory``| +| | with ``LocalStore`` | ++-------------------------+------------------------------------+ + +1. Replaced the ``MutableMapping`` base class in favor of a custom abstract base class + (:class:`zarr.abc.store.Store`). +2. Switched to an asynchronous interface for all store methods that result in IO. This + change ensures that all store methods are non-blocking and are as performant as + possible. + +Beyond the changes store interface, a number of deprecated stores were also removed in +Zarr-Python 3. See :issue:`1274` for more details on the removal of these stores. + +- ``N5Store`` - see https://github.com/zarr-developers/n5py for an alternative interface to + N5 formatted data. +- ``ABSStore`` - use the :class:`zarr.storage.FsspecStore` instead along with fsspec's + `adlfs backend `_. + +The following stores have been removed altogether. Users who need these stores will have to +implement their own version in zarr-python v3. + +- ``DBMStore`` +- ``LMDBStore`` +- ``SQLiteStore`` +- ``MongoDBStore`` +- ``RedisStore`` + +At present, the latter five stores in this list do not have an equivalent in Zarr-Python 3. +If you are interested in developing a custom store that targets these backends, see +:ref:`developing custom stores ` or open an +`issue `_ to discuss your use case. + +Dependencies +~~~~~~~~~~~~ + +When installing using ``pip``: + +- The new ``remote`` dependency group can be used to install a supported version of + ``fsspec``, required for remote data access. +- The new ``gpu`` dependency group can be used to install a supported version of + ``cuda``, required for GPU functionality. +- The ``jupyter`` optional dependency group has been removed, since v3 contains no + jupyter specific functionality. + +Miscellaneous +~~~~~~~~~~~~~ + +- The keyword argument ``zarr_version`` available in most creation functions in :mod:`zarr` + (e.g. :func:`zarr.create`, :func:`zarr.open`, :func:`zarr.group`, :func:`zarr.array`) has + been deprecated in favor of ``zarr_format``. + +🚧 Work in Progress 🚧 +---------------------- + +Zarr-Python 3 is still under active development, and is not yet fully complete. +The following list summarizes areas of the codebase that we expect to build out +after the 3.0.0 release. If features listed below are important to your use case +of Zarr-Python, please open (or comment on) a +`GitHub issue `_. + +- The following functions / methods have not been ported to Zarr-Python 3 yet: + + * :func:`zarr.copy` (:issue:`2407`) + * :func:`zarr.copy_all` (:issue:`2407`) + * :func:`zarr.copy_store` (:issue:`2407`) + * :func:`zarr.Group.move` (:issue:`2108`) + +- The following features (corresponding to function arguments to functions in + :mod:`zarr`) have not been ported to Zarr-Python 3 yet. Using these features + will raise a warning or a ``NotImplementedError``: + + * ``cache_attrs`` + * ``cache_metadata`` + * ``chunk_store`` (:issue:`2495`) + * ``meta_array`` + * ``object_codec`` (:issue:`2617`) + * ``synchronizer`` (:issue:`1596`) + * ``dimension_separator`` + +- The following features that were supported by Zarr-Python 2 have not been ported + to Zarr-Python 3 yet: + + * Structured arrays / dtypes (:issue:`2134`) + * Fixed-length string dtypes (:issue:`2347`) + * Datetime and timedelta dtypes (:issue:`2616`) + * Object dtypes (:issue:`2617`) + * Ragged arrays (:issue:`2618`) + * Groups and Arrays do not implement ``__enter__`` and ``__exit__`` protocols (:issue:`2619`) + * Big Endian dtypes (:issue:`2324`) + * Default filters for object dtypes for Zarr format 2 arrays (:issue:`2627`) diff --git a/fixture/.zattrs b/fixture/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/.zgroup b/fixture/.zgroup deleted file mode 100644 index 3b7daf227c..0000000000 --- a/fixture/.zgroup +++ /dev/null @@ -1,3 +0,0 @@ -{ - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/0/.zattrs b/fixture/0/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/0/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/0/.zgroup b/fixture/0/.zgroup deleted file mode 100644 index 3b7daf227c..0000000000 --- a/fixture/0/.zgroup +++ /dev/null @@ -1,3 +0,0 @@ -{ - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/0/0/.zarray b/fixture/0/0/.zarray deleted file mode 100644 index a5ceafaf51..0000000000 --- a/fixture/0/0/.zarray +++ /dev/null @@ -1,14 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": null, - "dtype": "|i1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 1111 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/0/0/.zattrs b/fixture/0/0/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/0/0/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/0/0/0 b/fixture/0/0/0 deleted file mode 100644 index e2ff041306..0000000000 Binary files a/fixture/0/0/0 and /dev/null differ diff --git a/fixture/0/0/1 b/fixture/0/0/1 deleted file mode 100644 index 78f25e295b..0000000000 --- a/fixture/0/0/1 +++ /dev/null @@ -1 +0,0 @@ -defghijklmnopqrstuvwxyz{|}~ \ No newline at end of file diff --git a/fixture/0/0/10 b/fixture/0/0/10 deleted file mode 100644 index 8877030829..0000000000 Binary files a/fixture/0/0/10 and /dev/null differ diff --git a/fixture/0/0/11 b/fixture/0/0/11 deleted file mode 100644 index 3cea8e9b57..0000000000 Binary files a/fixture/0/0/11 and /dev/null differ diff --git a/fixture/0/0/2 b/fixture/0/0/2 deleted file mode 100644 index a83c7e1089..0000000000 Binary files a/fixture/0/0/2 and /dev/null differ diff --git a/fixture/0/0/3 b/fixture/0/0/3 deleted file mode 100644 index 8c0210c510..0000000000 --- a/fixture/0/0/3 +++ /dev/null @@ -1 +0,0 @@ -,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ \ No newline at end of file diff --git a/fixture/0/0/4 b/fixture/0/0/4 deleted file mode 100644 index 660224ac06..0000000000 --- a/fixture/0/0/4 +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/fixture/0/0/5 b/fixture/0/0/5 deleted file mode 100644 index 9881b13bab..0000000000 Binary files a/fixture/0/0/5 and /dev/null differ diff --git a/fixture/0/0/6 b/fixture/0/0/6 deleted file mode 100644 index 1e8a425fa5..0000000000 --- a/fixture/0/0/6 +++ /dev/null @@ -1 +0,0 @@ -XYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ \ No newline at end of file diff --git a/fixture/0/0/7 b/fixture/0/0/7 deleted file mode 100644 index c0723b01b9..0000000000 Binary files a/fixture/0/0/7 and /dev/null differ diff --git a/fixture/0/0/8 b/fixture/0/0/8 deleted file mode 100644 index a38dc0f2ed..0000000000 --- a/fixture/0/0/8 +++ /dev/null @@ -1 +0,0 @@ - !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ \ No newline at end of file diff --git a/fixture/0/0/9 b/fixture/0/0/9 deleted file mode 100644 index dd735849d9..0000000000 --- a/fixture/0/0/9 +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/fixture/0/1/.zarray b/fixture/0/1/.zarray deleted file mode 100644 index 5bb56828ec..0000000000 --- a/fixture/0/1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "|i1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 1111 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/0/1/.zattrs b/fixture/0/1/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/0/1/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/0/1/0 b/fixture/0/1/0 deleted file mode 100644 index 15265bc589..0000000000 Binary files a/fixture/0/1/0 and /dev/null differ diff --git a/fixture/0/1/1 b/fixture/0/1/1 deleted file mode 100644 index ce1eab1e87..0000000000 Binary files a/fixture/0/1/1 and /dev/null differ diff --git a/fixture/0/1/10 b/fixture/0/1/10 deleted file mode 100644 index c078b2d562..0000000000 Binary files a/fixture/0/1/10 and /dev/null differ diff --git a/fixture/0/1/11 b/fixture/0/1/11 deleted file mode 100644 index 6023b8df53..0000000000 Binary files a/fixture/0/1/11 and /dev/null differ diff --git a/fixture/0/1/2 b/fixture/0/1/2 deleted file mode 100644 index 50ebada272..0000000000 Binary files a/fixture/0/1/2 and /dev/null differ diff --git a/fixture/0/1/3 b/fixture/0/1/3 deleted file mode 100644 index 972deb8eb7..0000000000 Binary files a/fixture/0/1/3 and /dev/null differ diff --git a/fixture/0/1/4 b/fixture/0/1/4 deleted file mode 100644 index 73036acec7..0000000000 Binary files a/fixture/0/1/4 and /dev/null differ diff --git a/fixture/0/1/5 b/fixture/0/1/5 deleted file mode 100644 index d529b5becf..0000000000 Binary files a/fixture/0/1/5 and /dev/null differ diff --git a/fixture/0/1/6 b/fixture/0/1/6 deleted file mode 100644 index 0a70e9d853..0000000000 Binary files a/fixture/0/1/6 and /dev/null differ diff --git a/fixture/0/1/7 b/fixture/0/1/7 deleted file mode 100644 index d77fbab72d..0000000000 Binary files a/fixture/0/1/7 and /dev/null differ diff --git a/fixture/0/1/8 b/fixture/0/1/8 deleted file mode 100644 index eba245b537..0000000000 Binary files a/fixture/0/1/8 and /dev/null differ diff --git a/fixture/0/1/9 b/fixture/0/1/9 deleted file mode 100644 index 384417a247..0000000000 Binary files a/fixture/0/1/9 and /dev/null differ diff --git a/fixture/0/2/.zarray b/fixture/0/2/.zarray deleted file mode 100644 index 29781e09b4..0000000000 --- a/fixture/0/2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "|i1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 1111 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/0/2/.zattrs b/fixture/0/2/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/0/2/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/0/2/0 b/fixture/0/2/0 deleted file mode 100644 index abd664663b..0000000000 Binary files a/fixture/0/2/0 and /dev/null differ diff --git a/fixture/0/2/1 b/fixture/0/2/1 deleted file mode 100644 index 1564164b4e..0000000000 Binary files a/fixture/0/2/1 and /dev/null differ diff --git a/fixture/0/2/10 b/fixture/0/2/10 deleted file mode 100644 index cab8dd9562..0000000000 Binary files a/fixture/0/2/10 and /dev/null differ diff --git a/fixture/0/2/11 b/fixture/0/2/11 deleted file mode 100644 index 68767304c9..0000000000 Binary files a/fixture/0/2/11 and /dev/null differ diff --git a/fixture/0/2/2 b/fixture/0/2/2 deleted file mode 100644 index 974b1371aa..0000000000 Binary files a/fixture/0/2/2 and /dev/null differ diff --git a/fixture/0/2/3 b/fixture/0/2/3 deleted file mode 100644 index c35d1c72df..0000000000 Binary files a/fixture/0/2/3 and /dev/null differ diff --git a/fixture/0/2/4 b/fixture/0/2/4 deleted file mode 100644 index 6a3be3ab38..0000000000 Binary files a/fixture/0/2/4 and /dev/null differ diff --git a/fixture/0/2/5 b/fixture/0/2/5 deleted file mode 100644 index 19c8db1db5..0000000000 Binary files a/fixture/0/2/5 and /dev/null differ diff --git a/fixture/0/2/6 b/fixture/0/2/6 deleted file mode 100644 index d643cd509e..0000000000 Binary files a/fixture/0/2/6 and /dev/null differ diff --git a/fixture/0/2/7 b/fixture/0/2/7 deleted file mode 100644 index df4e24e128..0000000000 Binary files a/fixture/0/2/7 and /dev/null differ diff --git a/fixture/0/2/8 b/fixture/0/2/8 deleted file mode 100644 index 7cd5d6ef34..0000000000 Binary files a/fixture/0/2/8 and /dev/null differ diff --git a/fixture/0/2/9 b/fixture/0/2/9 deleted file mode 100644 index 2f6acf8879..0000000000 Binary files a/fixture/0/2/9 and /dev/null differ diff --git a/fixture/0/3/.zarray b/fixture/0/3/.zarray deleted file mode 100644 index 72f8a611b7..0000000000 --- a/fixture/0/3/.zarray +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 0 - }, - "dtype": "|i1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 1111 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/0/3/.zattrs b/fixture/0/3/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/0/3/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/0/3/0 b/fixture/0/3/0 deleted file mode 100644 index fba4f86574..0000000000 Binary files a/fixture/0/3/0 and /dev/null differ diff --git a/fixture/0/3/1 b/fixture/0/3/1 deleted file mode 100644 index c9e50670fd..0000000000 Binary files a/fixture/0/3/1 and /dev/null differ diff --git a/fixture/0/3/10 b/fixture/0/3/10 deleted file mode 100644 index 778ada21c8..0000000000 Binary files a/fixture/0/3/10 and /dev/null differ diff --git a/fixture/0/3/11 b/fixture/0/3/11 deleted file mode 100644 index 059c531ef1..0000000000 Binary files a/fixture/0/3/11 and /dev/null differ diff --git a/fixture/0/3/2 b/fixture/0/3/2 deleted file mode 100644 index ced542b0e6..0000000000 Binary files a/fixture/0/3/2 and /dev/null differ diff --git a/fixture/0/3/3 b/fixture/0/3/3 deleted file mode 100644 index 0b88a8dbf2..0000000000 Binary files a/fixture/0/3/3 and /dev/null differ diff --git a/fixture/0/3/4 b/fixture/0/3/4 deleted file mode 100644 index f040a33d2b..0000000000 Binary files a/fixture/0/3/4 and /dev/null differ diff --git a/fixture/0/3/5 b/fixture/0/3/5 deleted file mode 100644 index 23f36c034d..0000000000 Binary files a/fixture/0/3/5 and /dev/null differ diff --git a/fixture/0/3/6 b/fixture/0/3/6 deleted file mode 100644 index 100978fe7d..0000000000 Binary files a/fixture/0/3/6 and /dev/null differ diff --git a/fixture/0/3/7 b/fixture/0/3/7 deleted file mode 100644 index 5c35201d19..0000000000 Binary files a/fixture/0/3/7 and /dev/null differ diff --git a/fixture/0/3/8 b/fixture/0/3/8 deleted file mode 100644 index 2e8c78de94..0000000000 Binary files a/fixture/0/3/8 and /dev/null differ diff --git a/fixture/0/3/9 b/fixture/0/3/9 deleted file mode 100644 index 47ae524b63..0000000000 Binary files a/fixture/0/3/9 and /dev/null differ diff --git a/fixture/0/4/.zarray b/fixture/0/4/.zarray deleted file mode 100644 index 44f2b6b28d..0000000000 --- a/fixture/0/4/.zarray +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "|i1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 1111 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/0/4/.zattrs b/fixture/0/4/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/0/4/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/0/4/0 b/fixture/0/4/0 deleted file mode 100644 index 2f2e353cdb..0000000000 Binary files a/fixture/0/4/0 and /dev/null differ diff --git a/fixture/0/4/1 b/fixture/0/4/1 deleted file mode 100644 index 66aa292396..0000000000 Binary files a/fixture/0/4/1 and /dev/null differ diff --git a/fixture/0/4/10 b/fixture/0/4/10 deleted file mode 100644 index 7152729709..0000000000 Binary files a/fixture/0/4/10 and /dev/null differ diff --git a/fixture/0/4/11 b/fixture/0/4/11 deleted file mode 100644 index 34487e2221..0000000000 Binary files a/fixture/0/4/11 and /dev/null differ diff --git a/fixture/0/4/2 b/fixture/0/4/2 deleted file mode 100644 index 6a4f50a5c6..0000000000 Binary files a/fixture/0/4/2 and /dev/null differ diff --git a/fixture/0/4/3 b/fixture/0/4/3 deleted file mode 100644 index e880807229..0000000000 Binary files a/fixture/0/4/3 and /dev/null differ diff --git a/fixture/0/4/4 b/fixture/0/4/4 deleted file mode 100644 index 361412ed89..0000000000 Binary files a/fixture/0/4/4 and /dev/null differ diff --git a/fixture/0/4/5 b/fixture/0/4/5 deleted file mode 100644 index ff99c56a58..0000000000 Binary files a/fixture/0/4/5 and /dev/null differ diff --git a/fixture/0/4/6 b/fixture/0/4/6 deleted file mode 100644 index 8d942d9f59..0000000000 Binary files a/fixture/0/4/6 and /dev/null differ diff --git a/fixture/0/4/7 b/fixture/0/4/7 deleted file mode 100644 index f06f5a69ef..0000000000 Binary files a/fixture/0/4/7 and /dev/null differ diff --git a/fixture/0/4/8 b/fixture/0/4/8 deleted file mode 100644 index 949d6169a9..0000000000 Binary files a/fixture/0/4/8 and /dev/null differ diff --git a/fixture/0/4/9 b/fixture/0/4/9 deleted file mode 100644 index e8e7bfbc6f..0000000000 Binary files a/fixture/0/4/9 and /dev/null differ diff --git a/fixture/0/5/.zarray b/fixture/0/5/.zarray deleted file mode 100644 index beafdea229..0000000000 --- a/fixture/0/5/.zarray +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 2 - }, - "dtype": "|i1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 1111 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/0/5/.zattrs b/fixture/0/5/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/0/5/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/0/5/0 b/fixture/0/5/0 deleted file mode 100644 index be5fbdb799..0000000000 Binary files a/fixture/0/5/0 and /dev/null differ diff --git a/fixture/0/5/1 b/fixture/0/5/1 deleted file mode 100644 index a533973891..0000000000 Binary files a/fixture/0/5/1 and /dev/null differ diff --git a/fixture/0/5/10 b/fixture/0/5/10 deleted file mode 100644 index 8da499ccdc..0000000000 Binary files a/fixture/0/5/10 and /dev/null differ diff --git a/fixture/0/5/11 b/fixture/0/5/11 deleted file mode 100644 index dccc6ff075..0000000000 Binary files a/fixture/0/5/11 and /dev/null differ diff --git a/fixture/0/5/2 b/fixture/0/5/2 deleted file mode 100644 index b23ccb7a41..0000000000 Binary files a/fixture/0/5/2 and /dev/null differ diff --git a/fixture/0/5/3 b/fixture/0/5/3 deleted file mode 100644 index 1a91655f65..0000000000 Binary files a/fixture/0/5/3 and /dev/null differ diff --git a/fixture/0/5/4 b/fixture/0/5/4 deleted file mode 100644 index 12218977e5..0000000000 Binary files a/fixture/0/5/4 and /dev/null differ diff --git a/fixture/0/5/5 b/fixture/0/5/5 deleted file mode 100644 index e99841ecc9..0000000000 Binary files a/fixture/0/5/5 and /dev/null differ diff --git a/fixture/0/5/6 b/fixture/0/5/6 deleted file mode 100644 index 8b5793f721..0000000000 Binary files a/fixture/0/5/6 and /dev/null differ diff --git a/fixture/0/5/7 b/fixture/0/5/7 deleted file mode 100644 index 4d51d23803..0000000000 Binary files a/fixture/0/5/7 and /dev/null differ diff --git a/fixture/0/5/8 b/fixture/0/5/8 deleted file mode 100644 index 5c709708a3..0000000000 Binary files a/fixture/0/5/8 and /dev/null differ diff --git a/fixture/0/5/9 b/fixture/0/5/9 deleted file mode 100644 index 45ff8d600a..0000000000 Binary files a/fixture/0/5/9 and /dev/null differ diff --git a/fixture/0/6/.zarray b/fixture/0/6/.zarray deleted file mode 100644 index 5cdc9c43ce..0000000000 --- a/fixture/0/6/.zarray +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "clevel": 1, - "cname": "lz4", - "id": "blosc", - "shuffle": 0 - }, - "dtype": "|i1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 1111 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/0/6/.zattrs b/fixture/0/6/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/0/6/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/0/6/0 b/fixture/0/6/0 deleted file mode 100644 index 3a1472f269..0000000000 Binary files a/fixture/0/6/0 and /dev/null differ diff --git a/fixture/0/6/1 b/fixture/0/6/1 deleted file mode 100644 index 1cdc89f84d..0000000000 Binary files a/fixture/0/6/1 and /dev/null differ diff --git a/fixture/0/6/10 b/fixture/0/6/10 deleted file mode 100644 index fd646f5b2a..0000000000 Binary files a/fixture/0/6/10 and /dev/null differ diff --git a/fixture/0/6/11 b/fixture/0/6/11 deleted file mode 100644 index 145a3e3312..0000000000 Binary files a/fixture/0/6/11 and /dev/null differ diff --git a/fixture/0/6/2 b/fixture/0/6/2 deleted file mode 100644 index 3531ed7d2f..0000000000 Binary files a/fixture/0/6/2 and /dev/null differ diff --git a/fixture/0/6/3 b/fixture/0/6/3 deleted file mode 100644 index bf93587bc7..0000000000 Binary files a/fixture/0/6/3 and /dev/null differ diff --git a/fixture/0/6/4 b/fixture/0/6/4 deleted file mode 100644 index 777a1e0096..0000000000 Binary files a/fixture/0/6/4 and /dev/null differ diff --git a/fixture/0/6/5 b/fixture/0/6/5 deleted file mode 100644 index d83297db06..0000000000 Binary files a/fixture/0/6/5 and /dev/null differ diff --git a/fixture/0/6/6 b/fixture/0/6/6 deleted file mode 100644 index 3658bddb50..0000000000 Binary files a/fixture/0/6/6 and /dev/null differ diff --git a/fixture/0/6/7 b/fixture/0/6/7 deleted file mode 100644 index deb2e30b64..0000000000 Binary files a/fixture/0/6/7 and /dev/null differ diff --git a/fixture/0/6/8 b/fixture/0/6/8 deleted file mode 100644 index aad890f334..0000000000 Binary files a/fixture/0/6/8 and /dev/null differ diff --git a/fixture/0/6/9 b/fixture/0/6/9 deleted file mode 100644 index 402ae9f831..0000000000 Binary files a/fixture/0/6/9 and /dev/null differ diff --git a/fixture/1/.zattrs b/fixture/1/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/1/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/1/.zgroup b/fixture/1/.zgroup deleted file mode 100644 index 3b7daf227c..0000000000 --- a/fixture/1/.zgroup +++ /dev/null @@ -1,3 +0,0 @@ -{ - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/1/0/.zarray b/fixture/1/0/.zarray deleted file mode 100644 index 30d16d3590..0000000000 --- a/fixture/1/0/.zarray +++ /dev/null @@ -1,14 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": null, - "dtype": "?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ \ No newline at end of file diff --git a/fixture/1/0/4 b/fixture/1/0/4 deleted file mode 100644 index 75c13b9686..0000000000 --- a/fixture/1/0/4 +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/fixture/1/0/5 b/fixture/1/0/5 deleted file mode 100644 index e7e7c3894f..0000000000 Binary files a/fixture/1/0/5 and /dev/null differ diff --git a/fixture/1/0/6 b/fixture/1/0/6 deleted file mode 100644 index 6efa2226c7..0000000000 --- a/fixture/1/0/6 +++ /dev/null @@ -1 +0,0 @@ -XYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ \ No newline at end of file diff --git a/fixture/1/0/7 b/fixture/1/0/7 deleted file mode 100644 index 40b6edb0e8..0000000000 Binary files a/fixture/1/0/7 and /dev/null differ diff --git a/fixture/1/0/8 b/fixture/1/0/8 deleted file mode 100644 index 28159ad9ad..0000000000 --- a/fixture/1/0/8 +++ /dev/null @@ -1 +0,0 @@ - !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ \ No newline at end of file diff --git a/fixture/1/0/9 b/fixture/1/0/9 deleted file mode 100644 index c5223e6cb7..0000000000 --- a/fixture/1/0/9 +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/fixture/1/1/.zarray b/fixture/1/1/.zarray deleted file mode 100644 index a97385dba8..0000000000 --- a/fixture/1/1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "4և?;?HS?&&?w?`t?3. ?jO ?ںq? Z?\\?T?(LG?N??Ο2?}?uAk"?I$I$?ݮ'?3})?ÄPz+? g.?j&E0?=w|R3?55?v=8?iܲ:?aB(=?_ ??2\ B?sD?FF? \ No newline at end of file diff --git a/fixture/10/0/18 b/fixture/10/0/18 deleted file mode 100644 index 6723998f09..0000000000 Binary files a/fixture/10/0/18 and /dev/null differ diff --git a/fixture/10/0/19 b/fixture/10/0/19 deleted file mode 100644 index d4939e2039..0000000000 Binary files a/fixture/10/0/19 and /dev/null differ diff --git a/fixture/10/0/2 b/fixture/10/0/2 deleted file mode 100644 index ca09d8e730..0000000000 Binary files a/fixture/10/0/2 and /dev/null differ diff --git a/fixture/10/0/20 b/fixture/10/0/20 deleted file mode 100644 index 1557320928..0000000000 Binary files a/fixture/10/0/20 and /dev/null differ diff --git a/fixture/10/0/21 b/fixture/10/0/21 deleted file mode 100644 index ba57279a90..0000000000 Binary files a/fixture/10/0/21 and /dev/null differ diff --git a/fixture/10/0/22 b/fixture/10/0/22 deleted file mode 100644 index 51a98b25b1..0000000000 Binary files a/fixture/10/0/22 and /dev/null differ diff --git a/fixture/10/0/23 b/fixture/10/0/23 deleted file mode 100644 index 93aaf87e1b..0000000000 Binary files a/fixture/10/0/23 and /dev/null differ diff --git a/fixture/10/0/24 b/fixture/10/0/24 deleted file mode 100644 index 8e16623744..0000000000 Binary files a/fixture/10/0/24 and /dev/null differ diff --git a/fixture/10/0/25 b/fixture/10/0/25 deleted file mode 100644 index c43b3e6f7b..0000000000 --- a/fixture/10/0/25 +++ /dev/null @@ -1,2 +0,0 @@ -Peu?r?z+1`?MC ?!J ?Wb?55?-?n׃ ?B(=?y^ ?ɯĀ?i*?k"k"?c$?7 [V'? -^N)?ݮ'A,?.?Pz+1?X3X3?+6?B#8?ғ_;?v=?y5T??LaB? D D?'LG?xI?pQ7L?n*N?Ak"Q?S? V V?]NX?ȳZ?bm]?6P;_? Wb?Jd?BgBg? i?W{-l?*5n?LGq?s?'v?xxxxxx?K1z?Cc}?j?ƻ]N? uÄ?m]8?@@?B#?O ?q?n׃?aB(=?5n??4TnY? Ο?9D?V'?)x9/?j??je6?w?K z?]g?Jd?3ں?OO?lvĿ??/d:?Bɯ?撢/%?[?4?``?4և?'A,p?w?ȳZ?m]?Uj&E?((? 0?\R? Z?vĿ?JO~%?7{??AVf?c? \ No newline at end of file diff --git a/fixture/10/0/26 b/fixture/10/0/26 deleted file mode 100644 index edd19678d3..0000000000 Binary files a/fixture/10/0/26 and /dev/null differ diff --git a/fixture/10/0/27 b/fixture/10/0/27 deleted file mode 100644 index 64162766cb..0000000000 Binary files a/fixture/10/0/27 and /dev/null differ diff --git a/fixture/10/0/28 b/fixture/10/0/28 deleted file mode 100644 index 7b802c59ef..0000000000 Binary files a/fixture/10/0/28 and /dev/null differ diff --git a/fixture/10/0/29 b/fixture/10/0/29 deleted file mode 100644 index 203b228a58..0000000000 Binary files a/fixture/10/0/29 and /dev/null differ diff --git a/fixture/10/0/3 b/fixture/10/0/3 deleted file mode 100644 index bc547b9434..0000000000 Binary files a/fixture/10/0/3 and /dev/null differ diff --git a/fixture/10/0/30 b/fixture/10/0/30 deleted file mode 100644 index 88173dc6c1..0000000000 Binary files a/fixture/10/0/30 and /dev/null differ diff --git a/fixture/10/0/31 b/fixture/10/0/31 deleted file mode 100644 index 427b2b0f05..0000000000 Binary files a/fixture/10/0/31 and /dev/null differ diff --git a/fixture/10/0/32 b/fixture/10/0/32 deleted file mode 100644 index f09a926dc3..0000000000 Binary files a/fixture/10/0/32 and /dev/null differ diff --git a/fixture/10/0/33 b/fixture/10/0/33 deleted file mode 100644 index 97fe96ef78..0000000000 Binary files a/fixture/10/0/33 and /dev/null differ diff --git a/fixture/10/0/4 b/fixture/10/0/4 deleted file mode 100644 index 854be0ff01..0000000000 Binary files a/fixture/10/0/4 and /dev/null differ diff --git a/fixture/10/0/5 b/fixture/10/0/5 deleted file mode 100644 index 9c2fd9c4f6..0000000000 Binary files a/fixture/10/0/5 and /dev/null differ diff --git a/fixture/10/0/6 b/fixture/10/0/6 deleted file mode 100644 index 0c04006d60..0000000000 Binary files a/fixture/10/0/6 and /dev/null differ diff --git a/fixture/10/0/7 b/fixture/10/0/7 deleted file mode 100644 index ddfa980a18..0000000000 Binary files a/fixture/10/0/7 and /dev/null differ diff --git a/fixture/10/0/8 b/fixture/10/0/8 deleted file mode 100644 index 7a783b85c2..0000000000 Binary files a/fixture/10/0/8 and /dev/null differ diff --git a/fixture/10/0/9 b/fixture/10/0/9 deleted file mode 100644 index af34267355..0000000000 Binary files a/fixture/10/0/9 and /dev/null differ diff --git a/fixture/10/1/.zarray b/fixture/10/1/.zarray deleted file mode 100644 index 22d0521bb4..0000000000 --- a/fixture/10/1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "9>DD.G۽*}+2>v]{P{P<}ҖyLT"|=WYJycJx8Y|tx)еș?[Ui^(fٯUY 1SryQ YyJPrIh竐JeCM*AѢe/w&z/"]^'Sj#eu[Ў÷ӷ1̬Fh3épN4QˈPԁF3A񺠳BmbNrE{kƆ{f`R"|Ѧ?찒Г6@?'msHp-pр 90WI~NIcNrt!h|E.wEw lLq<ńkg]K^ 'JmsǤe}>C3 YO5?g { \ No newline at end of file diff --git a/fixture/10/1/1 b/fixture/10/1/1 deleted file mode 100644 index 02b8839bbe..0000000000 Binary files a/fixture/10/1/1 and /dev/null differ diff --git a/fixture/10/1/10 b/fixture/10/1/10 deleted file mode 100644 index cb60d899ef..0000000000 Binary files a/fixture/10/1/10 and /dev/null differ diff --git a/fixture/10/1/11 b/fixture/10/1/11 deleted file mode 100644 index d56a6b14d9..0000000000 Binary files a/fixture/10/1/11 and /dev/null differ diff --git a/fixture/10/1/12 b/fixture/10/1/12 deleted file mode 100644 index fd2f26059f..0000000000 Binary files a/fixture/10/1/12 and /dev/null differ diff --git a/fixture/10/1/13 b/fixture/10/1/13 deleted file mode 100644 index 2842e2d286..0000000000 Binary files a/fixture/10/1/13 and /dev/null differ diff --git a/fixture/10/1/14 b/fixture/10/1/14 deleted file mode 100644 index 0c5434ef4e..0000000000 --- a/fixture/10/1/14 +++ /dev/null @@ -1,3 +0,0 @@ -xO=斚c%IPi<{!պݲ:6@pyT15y9kls5+ b/UV¥ԍ;݄nקs_b{ lbofh/-gW ">1B4*@7a6Ϙ%Zg1dVNeL{ox:Skl6ѿ5V;GW3t'3|;2n4pxgƸx 7#&vdp\nFsL.0V~*QXdg[!aь=Fqlg7M`l|`**3\*w(ţ2csLKN&O\Ilv:bD`X%.wlbc \ No newline at end of file diff --git a/fixture/10/1/15 b/fixture/10/1/15 deleted file mode 100644 index 1e89e57760..0000000000 --- a/fixture/10/1/15 +++ /dev/null @@ -1 +0,0 @@ -x%OuNf!TEtQ"]^)KB.]V&H9TF(2o6,z~{پ|RCVβ6St7щKbձw0>@%"P;r_\%@=jRfCyVp,ˆMW)ԕБ=ﳁքciwCV!K!i8lH8פQۙ0$%$BZk#OWK#!l]LXLjAAXoe#0Ґ^;A҈ ֍!|3#YUcA4*iWu>Ԣ1!PT-(`{+~g/\&* :* M+JBćZi\צ:B]e!a˼c̈́:ۢ[ %7yuU~z?{g/[`}A G6> !1]3O1[14/߻1e?XfTFc}V8k6 v;"hVWTƸ~]N*,-c_9c C- Ð4e"g 3ҥ0-ft$)c$Y:%0lFFp_*czpL3; sRXP<*Ћ4wQa1pIX<_w \ No newline at end of file diff --git a/fixture/10/1/16 b/fixture/10/1/16 deleted file mode 100644 index d5dc7cbf4d..0000000000 Binary files a/fixture/10/1/16 and /dev/null differ diff --git a/fixture/10/1/17 b/fixture/10/1/17 deleted file mode 100644 index 340d9e9f3e..0000000000 Binary files a/fixture/10/1/17 and /dev/null differ diff --git a/fixture/10/1/18 b/fixture/10/1/18 deleted file mode 100644 index 64de2010c8..0000000000 Binary files a/fixture/10/1/18 and /dev/null differ diff --git a/fixture/10/1/19 b/fixture/10/1/19 deleted file mode 100644 index e565f92a85..0000000000 Binary files a/fixture/10/1/19 and /dev/null differ diff --git a/fixture/10/1/2 b/fixture/10/1/2 deleted file mode 100644 index 66d3e9c4c0..0000000000 Binary files a/fixture/10/1/2 and /dev/null differ diff --git a/fixture/10/1/20 b/fixture/10/1/20 deleted file mode 100644 index 258a408bc0..0000000000 Binary files a/fixture/10/1/20 and /dev/null differ diff --git a/fixture/10/1/21 b/fixture/10/1/21 deleted file mode 100644 index 1dea1a1966..0000000000 Binary files a/fixture/10/1/21 and /dev/null differ diff --git a/fixture/10/1/22 b/fixture/10/1/22 deleted file mode 100644 index 1112800ad4..0000000000 Binary files a/fixture/10/1/22 and /dev/null differ diff --git a/fixture/10/1/23 b/fixture/10/1/23 deleted file mode 100644 index 8668791927..0000000000 Binary files a/fixture/10/1/23 and /dev/null differ diff --git a/fixture/10/1/24 b/fixture/10/1/24 deleted file mode 100644 index 45e2b33cca..0000000000 Binary files a/fixture/10/1/24 and /dev/null differ diff --git a/fixture/10/1/25 b/fixture/10/1/25 deleted file mode 100644 index 96fee7e4b6..0000000000 Binary files a/fixture/10/1/25 and /dev/null differ diff --git a/fixture/10/1/26 b/fixture/10/1/26 deleted file mode 100644 index 385173c4fe..0000000000 Binary files a/fixture/10/1/26 and /dev/null differ diff --git a/fixture/10/1/27 b/fixture/10/1/27 deleted file mode 100644 index d5a1e919e9..0000000000 Binary files a/fixture/10/1/27 and /dev/null differ diff --git a/fixture/10/1/28 b/fixture/10/1/28 deleted file mode 100644 index f5814dd451..0000000000 Binary files a/fixture/10/1/28 and /dev/null differ diff --git a/fixture/10/1/29 b/fixture/10/1/29 deleted file mode 100644 index c65ec5a0f7..0000000000 Binary files a/fixture/10/1/29 and /dev/null differ diff --git a/fixture/10/1/3 b/fixture/10/1/3 deleted file mode 100644 index 213eebd6da..0000000000 Binary files a/fixture/10/1/3 and /dev/null differ diff --git a/fixture/10/1/30 b/fixture/10/1/30 deleted file mode 100644 index e905d02969..0000000000 Binary files a/fixture/10/1/30 and /dev/null differ diff --git a/fixture/10/1/31 b/fixture/10/1/31 deleted file mode 100644 index 92ffd9fa57..0000000000 Binary files a/fixture/10/1/31 and /dev/null differ diff --git a/fixture/10/1/32 b/fixture/10/1/32 deleted file mode 100644 index a3848cf61d..0000000000 Binary files a/fixture/10/1/32 and /dev/null differ diff --git a/fixture/10/1/33 b/fixture/10/1/33 deleted file mode 100644 index f3df61f29d..0000000000 Binary files a/fixture/10/1/33 and /dev/null differ diff --git a/fixture/10/1/4 b/fixture/10/1/4 deleted file mode 100644 index 3594417999..0000000000 Binary files a/fixture/10/1/4 and /dev/null differ diff --git a/fixture/10/1/5 b/fixture/10/1/5 deleted file mode 100644 index 4b24bd2df3..0000000000 Binary files a/fixture/10/1/5 and /dev/null differ diff --git a/fixture/10/1/6 b/fixture/10/1/6 deleted file mode 100644 index 8df6aa4ab5..0000000000 Binary files a/fixture/10/1/6 and /dev/null differ diff --git a/fixture/10/1/7 b/fixture/10/1/7 deleted file mode 100644 index 2ac06323d0..0000000000 Binary files a/fixture/10/1/7 and /dev/null differ diff --git a/fixture/10/1/8 b/fixture/10/1/8 deleted file mode 100644 index 226233a422..0000000000 Binary files a/fixture/10/1/8 and /dev/null differ diff --git a/fixture/10/1/9 b/fixture/10/1/9 deleted file mode 100644 index 4b973979c2..0000000000 Binary files a/fixture/10/1/9 and /dev/null differ diff --git a/fixture/10/2/.zarray b/fixture/10/2/.zarray deleted file mode 100644 index d2e6713e1b..0000000000 --- a/fixture/10/2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "i5G8&9-/>6=.8幗4/7/=^=X<9=6:8U+ \ No newline at end of file diff --git a/fixture/11/0/1 b/fixture/11/0/1 deleted file mode 100644 index 13a9154b89..0000000000 --- a/fixture/11/0/1 +++ /dev/null @@ -1,3 +0,0 @@ -L<>Ĺ;8052U8=F:4u?q<޼w; :}'3??km91|8⽄0y<: \ No newline at end of file diff --git a/fixture/11/0/10 b/fixture/11/0/10 deleted file mode 100644 index 53ef107b49..0000000000 Binary files a/fixture/11/0/10 and /dev/null differ diff --git a/fixture/11/0/11 b/fixture/11/0/11 deleted file mode 100644 index d1cb1e56f1..0000000000 --- a/fixture/11/0/11 +++ /dev/null @@ -1,2 +0,0 @@ -66@8}>%ؽ)=:/,^=ԹD; -96N50=8·O?f->,;d%O87p909W(+=v:ѮC_4434#4e,!Ѻڽv;w7/=-$S:4=;1ȼ \ No newline at end of file diff --git a/fixture/11/0/13 b/fixture/11/0/13 deleted file mode 100644 index 15ff462e5a..0000000000 --- a/fixture/11/0/13 +++ /dev/null @@ -1,2 +0,0 @@ -K6u8Ku6eB=<Lw4:oH=4V⻟=̷E8=?i:q<0ոJ9?ҩ7ղM9u@ݶ8N=96Z8 ->9L:49 >^6TD<< \ No newline at end of file diff --git a/fixture/11/0/14 b/fixture/11/0/14 deleted file mode 100644 index 09b1a1b3c8..0000000000 --- a/fixture/11/0/14 +++ /dev/null @@ -1 +0,0 @@ --+;:%:1<7Ĺ>D9q:J2<:5.C5ռ5Z?*8 96 ;8_U5o1¾02ൣ@=5=,/6w<>A5--8?7Ͼh8;:<ߺ^8o}'Sˮ>@ 6fb \ No newline at end of file diff --git a/fixture/11/0/15 b/fixture/11/0/15 deleted file mode 100644 index 4f8776cf9c..0000000000 Binary files a/fixture/11/0/15 and /dev/null differ diff --git a/fixture/11/0/16 b/fixture/11/0/16 deleted file mode 100644 index 2cbb62bcae..0000000000 Binary files a/fixture/11/0/16 and /dev/null differ diff --git a/fixture/11/0/17 b/fixture/11/0/17 deleted file mode 100644 index 15fafb4084..0000000000 --- a/fixture/11/0/17 +++ /dev/null @@ -1,2 +0,0 @@ -!W&n^<:;?@^A130?&=WϺn ->5w=K<`Ź޾#(8,7:<]2>88_8Ÿ?V4;3K7I?4);:q:>24ḥ0Cw9s<u{:_@rp>4M%">0(..=. \ No newline at end of file diff --git a/fixture/11/0/18 b/fixture/11/0/18 deleted file mode 100644 index d8f260531e..0000000000 Binary files a/fixture/11/0/18 and /dev/null differ diff --git a/fixture/11/0/19 b/fixture/11/0/19 deleted file mode 100644 index 00f8d0cbe4..0000000000 --- a/fixture/11/0/19 +++ /dev/null @@ -1,2 +0,0 @@ -<=];'.<<4 ;l;<8v 4QͲ<8 j2\5޷ŷ/!.O;a<־9=;>*8; -9}!6pt7x?<9;9370;yb2. -b*8E;:`}>g97=P8pU3-:ӿ574L706lm \ No newline at end of file diff --git a/fixture/11/0/2 b/fixture/11/0/2 deleted file mode 100644 index 4299d85edb..0000000000 Binary files a/fixture/11/0/2 and /dev/null differ diff --git a/fixture/11/0/20 b/fixture/11/0/20 deleted file mode 100644 index c2b6993b4e..0000000000 --- a/fixture/11/0/20 +++ /dev/null @@ -1 +0,0 @@ -o=U8c= ?6#<=;<4]<<][:ұl00->w侼G<0;0X:t39:=t4ᴘ9޼>nѳ6~3,:P5˺.<N0'.ƴk9`0<?_(a8u=N㸒0!!b== i;Q3>轨T?޹\<% \ No newline at end of file diff --git a/fixture/11/0/21 b/fixture/11/0/21 deleted file mode 100644 index fb5e25e22c..0000000000 --- a/fixture/11/0/21 +++ /dev/null @@ -1 +0,0 @@ -@%<6@} >>`9m69C<8Sȸ?ʽܽB=K1A>@yGA(45Fn@t8M/fI7@9-S2i6x8?;d29s;9!O.;7b88qi': \ No newline at end of file diff --git a/fixture/11/0/22 b/fixture/11/0/22 deleted file mode 100644 index aa6a8b475c..0000000000 --- a/fixture/11/0/22 +++ /dev/null @@ -1 +0,0 @@ - 0~?;(`\2$8:@RΗw(?[>6P0V5%t8025&:30o6?@I8E6I4‹L9qJbg< \ No newline at end of file diff --git a/fixture/11/0/23 b/fixture/11/0/23 deleted file mode 100644 index d0fe213d8f..0000000000 Binary files a/fixture/11/0/23 and /dev/null differ diff --git a/fixture/11/0/24 b/fixture/11/0/24 deleted file mode 100644 index 1e8f3b67a5..0000000000 --- a/fixture/11/0/24 +++ /dev/null @@ -1 +0,0 @@ -68d+4T=r6/:6< ;5D.1:{:599;ͬܺJ@(;;7h2:6"=i@< L:5V\9<Ṡ448v8V9ej0*863)=86w9̶;<"2\7@3M;14jL=68 \ No newline at end of file diff --git a/fixture/11/0/27 b/fixture/11/0/27 deleted file mode 100644 index bfa91897e8..0000000000 Binary files a/fixture/11/0/27 and /dev/null differ diff --git a/fixture/11/0/28 b/fixture/11/0/28 deleted file mode 100644 index 99ccc787f1..0000000000 Binary files a/fixture/11/0/28 and /dev/null differ diff --git a/fixture/11/0/29 b/fixture/11/0/29 deleted file mode 100644 index 167b98e01d..0000000000 --- a/fixture/11/0/29 +++ /dev/null @@ -1 +0,0 @@ -N7L2+ݺL482ɹE;V8e;*/x9Ѫ׻W$9:wԽBY:65542":q9j?;;6:-B+66=)8C<%889+44 \ No newline at end of file diff --git a/fixture/11/0/3 b/fixture/11/0/3 deleted file mode 100644 index 905f9b7612..0000000000 --- a/fixture/11/0/3 +++ /dev/null @@ -1 +0,0 @@ -˶%<_8<6ۿݷ514_AI91:#o6+4E76R,7Bۼ?<ص̸q@3s%;56+<6Ӵ3&_<˷=35$:y9J/R#>i0Ҽ \ No newline at end of file diff --git a/fixture/11/0/30 b/fixture/11/0/30 deleted file mode 100644 index bc51790d37..0000000000 --- a/fixture/11/0/30 +++ /dev/null @@ -1 +0,0 @@ - ༬KG2?: `9/7{<8X@:$"U8(M5y>b74=HO/ 7Z.9XE@9><19>=8<+:A*\>¸28_6 @a3'f5ܶ-߼e~?V"l4:@<>a@{;.38p<5ƽ974{> \ No newline at end of file diff --git a/fixture/11/0/31 b/fixture/11/0/31 deleted file mode 100644 index 05cb06ebce..0000000000 Binary files a/fixture/11/0/31 and /dev/null differ diff --git a/fixture/11/0/32 b/fixture/11/0/32 deleted file mode 100644 index 1843ec7ad4..0000000000 --- a/fixture/11/0/32 +++ /dev/null @@ -1,3 +0,0 @@ -1}̬ -d8@k5:=;7G40F055&6#9,뵊8#ym.V5-92X,=?.8.B|4c/"7 -:Ȧ9m5;i4׹?;A@1G85^`4:n9*e>׺AF˾g?0~7d169 \ No newline at end of file diff --git a/fixture/11/0/33 b/fixture/11/0/33 deleted file mode 100644 index b10dfd066c..0000000000 Binary files a/fixture/11/0/33 and /dev/null differ diff --git a/fixture/11/0/34 b/fixture/11/0/34 deleted file mode 100644 index fb6593d730..0000000000 Binary files a/fixture/11/0/34 and /dev/null differ diff --git a/fixture/11/0/35 b/fixture/11/0/35 deleted file mode 100644 index c011536cb3..0000000000 --- a/fixture/11/0/35 +++ /dev/null @@ -1,2 +0,0 @@ -cf< ->3j0?S:4>;MԬ>;l0ʹ0?089:;v@./=8{:`:>6<;1?A'>7@8C::`vG8M90>J6=9= +53Q>0B V.:Q(4`639ԯ>A9L38g1ڹ \ No newline at end of file diff --git a/fixture/11/0/36 b/fixture/11/0/36 deleted file mode 100644 index 993c47dc49..0000000000 --- a/fixture/11/0/36 +++ /dev/null @@ -1 +0,0 @@ -8й}5A9_Tz@r>=26<ѹC=~<26)-;[9'/U <{-;@8R>ټR@r \ No newline at end of file diff --git a/fixture/11/0/37 b/fixture/11/0/37 deleted file mode 100644 index 2fc46c23b9..0000000000 --- a/fixture/11/0/37 +++ /dev/null @@ -1 +0,0 @@ -|8lkX=O$ix7R:;I-2;̹N#=-i@g;r6P7D; 88<<뿾]5lU603@~'6Y=d57>F>û>;eY3V+44*b;95O-=! )D8 \ No newline at end of file diff --git a/fixture/11/0/39 b/fixture/11/0/39 deleted file mode 100644 index 263bbde985..0000000000 --- a/fixture/11/0/39 +++ /dev/null @@ -1 +0,0 @@ -7ӵ;n=7jvc8J38⺫ٷ̰Q?$8=;˻A øZ(99;;)(<8B9u >ᰉ:}$;A4#8+6:!:3/=MҺ2I?6Q8<7?`kF9c:1%?8P*289=Q'+=8A \ No newline at end of file diff --git a/fixture/11/0/4 b/fixture/11/0/4 deleted file mode 100644 index ce9d4b174c..0000000000 --- a/fixture/11/0/4 +++ /dev/null @@ -1 +0,0 @@ - 8D29ٽg>:ּ/ p9:=?{::x7 81L70o6)@8967䶭%0Ӵ|:9v<66TB<&P_к<$ \ No newline at end of file diff --git a/fixture/11/0/40 b/fixture/11/0/40 deleted file mode 100644 index 4eb39f7e4d..0000000000 --- a/fixture/11/0/40 +++ /dev/null @@ -1 +0,0 @@ -`,>n*;Q<8!0y28J5P<\,7i8;;߷;2<7D>9~86h׻<9/y/><7=j9\8!8. 38p98,#8=84>4l6Ͻ,8Z7)79(7;;2R \ No newline at end of file diff --git a/fixture/11/0/41 b/fixture/11/0/41 deleted file mode 100644 index 8a93d9202f..0000000000 Binary files a/fixture/11/0/41 and /dev/null differ diff --git a/fixture/11/0/42 b/fixture/11/0/42 deleted file mode 100644 index 5e2efd24b3..0000000000 --- a/fixture/11/0/42 +++ /dev/null @@ -1 +0,0 @@ -=,>4޾Q=2=7649%<:K-\; _G9Ķ2̷⭛64OHD;%736>@4N??96W¼>94;r!X99y<<|M8m8 \ No newline at end of file diff --git a/fixture/11/0/43 b/fixture/11/0/43 deleted file mode 100644 index 00bdfb3836..0000000000 Binary files a/fixture/11/0/43 and /dev/null differ diff --git a/fixture/11/0/44 b/fixture/11/0/44 deleted file mode 100644 index 91d5de0a32..0000000000 Binary files a/fixture/11/0/44 and /dev/null differ diff --git a/fixture/11/0/5 b/fixture/11/0/5 deleted file mode 100644 index 3ec1c53e47..0000000000 Binary files a/fixture/11/0/5 and /dev/null differ diff --git a/fixture/11/0/6 b/fixture/11/0/6 deleted file mode 100644 index e1a103c1d6..0000000000 Binary files a/fixture/11/0/6 and /dev/null differ diff --git a/fixture/11/0/7 b/fixture/11/0/7 deleted file mode 100644 index ad95b008a1..0000000000 --- a/fixture/11/0/7 +++ /dev/null @@ -1,2 +0,0 @@ -9&8Dw9K<55;E8ȥ&,]=j^3,}>?/H2b9Ey&<Ύ94$>A -m41]&ϴ8AX.=[ø e4G<-<<9%<:χU<9=Z.<5 \ No newline at end of file diff --git a/fixture/11/0/8 b/fixture/11/0/8 deleted file mode 100644 index 5763394a9c..0000000000 --- a/fixture/11/0/8 +++ /dev/null @@ -1 +0,0 @@ -A#Ժu-<@B9?Q:35%=kw='T@8] y:!: <<=2ջi5α%!}lԲ;U>y?<18@W0=ɵyi9,-s;4b\;1=/Un:*  1<5ߩa<60;z8}DAA817=V \ No newline at end of file diff --git a/fixture/11/0/9 b/fixture/11/0/9 deleted file mode 100644 index fa532219b7..0000000000 --- a/fixture/11/0/9 +++ /dev/null @@ -1 +0,0 @@ -:H>60v:̽75<*8` 8i<4p541ܵH \ No newline at end of file diff --git a/fixture/11/1/.zarray b/fixture/11/1/.zarray deleted file mode 100644 index f9b2b175cd..0000000000 --- a/fixture/11/1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "!=j ܾ?k>=A?n?3?唾SþM=1d=Z@q>+Ȇ>7M?92?CIZ?a'?Tfpf]>U$aC>n6zZ~?=?lp? ?1vdVeԾVIR@G>P$?zzV==????p?^?^w@p*>>h@Fv.[Q?z?3$@y!= ܾ \ No newline at end of file diff --git a/fixture/12/0/1 b/fixture/12/0/1 deleted file mode 100644 index 9b9429fe11..0000000000 Binary files a/fixture/12/0/1 and /dev/null differ diff --git a/fixture/12/0/10 b/fixture/12/0/10 deleted file mode 100644 index f83a3a63a9..0000000000 Binary files a/fixture/12/0/10 and /dev/null differ diff --git a/fixture/12/0/11 b/fixture/12/0/11 deleted file mode 100644 index 3e466a751e..0000000000 Binary files a/fixture/12/0/11 and /dev/null differ diff --git a/fixture/12/0/12 b/fixture/12/0/12 deleted file mode 100644 index 489b1377b4..0000000000 Binary files a/fixture/12/0/12 and /dev/null differ diff --git a/fixture/12/0/13 b/fixture/12/0/13 deleted file mode 100644 index 8afa5f2795..0000000000 Binary files a/fixture/12/0/13 and /dev/null differ diff --git a/fixture/12/0/14 b/fixture/12/0/14 deleted file mode 100644 index 8d55d29a4e..0000000000 Binary files a/fixture/12/0/14 and /dev/null differ diff --git a/fixture/12/0/15 b/fixture/12/0/15 deleted file mode 100644 index 8f26830f6b..0000000000 Binary files a/fixture/12/0/15 and /dev/null differ diff --git a/fixture/12/0/16 b/fixture/12/0/16 deleted file mode 100644 index 4235871468..0000000000 --- a/fixture/12/0/16 +++ /dev/null @@ -1,3 +0,0 @@ -D*?7?"A3ɾI>?C:t,ݸ?*d?E?>>H씾dN ?x>(An4^_x3?9,?+ ?Q >Ӂ_/9m=\T9?=G9Į>DV @{8>X?U?ʿоXX>Y&?T>?w?= ? `?@^ -߽\|s?|׿i?t ?pee?ῠq_ -y?ǃI=E?s}v&̾V'>D[84?D␾"?N?2O?k|X?;9@@K?|R?Wf? \ No newline at end of file diff --git a/fixture/12/0/17 b/fixture/12/0/17 deleted file mode 100644 index 68d2a64bd1..0000000000 --- a/fixture/12/0/17 +++ /dev/null @@ -1 +0,0 @@ -~?$V?4?4ugg?T?9#?~Gx?q>[lz???->?_!np?AQDT]?G7?WU@.>b)' >>׹>F?D)J>@kŕh>柱=J ?2d>e?#-?=,k > ;j1]>Ţ?*Y?%?9@>#?{>;>8+?' >3_>`f;?+F`ڿ0>yag?׿8v?+R?[=b_2ܿu>anQ2:?HR?r.&˾1,ſo?K?O?]>t޿@6>x >I෾?b ?`UE? \ No newline at end of file diff --git a/fixture/12/0/18 b/fixture/12/0/18 deleted file mode 100644 index 97165ea436..0000000000 --- a/fixture/12/0/18 +++ /dev/null @@ -1,3 +0,0 @@ ->_>M?J^s(>fvH>(?+??: v?F=>;? ʾ wF??+7 ->w?ۨ| -'?O?[o0a>"?;@f?5þ{˿?4?m~>E?1DqĔO?p?If?u?r7b=BûD>oX=j>F">@߱?0=e]>F?R\?OEr??Z?Q;-q>6;?i8p5?%Qȿ">s>#;=a= *& x??T׾Իǿ+WG.%?8ǽ1+{>ՓI@T?ֽ?5xn.CƳ=򊥿ꜾZ_?Y -j 3>O?35 ?w>oē6-?S?@r.q`?ſ?Oz>E3?/w%#>sZ|OzNŚ?μ> K*m_?<b2???BA}>XGŵc>>#@ I3K?>4-?'N>Q[??R>_8s,@8?'#?>'Nѿǿ̾ h⪾g?pɋәl(?%?> >vk>!Ƕ?g>G?J־{G+v,a? \ No newline at end of file diff --git a/fixture/12/0/2 b/fixture/12/0/2 deleted file mode 100644 index c65cb4faf6..0000000000 Binary files a/fixture/12/0/2 and /dev/null differ diff --git a/fixture/12/0/20 b/fixture/12/0/20 deleted file mode 100644 index 1fa5d6a28c..0000000000 Binary files a/fixture/12/0/20 and /dev/null differ diff --git a/fixture/12/0/21 b/fixture/12/0/21 deleted file mode 100644 index b5e98ec68f..0000000000 Binary files a/fixture/12/0/21 and /dev/null differ diff --git a/fixture/12/0/22 b/fixture/12/0/22 deleted file mode 100644 index 44a174c497..0000000000 --- a/fixture/12/0/22 +++ /dev/null @@ -1,2 +0,0 @@ -*N?i;-'q*W?n$>U"*2:I-#9?S A>>i;?؎? ÿ&t@.># ?apm?P?,lw=ATo?9=o>/ ?>ܰE?ݾ|d?m&?w?=]P?f?Ҿ> |3@o*? -@TWpA|< Ϳu2Ti}dE?ϛIf??mq>׿ M1|>NHz(7p؄ӾV>7>|?!>>X6ajӾ(:"*sXW?;?iCď???E :(>JZ>?t)W)A* \ No newline at end of file diff --git a/fixture/12/0/23 b/fixture/12/0/23 deleted file mode 100644 index 923ef95c25..0000000000 Binary files a/fixture/12/0/23 and /dev/null differ diff --git a/fixture/12/0/24 b/fixture/12/0/24 deleted file mode 100644 index aea1365f3b..0000000000 Binary files a/fixture/12/0/24 and /dev/null differ diff --git a/fixture/12/0/25 b/fixture/12/0/25 deleted file mode 100644 index 44910af4de..0000000000 Binary files a/fixture/12/0/25 and /dev/null differ diff --git a/fixture/12/0/26 b/fixture/12/0/26 deleted file mode 100644 index c4ecd62445..0000000000 Binary files a/fixture/12/0/26 and /dev/null differ diff --git a/fixture/12/0/27 b/fixture/12/0/27 deleted file mode 100644 index e79cd846aa..0000000000 --- a/fixture/12/0/27 +++ /dev/null @@ -1,2 +0,0 @@ -?(K?iy?dտD~(;B?ZC?n)l* _'>MֿT}c}?{TP>H?S8I?D??Cj>~1%=OF?:B?7)?Ӑ?9T1ɾ?g>>#?aJ'?>/?_g>?!;>Onӓ;g-׾@;N>?:S޾?M>7;G+? kZJ>?>M?(p> -8c>秫C-cjTҿH>gl>d?o`"?r|<0@ !A7n'?h@?vIm]?g鐿0>>>> \ No newline at end of file diff --git a/fixture/12/0/28 b/fixture/12/0/28 deleted file mode 100644 index d1cb366014..0000000000 Binary files a/fixture/12/0/28 and /dev/null differ diff --git a/fixture/12/0/29 b/fixture/12/0/29 deleted file mode 100644 index 4fd9f62569..0000000000 --- a/fixture/12/0/29 +++ /dev/null @@ -1,3 +0,0 @@ -?mze?Du5[]?Ĝ ?Խ=5NR\6e>ŢXq?E?S/Ñ?Ҿ6Ռ?9?H?Q>D\?9 >顳?k>N?dUluJP?GV9?f?=&qq@=w>w] U?'>ׇ>?T,= ?ݶ>hJ?,B?vfڿ"E>Xk?WѾ @Vs f -?.l>4z&?l>=?nGC?s!}Z1 ->JTP?Q!?@?z>13?jf٢?6Kvb?h?nj@?$?Y|B@?s>Xž9E?v_ \ No newline at end of file diff --git a/fixture/12/0/3 b/fixture/12/0/3 deleted file mode 100644 index 4b438ca798..0000000000 Binary files a/fixture/12/0/3 and /dev/null differ diff --git a/fixture/12/0/30 b/fixture/12/0/30 deleted file mode 100644 index ddfb4c0542..0000000000 Binary files a/fixture/12/0/30 and /dev/null differ diff --git a/fixture/12/0/31 b/fixture/12/0/31 deleted file mode 100644 index 1ba45c2c63..0000000000 Binary files a/fixture/12/0/31 and /dev/null differ diff --git a/fixture/12/0/32 b/fixture/12/0/32 deleted file mode 100644 index a53c50a212..0000000000 Binary files a/fixture/12/0/32 and /dev/null differ diff --git a/fixture/12/0/33 b/fixture/12/0/33 deleted file mode 100644 index 57dad6a2ff..0000000000 --- a/fixture/12/0/33 +++ /dev/null @@ -1,3 +0,0 @@ -r>rp?L@4ZW(ޝ?\>z >rҖ?YB5>+>˦1=I?0 -=v=˿?FD^?(d 15]>q??T?W=JmZ?}E>+ ?K|=(W?")>j}?z -><=~zc=?3[ξi#ZϾԶ=8zuj"=?t?ͧ ? }Cde=p?ſ6z>6?ѾW=P?)͐q=k?ň?q?;n{E>PE?ֆ'Kw=^3=?!F?oJږ_?+7=ᐿF?C@˖&$ \ No newline at end of file diff --git a/fixture/12/0/34 b/fixture/12/0/34 deleted file mode 100644 index 3c685db58a..0000000000 Binary files a/fixture/12/0/34 and /dev/null differ diff --git a/fixture/12/0/35 b/fixture/12/0/35 deleted file mode 100644 index 6565391175..0000000000 --- a/fixture/12/0/35 +++ /dev/null @@ -1,3 +0,0 @@ ->?g)? -@3/>`1^:@t?[Xc?qp+?T?V t? (Hk']p=%*xd?!?,h>7e+˿Y?%;VJ!,??/?eCbm>7>?f?owG @ig?>=wM>NKM>>G9*ѿUVMc?BʿA> 7F>IΰҼ隿aж?|ܿg_? >NJӺ?-C?>8mP?=[?9^ԡ \ No newline at end of file diff --git a/fixture/12/0/36 b/fixture/12/0/36 deleted file mode 100644 index 4880d11394..0000000000 Binary files a/fixture/12/0/36 and /dev/null differ diff --git a/fixture/12/0/37 b/fixture/12/0/37 deleted file mode 100644 index d96b7f9322..0000000000 Binary files a/fixture/12/0/37 and /dev/null differ diff --git a/fixture/12/0/38 b/fixture/12/0/38 deleted file mode 100644 index 98effd7ff1..0000000000 Binary files a/fixture/12/0/38 and /dev/null differ diff --git a/fixture/12/0/39 b/fixture/12/0/39 deleted file mode 100644 index 6d42fb1981..0000000000 --- a/fixture/12/0/39 +++ /dev/null @@ -1,3 +0,0 @@ -oS"(_>cljP>Ț[__U -@^)뇽?ο4YuϿN&;$>\;L>@=6gS?ڕD>L=ұS>+=9?}7~>V.=`͏{=?- -ֿ(UNpt|>?JU??/?"f?V79?>&6w~iq?<,A?U:#?@㾅1t>x??YA?ӧ9P?H|=>?gM??猵V >\?Wm= ?0?n N6?޾St>G]?cD?|?0>#>. S:>t쏿9࿗}? \ No newline at end of file diff --git a/fixture/12/0/4 b/fixture/12/0/4 deleted file mode 100644 index a5df461dfe..0000000000 --- a/fixture/12/0/4 +++ /dev/null @@ -1,3 +0,0 @@ -c3? 7?IPޔ?J -=0ÿ8B?=q@}?.=B?C?=Ӿ-d>?~Q?X]t;\?ct?W[`*.?r>e?)ؒ?kq?y4?#IͿ??ٝ?LA? %>ö>}f?X?-=?u>ʯZ>62PtX>`? uq?~֝aQ@? -?H?`?f"?y>P?*j{?\qrQ+? ۿsS!?;{DOڿ"?^ݍ=SRE?eA}?Fq]pnEG೿G&>G2 \ No newline at end of file diff --git a/fixture/12/0/40 b/fixture/12/0/40 deleted file mode 100644 index e4f807ccc8..0000000000 Binary files a/fixture/12/0/40 and /dev/null differ diff --git a/fixture/12/0/41 b/fixture/12/0/41 deleted file mode 100644 index 23862be11c..0000000000 --- a/fixture/12/0/41 +++ /dev/null @@ -1,3 +0,0 @@ -U/q>p/x \?.?X=_w -%~>K ? F>`?՚g?R?:2 ?KR9?&> P>Ra(u=G0x?^t?W`k?q??>{J,¿?]eS+>*ʾaR]?>4?t? HFg_??сx:?щ?h*?>Ó>HAֻ*Fc@*n >'?$&I7$D?J@p>W -ѾxyB?=53nJ?Td]7?wL8fH?l?D@<{>>X>>3@ \ No newline at end of file diff --git a/fixture/12/0/42 b/fixture/12/0/42 deleted file mode 100644 index 8b46168136..0000000000 Binary files a/fixture/12/0/42 and /dev/null differ diff --git a/fixture/12/0/43 b/fixture/12/0/43 deleted file mode 100644 index 73d8748d67..0000000000 Binary files a/fixture/12/0/43 and /dev/null differ diff --git a/fixture/12/0/44 b/fixture/12/0/44 deleted file mode 100644 index 0f4fbe6750..0000000000 Binary files a/fixture/12/0/44 and /dev/null differ diff --git a/fixture/12/0/5 b/fixture/12/0/5 deleted file mode 100644 index ccb46ccd84..0000000000 Binary files a/fixture/12/0/5 and /dev/null differ diff --git a/fixture/12/0/6 b/fixture/12/0/6 deleted file mode 100644 index bd4fbccf89..0000000000 Binary files a/fixture/12/0/6 and /dev/null differ diff --git a/fixture/12/0/7 b/fixture/12/0/7 deleted file mode 100644 index 819851265f..0000000000 Binary files a/fixture/12/0/7 and /dev/null differ diff --git a/fixture/12/0/8 b/fixture/12/0/8 deleted file mode 100644 index f20d0033a7..0000000000 Binary files a/fixture/12/0/8 and /dev/null differ diff --git a/fixture/12/0/9 b/fixture/12/0/9 deleted file mode 100644 index 7a3eb5aee7..0000000000 Binary files a/fixture/12/0/9 and /dev/null differ diff --git a/fixture/12/1/.zarray b/fixture/12/1/.zarray deleted file mode 100644 index fcf798d35f..0000000000 --- a/fixture/12/1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "!=j ܾ?k>=A?n?3?唾SþM=1d=Z@q>+Ȇ>7M?92?CIZ?a'?Tfpf]>U$aC>n6zZ~?=?lp? ?1vdVeԾVIR@G>P$?zzV==????p?^?^w@p*>>h@Fv.[Q?z?3$@y!= ܾƤ \ No newline at end of file diff --git a/fixture/12/1/1 b/fixture/12/1/1 deleted file mode 100644 index 07917e896b..0000000000 Binary files a/fixture/12/1/1 and /dev/null differ diff --git a/fixture/12/1/10 b/fixture/12/1/10 deleted file mode 100644 index 481f147b51..0000000000 Binary files a/fixture/12/1/10 and /dev/null differ diff --git a/fixture/12/1/11 b/fixture/12/1/11 deleted file mode 100644 index 3cd01382cb..0000000000 Binary files a/fixture/12/1/11 and /dev/null differ diff --git a/fixture/12/1/12 b/fixture/12/1/12 deleted file mode 100644 index 3cbc8eee68..0000000000 Binary files a/fixture/12/1/12 and /dev/null differ diff --git a/fixture/12/1/13 b/fixture/12/1/13 deleted file mode 100644 index 096a4829b4..0000000000 Binary files a/fixture/12/1/13 and /dev/null differ diff --git a/fixture/12/1/14 b/fixture/12/1/14 deleted file mode 100644 index 30200df856..0000000000 Binary files a/fixture/12/1/14 and /dev/null differ diff --git a/fixture/12/1/15 b/fixture/12/1/15 deleted file mode 100644 index 47f4892ad8..0000000000 Binary files a/fixture/12/1/15 and /dev/null differ diff --git a/fixture/12/1/16 b/fixture/12/1/16 deleted file mode 100644 index 893273da09..0000000000 --- a/fixture/12/1/16 +++ /dev/null @@ -1,3 +0,0 @@ -xoD*?7?"A3ɾI>?C:t,ݸ?*d?E?>>H씾dN ?x>(An4^_x3?9,?+ ?Q >Ӂ_/9m=\T9?=G9Į>DV @{8>X?U?ʿоXX>Y&?T>?w?= ? `?@^ -߽\|s?|׿i?t ?pee?ῠq_ -y?ǃI=E?s}v&̾V'>D[84?D␾"?N?2O?k|X?;9@@K?|R?Wf?8 \ No newline at end of file diff --git a/fixture/12/1/17 b/fixture/12/1/17 deleted file mode 100644 index 7b82f5d416..0000000000 --- a/fixture/12/1/17 +++ /dev/null @@ -1 +0,0 @@ -xo~?$V?4?4ugg?T?9#?~Gx?q>[lz???->?_!np?AQDT]?G7?WU@.>b)' >>׹>F?D)J>@kŕh>柱=J ?2d>e?#-?=,k > ;j1]>Ţ?*Y?%?9@>#?{>;>8+?' >3_>`f;?+F`ڿ0>yag?׿8v?+R?[=b_2ܿu>anQ2:?HR?r.&˾1,ſo?K?O?]>t޿@6>x >I෾?b ?`UE? #| \ No newline at end of file diff --git a/fixture/12/1/18 b/fixture/12/1/18 deleted file mode 100644 index ecc2687456..0000000000 --- a/fixture/12/1/18 +++ /dev/null @@ -1,3 +0,0 @@ -xo>_>M?J^s(>fvH>(?+??: v?F=>;? ʾ wF??+7 ->w?ۨ| -'?O?[o0a>"?;@f?5þ{˿?4?m~>E?1DqĔO?p?If?u?r7b=BûD>oX=j>F">@߱?0=e]>F?R\?OEr??Z?Q;-q>6;?i8p5?%Qȿ">s>#;=a= *& x??T׾Իǿ+WG.%?8ǽ1+{>ՓI@T?ֽ?5xn.CƳ=򊥿ꜾZ_?Y -j 3>O?35 ?w>oē6-?S?@r.q`?ſ?Oz>E3?/w%#>sZ|OzNŚ?μ> K*m_?<b2???BA}>XGŵc>>#@ I3K?>4-?'N>Q[??R>_8s,@8?'#?>'Nѿǿ̾ h⪾g?pɋәl(?%?> >vk>!Ƕ?g>G?J־{G+v,a?m \ No newline at end of file diff --git a/fixture/12/1/2 b/fixture/12/1/2 deleted file mode 100644 index 78f392aee5..0000000000 Binary files a/fixture/12/1/2 and /dev/null differ diff --git a/fixture/12/1/20 b/fixture/12/1/20 deleted file mode 100644 index 5fb1a1ee53..0000000000 Binary files a/fixture/12/1/20 and /dev/null differ diff --git a/fixture/12/1/21 b/fixture/12/1/21 deleted file mode 100644 index 6178369a2d..0000000000 Binary files a/fixture/12/1/21 and /dev/null differ diff --git a/fixture/12/1/22 b/fixture/12/1/22 deleted file mode 100644 index 04183b5b15..0000000000 --- a/fixture/12/1/22 +++ /dev/null @@ -1,2 +0,0 @@ -xo*N?i;-'q*W?n$>U"*2:I-#9?S A>>i;?؎? ÿ&t@.># ?apm?P?,lw=ATo?9=o>/ ?>ܰE?ݾ|d?m&?w?=]P?f?Ҿ> |3@o*? -@TWpA|< Ϳu2Ti}dE?ϛIf??mq>׿ M1|>NHz(7p؄ӾV>7>|?!>>X6ajӾ(:"*sXW?;?iCď???E :(>JZ>?t)W)A*w,T \ No newline at end of file diff --git a/fixture/12/1/23 b/fixture/12/1/23 deleted file mode 100644 index 85a99ec544..0000000000 Binary files a/fixture/12/1/23 and /dev/null differ diff --git a/fixture/12/1/24 b/fixture/12/1/24 deleted file mode 100644 index 342dff4eb5..0000000000 Binary files a/fixture/12/1/24 and /dev/null differ diff --git a/fixture/12/1/25 b/fixture/12/1/25 deleted file mode 100644 index 1b9f0f0359..0000000000 Binary files a/fixture/12/1/25 and /dev/null differ diff --git a/fixture/12/1/26 b/fixture/12/1/26 deleted file mode 100644 index f5301aec4e..0000000000 Binary files a/fixture/12/1/26 and /dev/null differ diff --git a/fixture/12/1/27 b/fixture/12/1/27 deleted file mode 100644 index 2105c15f4c..0000000000 Binary files a/fixture/12/1/27 and /dev/null differ diff --git a/fixture/12/1/28 b/fixture/12/1/28 deleted file mode 100644 index 92350f4083..0000000000 Binary files a/fixture/12/1/28 and /dev/null differ diff --git a/fixture/12/1/29 b/fixture/12/1/29 deleted file mode 100644 index d3cf090614..0000000000 --- a/fixture/12/1/29 +++ /dev/null @@ -1,3 +0,0 @@ -xo?mze?Du5[]?Ĝ ?Խ=5NR\6e>ŢXq?E?S/Ñ?Ҿ6Ռ?9?H?Q>D\?9 >顳?k>N?dUluJP?GV9?f?=&qq@=w>w] U?'>ׇ>?T,= ?ݶ>hJ?,B?vfڿ"E>Xk?WѾ @Vs f -?.l>4z&?l>=?nGC?s!}Z1 ->JTP?Q!?@?z>13?jf٢?6Kvb?h?nj@?$?Y|B@?s>Xž9E?v_dǞ \ No newline at end of file diff --git a/fixture/12/1/3 b/fixture/12/1/3 deleted file mode 100644 index c97825a497..0000000000 Binary files a/fixture/12/1/3 and /dev/null differ diff --git a/fixture/12/1/30 b/fixture/12/1/30 deleted file mode 100644 index 869d74e79b..0000000000 Binary files a/fixture/12/1/30 and /dev/null differ diff --git a/fixture/12/1/31 b/fixture/12/1/31 deleted file mode 100644 index 3a4b307c88..0000000000 Binary files a/fixture/12/1/31 and /dev/null differ diff --git a/fixture/12/1/32 b/fixture/12/1/32 deleted file mode 100644 index 7ef3bb1dee..0000000000 Binary files a/fixture/12/1/32 and /dev/null differ diff --git a/fixture/12/1/33 b/fixture/12/1/33 deleted file mode 100644 index 7e1a04fb63..0000000000 --- a/fixture/12/1/33 +++ /dev/null @@ -1,3 +0,0 @@ -xor>rp?L@4ZW(ޝ?\>z >rҖ?YB5>+>˦1=I?0 -=v=˿?FD^?(d 15]>q??T?W=JmZ?}E>+ ?K|=(W?")>j}?z -><=~zc=?3[ξi#ZϾԶ=8zuj"=?t?ͧ ? }Cde=p?ſ6z>6?ѾW=P?)͐q=k?ň?q?;n{E>PE?ֆ'Kw=^3=?!F?oJږ_?+7=ᐿF?C@˖&$sx \ No newline at end of file diff --git a/fixture/12/1/34 b/fixture/12/1/34 deleted file mode 100644 index ef6e3a2354..0000000000 Binary files a/fixture/12/1/34 and /dev/null differ diff --git a/fixture/12/1/35 b/fixture/12/1/35 deleted file mode 100644 index f4b39eafdb..0000000000 --- a/fixture/12/1/35 +++ /dev/null @@ -1,3 +0,0 @@ -xo>?g)? -@3/>`1^:@t?[Xc?qp+?T?V t? (Hk']p=%*xd?!?,h>7e+˿Y?%;VJ!,??/?eCbm>7>?f?owG @ig?>=wM>NKM>>G9*ѿUVMc?BʿA> 7F>IΰҼ隿aж?|ܿg_? >NJӺ?-C?>8mP?=[?9^ԡysI \ No newline at end of file diff --git a/fixture/12/1/36 b/fixture/12/1/36 deleted file mode 100644 index 3def4de46f..0000000000 Binary files a/fixture/12/1/36 and /dev/null differ diff --git a/fixture/12/1/37 b/fixture/12/1/37 deleted file mode 100644 index 32893fc5bc..0000000000 Binary files a/fixture/12/1/37 and /dev/null differ diff --git a/fixture/12/1/38 b/fixture/12/1/38 deleted file mode 100644 index caa67b1aba..0000000000 Binary files a/fixture/12/1/38 and /dev/null differ diff --git a/fixture/12/1/39 b/fixture/12/1/39 deleted file mode 100644 index c6ce184496..0000000000 --- a/fixture/12/1/39 +++ /dev/null @@ -1,3 +0,0 @@ -xooS"(_>cljP>Ț[__U -@^)뇽?ο4YuϿN&;$>\;L>@=6gS?ڕD>L=ұS>+=9?}7~>V.=`͏{=?- -ֿ(UNpt|>?JU??/?"f?V79?>&6w~iq?<,A?U:#?@㾅1t>x??YA?ӧ9P?H|=>?gM??猵V >\?Wm= ?0?n N6?޾St>G]?cD?|?0>#>. S:>t쏿9࿗}?SR \ No newline at end of file diff --git a/fixture/12/1/4 b/fixture/12/1/4 deleted file mode 100644 index 60cc292a07..0000000000 --- a/fixture/12/1/4 +++ /dev/null @@ -1,3 +0,0 @@ -xoc3? 7?IPޔ?J -=0ÿ8B?=q@}?.=B?C?=Ӿ-d>?~Q?X]t;\?ct?W[`*.?r>e?)ؒ?kq?y4?#IͿ??ٝ?LA? %>ö>}f?X?-=?u>ʯZ>62PtX>`? uq?~֝aQ@? -?H?`?f"?y>P?*j{?\qrQ+? ۿsS!?;{DOڿ"?^ݍ=SRE?eA}?Fq]pnEG೿G&>G2j \ No newline at end of file diff --git a/fixture/12/1/40 b/fixture/12/1/40 deleted file mode 100644 index eca2cc90c3..0000000000 Binary files a/fixture/12/1/40 and /dev/null differ diff --git a/fixture/12/1/41 b/fixture/12/1/41 deleted file mode 100644 index 9300eb406a..0000000000 --- a/fixture/12/1/41 +++ /dev/null @@ -1,3 +0,0 @@ -xoU/q>p/x \?.?X=_w -%~>K ? F>`?՚g?R?:2 ?KR9?&> P>Ra(u=G0x?^t?W`k?q??>{J,¿?]eS+>*ʾaR]?>4?t? HFg_??сx:?щ?h*?>Ó>HAֻ*Fc@*n >'?$&I7$D?J@p>W -ѾxyB?=53nJ?Td]7?wL8fH?l?D@<{>>X>>3@.R \ No newline at end of file diff --git a/fixture/12/1/42 b/fixture/12/1/42 deleted file mode 100644 index f2dca581f1..0000000000 Binary files a/fixture/12/1/42 and /dev/null differ diff --git a/fixture/12/1/43 b/fixture/12/1/43 deleted file mode 100644 index 8ba2dd4a3a..0000000000 Binary files a/fixture/12/1/43 and /dev/null differ diff --git a/fixture/12/1/44 b/fixture/12/1/44 deleted file mode 100644 index ba549052b4..0000000000 Binary files a/fixture/12/1/44 and /dev/null differ diff --git a/fixture/12/1/5 b/fixture/12/1/5 deleted file mode 100644 index a96d45d4f4..0000000000 Binary files a/fixture/12/1/5 and /dev/null differ diff --git a/fixture/12/1/6 b/fixture/12/1/6 deleted file mode 100644 index c8b84989a6..0000000000 Binary files a/fixture/12/1/6 and /dev/null differ diff --git a/fixture/12/1/7 b/fixture/12/1/7 deleted file mode 100644 index d4015923be..0000000000 Binary files a/fixture/12/1/7 and /dev/null differ diff --git a/fixture/12/1/8 b/fixture/12/1/8 deleted file mode 100644 index e02434fc4b..0000000000 Binary files a/fixture/12/1/8 and /dev/null differ diff --git a/fixture/12/1/9 b/fixture/12/1/9 deleted file mode 100644 index 4356b3c24f..0000000000 Binary files a/fixture/12/1/9 and /dev/null differ diff --git a/fixture/12/2/.zarray b/fixture/12/2/.zarray deleted file mode 100644 index 7d45c627eb..0000000000 --- a/fixture/12/2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "O.=Gu>>f$?zotԗoS \ No newline at end of file diff --git a/fixture/15/1/1 b/fixture/15/1/1 deleted file mode 100644 index 45aae56240..0000000000 --- a/fixture/15/1/1 +++ /dev/null @@ -1,2 +0,0 @@ -x -@U@Rƅai_۶f^3yμ}y3W9xNK>}^sߖokw~g켄S|o7}'&gULԇ}{:=˧_s*{폞[=ܾK \ No newline at end of file diff --git a/fixture/15/1/10 b/fixture/15/1/10 deleted file mode 100644 index 7437e5dfad..0000000000 --- a/fixture/15/1/10 +++ /dev/null @@ -1,2 +0,0 @@ -xA -0 ki S0iW[kqg#og=yg8O>WfcfYoջ=v{_?_J_w?eg|~7}`泾bewpw?3zzI \ No newline at end of file diff --git a/fixture/15/1/11 b/fixture/15/1/11 deleted file mode 100644 index a5b9b2c595..0000000000 --- a/fixture/15/1/11 +++ /dev/null @@ -1,2 +0,0 @@ -xA -P U @_(24I|./5G|֛֚y՟w}}ywM}w7ӏ{.O_p'Oz}O{ͩ7fɇ7;0ĩ_fѧ?( \ No newline at end of file diff --git a/fixture/15/1/12 b/fixture/15/1/12 deleted file mode 100644 index d00873aed2..0000000000 --- a/fixture/15/1/12 +++ /dev/null @@ -1,2 +0,0 @@ -x -0 Ca`>5+ȒeqzWR ok?zՉ^w7o~XO>u_3߼1f>3|g1N;O^3vxֻ?'y=R3ĻLO>- \ No newline at end of file diff --git a/fixture/15/1/13 b/fixture/15/1/13 deleted file mode 100644 index d4f2a83fa1..0000000000 --- a/fixture/15/1/13 +++ /dev/null @@ -1,2 +0,0 @@ -xA -0 kiPʐ Dk+'s^zT}c~Ogտ<ɳf:=ɇ~s>~owyֻޜwߓ>zya;n~irK~Oo=uz7>ߪ \ No newline at end of file diff --git a/fixture/15/1/14 b/fixture/15/1/14 deleted file mode 100644 index e88b8c90b8..0000000000 --- a/fixture/15/1/14 +++ /dev/null @@ -1,2 +0,0 @@ -x -@W_뻂2eȐ$sVgͣ]?kY__OOgw7$}w}ykݜo?}fys_sʟկs?ۗy'v}};<#_k׈ \ No newline at end of file diff --git a/fixture/15/1/15 b/fixture/15/1/15 deleted file mode 100644 index b349a1b7bb..0000000000 --- a/fixture/15/1/15 +++ /dev/null @@ -1,2 +0,0 @@ -x -0 ܁ YZnҹ6U^ֳ꺽?o.k[<=&{,Wǜ浞?3wϋI~3~7֛Ǐ|}g?[|7[oN?C{IS?Gy4/^᫾gS3Ss~~Iǟ{;>t8;~oy>wXOl籎_ʇn?^:7|zw?~;;}GyҼu3?z?iwȉ \ No newline at end of file diff --git a/fixture/15/1/17 b/fixture/15/1/17 deleted file mode 100644 index c1dbc1684e..0000000000 --- a/fixture/15/1/17 +++ /dev/null @@ -1,2 +0,0 @@ -xA -@ zث산z3`4!;1VsSs<3^3%23W~Oi^~}I}^wuhWu~{~<~;N#Ͼ_yUe{ɛo'ͽgۿ݉= \ No newline at end of file diff --git a/fixture/15/1/18 b/fixture/15/1/18 deleted file mode 100644 index fee75855f0..0000000000 --- a/fixture/15/1/18 +++ /dev/null @@ -1,2 +0,0 @@ -xA -Az*+ŭKA * R$tnkq^󿉯×99_[7M8{R_o=1S6u5/ݷߜާy#,{3{6}`색N \ No newline at end of file diff --git a/fixture/15/1/19 b/fixture/15/1/19 deleted file mode 100644 index e15f65db82..0000000000 --- a/fixture/15/1/19 +++ /dev/null @@ -1,2 +0,0 @@ -x -0 !גB +C}he̷ּx9A=0ѧ?g}Ǔi߿y8'I9/~0|>wNO߄o;5{=Oti?/0. \ No newline at end of file diff --git a/fixture/15/1/2 b/fixture/15/1/2 deleted file mode 100644 index 2d656020c8..0000000000 --- a/fixture/15/1/2 +++ /dev/null @@ -1,4 +0,0 @@ -xA -@zث(˾~[@ - -3:1:[\S^lZ{nNy;y}|3oNy_0w^b䥾w~֛۷>3vzyf7~z?s>ڟ3g9Os@ \ No newline at end of file diff --git a/fixture/15/1/20 b/fixture/15/1/20 deleted file mode 100644 index 773b02aea9..0000000000 --- a/fixture/15/1/20 +++ /dev/null @@ -1,3 +0,0 @@ -x -@ CWU\X ->XŁ2I.c5EE}uѯgo7Jփ߼+=x.&__G;qG>{oc뛷gY^oC=o j \ No newline at end of file diff --git a/fixture/15/1/21 b/fixture/15/1/21 deleted file mode 100644 index 7bd6a0e130..0000000000 Binary files a/fixture/15/1/21 and /dev/null differ diff --git a/fixture/15/1/22 b/fixture/15/1/22 deleted file mode 100644 index 821c481474..0000000000 Binary files a/fixture/15/1/22 and /dev/null differ diff --git a/fixture/15/1/23 b/fixture/15/1/23 deleted file mode 100644 index 6a680d1e8d..0000000000 --- a/fixture/15/1/23 +++ /dev/null @@ -1,2 +0,0 @@ -xA -0kIP!s"%i%sηgͣ標gy3y{u}o}{e^OzSYߪz}џg' \ No newline at end of file diff --git a/fixture/15/1/24 b/fixture/15/1/24 deleted file mode 100644 index 333ff4b1dc..0000000000 --- a/fixture/15/1/24 +++ /dev/null @@ -1,2 +0,0 @@ -xA -@ !BB€Yt!Ym:ۢs7+y;kϳ7ow~c3?O_幫}C凷|w1~W=~=g~gUy3*>}W|5_? \ No newline at end of file diff --git a/fixture/15/1/25 b/fixture/15/1/25 deleted file mode 100644 index 0c81e7e043..0000000000 --- a/fixture/15/1/25 +++ /dev/null @@ -1,2 +0,0 @@ -x -P ߯CbA(F\]B(kٍ.cugߪWC?>K6T4?ۿ||[};Oy:Nwtw~O~yk|KKߧ \ No newline at end of file diff --git a/fixture/15/1/26 b/fixture/15/1/26 deleted file mode 100644 index dba3814a87..0000000000 --- a/fixture/15/1/26 +++ /dev/null @@ -1,3 +0,0 @@ -x -1 ᾪ{* -s 9M23c[իj~ϪY?|ٜVOWyyd篲y«o}9߱S|_sk|՗'>]_=ټrsx7o?|+}|y;=7Q \ No newline at end of file diff --git a/fixture/15/1/27 b/fixture/15/1/27 deleted file mode 100644 index 5b7a815277..0000000000 --- a/fixture/15/1/27 +++ /dev/null @@ -1,2 +0,0 @@ -xA -@D\UnEAO OM(w%eYrs[>111'}}/7?o?^ߣ~~߼Z:ޏ__~U{<7ׯo>m__yۧ幗~z|^^}v߼:Vy? \ No newline at end of file diff --git a/fixture/15/1/28 b/fixture/15/1/28 deleted file mode 100644 index 92c14290e9..0000000000 --- a/fixture/15/1/28 +++ /dev/null @@ -1,2 +0,0 @@ -x -@CWЫ(;Jʃ, KLmugu?ooO>om?;>3ny'go \ No newline at end of file diff --git a/fixture/15/1/29 b/fixture/15/1/29 deleted file mode 100644 index dacc0d449f..0000000000 --- a/fixture/15/1/29 +++ /dev/null @@ -1,2 +0,0 @@ -x -P ߯CłPd`E C6cmOGջj>Ϫ^'ԳV?[/_~0{_Ol=l]yf%w={3>3~;o?ϛ<͞~>#qCO?}7ɳo}c_ȇS~K \ No newline at end of file diff --git a/fixture/15/1/3 b/fixture/15/1/3 deleted file mode 100644 index c86f01f657..0000000000 Binary files a/fixture/15/1/3 and /dev/null differ diff --git a/fixture/15/1/30 b/fixture/15/1/30 deleted file mode 100644 index b070f41e8c..0000000000 --- a/fixture/15/1/30 +++ /dev/null @@ -1,2 +0,0 @@ -x -@ DW_Kˁ0sm^YOPO[ifϔq}u}oLo1׸g?{G%ݏ|捭}Ic>at~ħߘ|^ѧ~|s_: \ No newline at end of file diff --git a/fixture/15/1/31 b/fixture/15/1/31 deleted file mode 100644 index 18c9fd3e32..0000000000 --- a/fixture/15/1/31 +++ /dev/null @@ -1,2 +0,0 @@ -x -@CWЫT(|Pٺ0,!L"cfZ_{菋/c,Sϼ$޻z9yN]yӯ=a>ç}ǻ1g{덭7Nپ߿wGoL=Лw_g{GO߮wywy~O^)?iy~?yY'=ylz>ֳ>YT \ No newline at end of file diff --git a/fixture/15/1/33 b/fixture/15/1/33 deleted file mode 100644 index b1d98acbc8..0000000000 --- a/fixture/15/1/33 +++ /dev/null @@ -1,2 +0,0 @@ -xA -@ ͡ג@uh2`I"$˲1OKգ[ozunͻ8qϾ<~v>Yyk7~ zgy<'xv>c1>w.~c7&{ycؓ~x>?oϞo=<}U}IC \ No newline at end of file diff --git a/fixture/15/1/35 b/fixture/15/1/35 deleted file mode 100644 index 393382fadc..0000000000 --- a/fixture/15/1/35 +++ /dev/null @@ -1,2 +0,0 @@ -x -@ W탯҂ ~92+61ƣ[3f1U>sto~Vy=tǿobͩO?X~][7ӗuO^OovUSu9}N~1'u3X=t|t`?pej \ No newline at end of file diff --git a/fixture/15/1/36 b/fixture/15/1/36 deleted file mode 100644 index 1dea3a54bd..0000000000 --- a/fixture/15/1/36 +++ /dev/null @@ -1,2 +0,0 @@ -x -P ߯bA(s\RBc6[Kէj~[~7=Qϣг_O}cO}[٧eN~yz3ܓΟy/}·<}ל~wױwz U \ No newline at end of file diff --git a/fixture/15/1/37 b/fixture/15/1/37 deleted file mode 100644 index 4a4f90a683..0000000000 --- a/fixture/15/1/37 +++ /dev/null @@ -1,3 +0,0 @@ -xA -@ a@eK&ɒs~c - o5z'=/<뷚~߼O`Mw^w';`ޯo3N8}?}ۯoL>~·Gx3N|Ӈ7 @~ \ No newline at end of file diff --git a/fixture/15/1/38 b/fixture/15/1/38 deleted file mode 100644 index 5e55c34981..0000000000 --- a/fixture/15/1/38 +++ /dev/null @@ -1,2 +0,0 @@ -x -P ߯C HT:.0dټiqzVo:fݷZGNz};c<~]fϛw5o#|f}>~f{__wy`ֻoN.)Γ>"fo>1y| \ No newline at end of file diff --git a/fixture/15/1/39 b/fixture/15/1/39 deleted file mode 100644 index b80e96596d..0000000000 --- a/fixture/15/1/39 +++ /dev/null @@ -1,2 +0,0 @@ -x -0 CۡAa精cMW0A,:؎zyU=±8jVʷs5gov>o{_qG`u|s>cn}w{O|Ky>a9i>Y~_}Ƀ?*k \ No newline at end of file diff --git a/fixture/15/1/4 b/fixture/15/1/4 deleted file mode 100644 index 9f2b544818..0000000000 Binary files a/fixture/15/1/4 and /dev/null differ diff --git a/fixture/15/1/40 b/fixture/15/1/40 deleted file mode 100644 index 65e84b7941..0000000000 --- a/fixture/15/1/40 +++ /dev/null @@ -1,2 +0,0 @@ -x -@ DWEP͂@U8-u5[kx+9{?WxOyo|{=C?c?}wNu>0``{o?IO?&4p~.o=}]?[^O7o~+3'~w_c%>Yo87o<ߟ~ѓN|޼/ \ No newline at end of file diff --git a/fixture/15/1/42 b/fixture/15/1/42 deleted file mode 100644 index 1ffee1d827..0000000000 Binary files a/fixture/15/1/42 and /dev/null differ diff --git a/fixture/15/1/43 b/fixture/15/1/43 deleted file mode 100644 index 66a592c592..0000000000 --- a/fixture/15/1/43 +++ /dev/null @@ -1,2 +0,0 @@ -x -@ W냯EA9p@@B2\5j{]j~foZ=O?<3~1s/ٻoғ也;_{O^~}仟o|1[Oy/]~g=y<߇yS_aΏ}ͩ~f8 \ No newline at end of file diff --git a/fixture/15/1/44 b/fixture/15/1/44 deleted file mode 100644 index 39caa64600..0000000000 --- a/fixture/15/1/44 +++ /dev/null @@ -1,2 +0,0 @@ -x -@ W탯 _o:PB(f7k[իjG?}og^4ywqV47sOoL4뷞xz~Wa켎s=/ov^ӟu=t`w_[O|7~OG&wUg~pt_c&?|tKy=~j}y].~'i޺y=ofNN/Ӽu3y]//n \ No newline at end of file diff --git a/fixture/15/1/48 b/fixture/15/1/48 deleted file mode 100644 index b57b31cdfa..0000000000 --- a/fixture/15/1/48 +++ /dev/null @@ -1,2 +0,0 @@ -x -0 烯rpXb A&ۮcS_Pgz]u7[o>~3~w^ݷl=y|o>Þ~9!ggI$=y;|sOix?}}~gͷb~Ï \ No newline at end of file diff --git a/fixture/15/1/49 b/fixture/15/1/49 deleted file mode 100644 index 283c33c31f..0000000000 --- a/fixture/15/1/49 +++ /dev/null @@ -1,2 +0,0 @@ -x -@ kQį7e dW*,˘]#bϑ*^?+ʳ}uϟ<*Ư;a,;ԫ}g\?7[?OGf֭yp/uez<.~֛wl?](W|Ǐ+7_K \ No newline at end of file diff --git a/fixture/15/1/5 b/fixture/15/1/5 deleted file mode 100644 index 0b390ec3a2..0000000000 --- a/fixture/15/1/5 +++ /dev/null @@ -1,2 +0,0 @@ -x -@CWP< °$]1:5ck;·WKxx񻊝><]_x'0:zc}=<}}7f~cIO~~7/Gq|֏}Gold` \ No newline at end of file diff --git a/fixture/15/1/50 b/fixture/15/1/50 deleted file mode 100644 index 96ba17b620..0000000000 --- a/fixture/15/1/50 +++ /dev/null @@ -1,2 +0,0 @@ -x -@CW(; }WII2cj^5yּz8~.M$_O<|cϼg}IwUw'ol`? ?w_>+U>ѫ̳/Y~w}t=~ͳw7'yG}wʗ<)\_߽?oɯ =g}.0zyUafyons_'~C^ϛu>뙻|3~׷>ٿ?Wysʟ|c}_e{uy胈 \ No newline at end of file diff --git a/fixture/15/1/53 b/fixture/15/1/53 deleted file mode 100644 index bf0e8df5d1..0000000000 Binary files a/fixture/15/1/53 and /dev/null differ diff --git a/fixture/15/1/54 b/fixture/15/1/54 deleted file mode 100644 index d8d820fd2f..0000000000 Binary files a/fixture/15/1/54 and /dev/null differ diff --git a/fixture/15/1/55 b/fixture/15/1/55 deleted file mode 100644 index f2d7c6fc55..0000000000 --- a/fixture/15/1/55 +++ /dev/null @@ -1,2 +0,0 @@ -xA -0 $RA_o.@Y&TadKDl5sR#kg8tW?Y_abO[z1ds멋5scӽGut UKܹ_8Uv \ No newline at end of file diff --git a/fixture/15/1/6 b/fixture/15/1/6 deleted file mode 100644 index 1395b624a7..0000000000 --- a/fixture/15/1/6 +++ /dev/null @@ -1,2 +0,0 @@ -x -0 C!ױA>({-H1wrzqb񌳳_j?y[ol=8{?K?ݾwӧ/gy޿g2?}>]߼~k}Yo\3O_L^G~g; \ No newline at end of file diff --git a/fixture/15/1/7 b/fixture/15/1/7 deleted file mode 100644 index d62802524b..0000000000 Binary files a/fixture/15/1/7 and /dev/null differ diff --git a/fixture/15/1/8 b/fixture/15/1/8 deleted file mode 100644 index 2a8fc693ab..0000000000 --- a/fixture/15/1/8 +++ /dev/null @@ -1,2 +0,0 @@ -xA -@ az#X MT!HN3ST߷wJtc>'s=_3O~<<֛g=us[&pҧއ[}|_ӏ~s]/Gz)/y7_z&/џX߽~'~}~ڇ|;S \ No newline at end of file diff --git a/fixture/15/2/.zarray b/fixture/15/2/.zarray deleted file mode 100644 index eec9c32ce1..0000000000 --- a/fixture/15/2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "^==Ϋ:z}ɡ9W]>듯<7Oe_o =7"+`uwn}#G7O!~b \ No newline at end of file diff --git a/fixture/16/1/1 b/fixture/16/1/1 deleted file mode 100644 index aa9e976823..0000000000 Binary files a/fixture/16/1/1 and /dev/null differ diff --git a/fixture/16/1/10 b/fixture/16/1/10 deleted file mode 100644 index 0fe5bdce9e..0000000000 --- a/fixture/16/1/10 +++ /dev/null @@ -1 +0,0 @@ -x 0DQ1}L)$ԓS.T^|=QE.e6L6.]Svmc7-E>S'R>5Mu^o8ʟ}9e>k{'<5q4y=orΏ9Nȟ7 \ No newline at end of file diff --git a/fixture/16/1/11 b/fixture/16/1/11 deleted file mode 100644 index d5662c3206..0000000000 --- a/fixture/16/1/11 +++ /dev/null @@ -1,4 +0,0 @@ -x} FQ -qi-` -zzf -y1t_%Ro2ukԛ;uWdy~/uy.%{%s>9_)̏z$t/hz:!;׽&US \ No newline at end of file diff --git a/fixture/16/1/12 b/fixture/16/1/12 deleted file mode 100644 index b9f7cd8a97..0000000000 --- a/fixture/16/1/12 +++ /dev/null @@ -1 +0,0 @@ -xuҽ 0 BA;0 KLO;Ddt?D޶3uh<_8<ϗ eO3}\]Gt}&>yz͏Χ*_aeOr}tfoί\9}0}9=kS \ No newline at end of file diff --git a/fixture/16/1/13 b/fixture/16/1/13 deleted file mode 100644 index e632b0bed1..0000000000 --- a/fixture/16/1/13 +++ /dev/null @@ -1 +0,0 @@ -xu0 @“>BzxzX<$ܜtwukyΗqWSrq z+B9W}Ν?_Gd[C}aGT.Ǿ8>~rswV%y u齽Ԯ \ No newline at end of file diff --git a/fixture/16/1/14 b/fixture/16/1/14 deleted file mode 100644 index 5818cddc89..0000000000 --- a/fixture/16/1/14 +++ /dev/null @@ -1 +0,0 @@ -xu 0D-cRԓS.T^| 7Y i4?ӺL64}ёC]ܫ'sxUN\悔tDo ڠUG>Qyz8y< \ No newline at end of file diff --git a/fixture/16/1/15 b/fixture/16/1/15 deleted file mode 100644 index 7a453d6a7f..0000000000 --- a/fixture/16/1/15 +++ /dev/null @@ -1,2 +0,0 @@ -x 0+'}M) -C'!]Y.vzk=n˃x ٧n<y^-Y'1Z̃:ck|H澾w'G>tׇZ=3|$e^=ۿ#_Ӭݖ \ No newline at end of file diff --git a/fixture/16/1/16 b/fixture/16/1/16 deleted file mode 100644 index 1ea850689c..0000000000 --- a/fixture/16/1/16 +++ /dev/null @@ -1,2 +0,0 @@ -xu 0+'}MBzʇ -O"`Ivowu1{{lV9y_^<џ2熽zpNo^{p|j^|p}rt|YГt(C=rۛ9f}#7B2 \ No newline at end of file diff --git a/fixture/16/1/17 b/fixture/16/1/17 deleted file mode 100644 index 03851a6fc8..0000000000 Binary files a/fixture/16/1/17 and /dev/null differ diff --git a/fixture/16/1/18 b/fixture/16/1/18 deleted file mode 100644 index b39626a2f1..0000000000 --- a/fixture/16/1/18 +++ /dev/null @@ -1 +0,0 @@ -x 0+$;T'|RAz5|e:lPώ߾zSnpS λw|3t݃@h|E~п \ No newline at end of file diff --git a/fixture/16/1/23 b/fixture/16/1/23 deleted file mode 100644 index aeaf1a1efc..0000000000 --- a/fixture/16/1/23 +++ /dev/null @@ -1 +0,0 @@ -x}DP dp@D 8E#2e)3=F馦0Ϋ/ׯ?˰sO#٧s._~EΡ#rhӧQ9x#ʡ篹94qJDELJw^9Z>A(gZ \ No newline at end of file diff --git a/fixture/16/1/24 b/fixture/16/1/24 deleted file mode 100644 index c363425923..0000000000 --- a/fixture/16/1/24 +++ /dev/null @@ -1,2 +0,0 @@ -x} @ @“>xS@ -z3drtʻ6'y0~'NJxᵜ97ΑAynsۛ}P?\uѡ.5_?Û}ڟ}__k9x>L1 \ No newline at end of file diff --git a/fixture/16/1/25 b/fixture/16/1/25 deleted file mode 100644 index af7024af3f..0000000000 --- a/fixture/16/1/25 +++ /dev/null @@ -1,3 +0,0 @@ -x 0-s -H!@=rqEǘg D;i[mYH7ykE?t=}üGOڡ@◟͡d~r ^(z\.=+Nsw1 /# -2ow` \ No newline at end of file diff --git a/fixture/16/1/26 b/fixture/16/1/26 deleted file mode 100644 index 6457124755..0000000000 Binary files a/fixture/16/1/26 and /dev/null differ diff --git a/fixture/16/1/27 b/fixture/16/1/27 deleted file mode 100644 index 4324703a1b..0000000000 Binary files a/fixture/16/1/27 and /dev/null differ diff --git a/fixture/16/1/28 b/fixture/16/1/28 deleted file mode 100644 index 4f61f16a26..0000000000 Binary files a/fixture/16/1/28 and /dev/null differ diff --git a/fixture/16/1/29 b/fixture/16/1/29 deleted file mode 100644 index 3b378b05ed..0000000000 --- a/fixture/16/1/29 +++ /dev/null @@ -1 +0,0 @@ -xu 0-#}LBz8qz(i:Xx/zn_;:Qԫ?G;nz#둇z#pš^|ݯ=rݧ+{&U_+>\s~84v6 \ No newline at end of file diff --git a/fixture/16/1/3 b/fixture/16/1/3 deleted file mode 100644 index ee4392ca9c..0000000000 --- a/fixture/16/1/3 +++ /dev/null @@ -1 +0,0 @@ -xϱ PPI A2߁yd >BXپcǩYy-ԟ7Z\rPpx-w_Cy/}7s?~|k'K?n.O^і \ No newline at end of file diff --git a/fixture/16/1/30 b/fixture/16/1/30 deleted file mode 100644 index 6badc05fd4..0000000000 --- a/fixture/16/1/30 +++ /dev/null @@ -1 +0,0 @@ -xu 0-#}LBz8qz(S@,KT}76|Y]n\mw43~s{~wn\.?}8]_"]/.Ϗ7~ByOzroa6Oǧ;!Oyg?6޼>x93=7t" \ No newline at end of file diff --git a/fixture/16/1/32 b/fixture/16/1/32 deleted file mode 100644 index 4678297ce9..0000000000 Binary files a/fixture/16/1/32 and /dev/null differ diff --git a/fixture/16/1/33 b/fixture/16/1/33 deleted file mode 100644 index 7a6c03db41..0000000000 --- a/fixture/16/1/33 +++ /dev/null @@ -1 +0,0 @@ -xұ @D-}S zp=H~|:i4;ne|:y[a_=[ǡx/Oy|s49w5y'w7We?>Kc \ No newline at end of file diff --git a/fixture/16/1/34 b/fixture/16/1/34 deleted file mode 100644 index e25a1b5526..0000000000 --- a/fixture/16/1/34 +++ /dev/null @@ -1,2 +0,0 @@ -xѽ P `BA ;0 KJBOBDO:}?Ry/<<.M}e -ï.߼1z!qs<{9'C:=<鳇˞]?tHO>skn/8l \ No newline at end of file diff --git a/fixture/16/1/35 b/fixture/16/1/35 deleted file mode 100644 index 407d63ba1b..0000000000 Binary files a/fixture/16/1/35 and /dev/null differ diff --git a/fixture/16/1/36 b/fixture/16/1/36 deleted file mode 100644 index 5d993a5be6..0000000000 --- a/fixture/16/1/36 +++ /dev/null @@ -1 +0,0 @@ -x 0DQ‘>rBzrBXGQ٬S8׽ez\o>}o=VqX9֭7S<|5tgezg|OWrT~jP9z>U|nM2 \ No newline at end of file diff --git a/fixture/16/1/37 b/fixture/16/1/37 deleted file mode 100644 index b053686bf5..0000000000 --- a/fixture/16/1/37 +++ /dev/null @@ -1 +0,0 @@ -x}б 0$p2H&ǕL]TppG@OeӲN><0qj[E5뷽7s_G|C8CrG5{>;>:\λU._{Qo? \ No newline at end of file diff --git a/fixture/16/1/38 b/fixture/16/1/38 deleted file mode 100644 index b7e11c34a3..0000000000 Binary files a/fixture/16/1/38 and /dev/null differ diff --git a/fixture/16/1/39 b/fixture/16/1/39 deleted file mode 100644 index 4af7763708..0000000000 --- a/fixture/16/1/39 +++ /dev/null @@ -1 +0,0 @@ -xu 1 “>MBzxzɓEe (׻\ky^|zOOA:Xu rF>=Odx=r=r^k7:}:?}H}:T/xO~:'t9Tٹ \ No newline at end of file diff --git a/fixture/16/1/4 b/fixture/16/1/4 deleted file mode 100644 index 5b4dd4281a..0000000000 --- a/fixture/16/1/4 +++ /dev/null @@ -1 +0,0 @@ -xu 0PcRH*pԓS.T@/>ƀ#aY^gg-C)ju/x,|ZzZx~XnVvS<%fg.>|/uG|%ĽU~|Uz,/8~<,<: \ No newline at end of file diff --git a/fixture/16/1/40 b/fixture/16/1/40 deleted file mode 100644 index 27acb8204e..0000000000 --- a/fixture/16/1/40 +++ /dev/null @@ -1 +0,0 @@ -xeһ@ aB>)P\h`nYꂋ8޾i?xڇxg?X׼ohKT9:_|΋s}u#qOOU\o#G=OQx7y~8y{ \ No newline at end of file diff --git a/fixture/16/1/41 b/fixture/16/1/41 deleted file mode 100644 index 12617e423b..0000000000 Binary files a/fixture/16/1/41 and /dev/null differ diff --git a/fixture/16/1/42 b/fixture/16/1/42 deleted file mode 100644 index cf30a62450..0000000000 --- a/fixture/16/1/42 +++ /dev/null @@ -1 +0,0 @@ -x 0@gRH*pԓW> 4X>!ݽ;ԎY6Z/V/G=w,y)08]P.A>/L}v}SoĒߏcz~^u!whw \ No newline at end of file diff --git a/fixture/16/1/43 b/fixture/16/1/43 deleted file mode 100644 index 25dbc980f3..0000000000 Binary files a/fixture/16/1/43 and /dev/null differ diff --git a/fixture/16/1/44 b/fixture/16/1/44 deleted file mode 100644 index 918b16ea53..0000000000 --- a/fixture/16/1/44 +++ /dev/null @@ -1,2 +0,0 @@ -xα @ DQBHpP@=]Bˆ i4_8oiCakS}{Ww28n3ok뙷k>9r=zϹO\q9GyD>{h -Z@ \ No newline at end of file diff --git a/fixture/16/1/45 b/fixture/16/1/45 deleted file mode 100644 index e92de042e6..0000000000 --- a/fixture/16/1/45 +++ /dev/null @@ -1 +0,0 @@ -xuл 0`BA;0O4L..|D>ɑO)yo]߶tռgV=7]fO[Ïx3紗HWƽR|n=r3_0{$o7o~bg}Og \ No newline at end of file diff --git a/fixture/16/1/46 b/fixture/16/1/46 deleted file mode 100644 index a6aacbe767..0000000000 Binary files a/fixture/16/1/46 and /dev/null differ diff --git a/fixture/16/1/47 b/fixture/16/1/47 deleted file mode 100644 index e9d78090a7..0000000000 --- a/fixture/16/1/47 +++ /dev/null @@ -1,2 +0,0 @@ -x] P -}RH*דS. p~B Bg܎s;>{u]r|}[SlUN"?į9s_\ujBJ9ï>>>nO\!gΥ9sQ'>]ssR'ʃr \ No newline at end of file diff --git a/fixture/16/1/48 b/fixture/16/1/48 deleted file mode 100644 index 89e635c152..0000000000 --- a/fixture/16/1/48 +++ /dev/null @@ -1 +0,0 @@ -xα @ @ 2{|2''UO]\ư}xHyIQu՞xO-h_9sqyr*[uy.Ϟ~Hכzo9|9'ΧQzWkz_'>wG_\ \ No newline at end of file diff --git a/fixture/16/1/49 b/fixture/16/1/49 deleted file mode 100644 index 15cbd1bb4d..0000000000 Binary files a/fixture/16/1/49 and /dev/null differ diff --git a/fixture/16/1/5 b/fixture/16/1/5 deleted file mode 100644 index 4d42627087..0000000000 Binary files a/fixture/16/1/5 and /dev/null differ diff --git a/fixture/16/1/50 b/fixture/16/1/50 deleted file mode 100644 index c3e5fdf72d..0000000000 Binary files a/fixture/16/1/50 and /dev/null differ diff --git a/fixture/16/1/51 b/fixture/16/1/51 deleted file mode 100644 index 6cbc95f96a..0000000000 --- a/fixture/16/1/51 +++ /dev/null @@ -1 +0,0 @@ -x 0-#}LB!pB#&(nfOg˘m6}G<{3#ǯ̏W:CzssO(>~g<7߃੏P[/}: \ No newline at end of file diff --git a/fixture/16/1/52 b/fixture/16/1/52 deleted file mode 100644 index 20e869ca1c..0000000000 --- a/fixture/16/1/52 +++ /dev/null @@ -1,2 +0,0 @@ -x} 0@3}N) -C+*H/~ŒXnvaٷe^kXjq:ڭ۽_Uu΍yι|)7y<x:5+:y5~g[c=GOkZ~}r/KN9rBz8qz#"&|~03G> -C}_d^>&uͳGD3ܪE-̃g; {>^j*\}O|/%都_s$*v?L_t}_< \ No newline at end of file diff --git a/fixture/16/1/55 b/fixture/16/1/55 deleted file mode 100644 index be5892bd8b..0000000000 --- a/fixture/16/1/55 +++ /dev/null @@ -1,2 +0,0 @@ -xб @D->-B -PQr f|fgZlXǝ<#+x()zɗ}P^r=y?/Sѯ9 \ No newline at end of file diff --git a/fixture/16/1/8 b/fixture/16/1/8 deleted file mode 100644 index b6b547a685..0000000000 Binary files a/fixture/16/1/8 and /dev/null differ diff --git a/fixture/16/1/9 b/fixture/16/1/9 deleted file mode 100644 index 4cc02933ea..0000000000 --- a/fixture/16/1/9 +++ /dev/null @@ -1,3 +0,0 @@ -xu @ P‘>L) -ą -e$S*"ƞeٞWΗ}4{FǫtoϾJFur3Jo>>N:{"~ڧ3N;p/11ܶ \ No newline at end of file diff --git a/fixture/16/2/.zarray b/fixture/16/2/.zarray deleted file mode 100644 index c4bed6fbc7..0000000000 --- a/fixture/16/2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "fl`#DEsIgLu2L((eʩ<߭ tiI/eyCF -!!!4"TX^!Lb2죄#|I)'8)bhGxx:Ӆd3d*'j &d0^"ьa,l|N18jG4mxXLd%,Tr&MOՋ$ a(F -Ƨ=쥡lD8x(Z 0ILf:bsyy$82sΆ ]oI??+Yj76OFp}ҐO3F3LLdo0C'8˷}@'~Eg,=Xu|znrԾ7`Q a(x^c^Oq9Nцvt):1"e1\{pqF&y31d[mf?8HphF+5myvL 3?3,dgs\\׹Aу~ dYGg g+;.9ӀH)1y LLc18g7gE.qƯNoЗ| -Wֱl!ȁs' A"HF')Le?8H_qrNsiG;x$ XB,'5׹A.a dIeF82NvQ짔sv"GhKJx@;"10qd2d&f>9s3T*$FiG7x$3YZ -M"BkG4&(M:`2 fE)G)PN$>Z;$ғ^$2X͇䳁PCvPP҈pRNcyt&2%$ю8:8t ]f>9,=rY*VSUOJ aD*X -N{bq#DҎh1:0d1f1KXJ -?PMnqjGyd0al`#lS -3A \ No newline at end of file diff --git a/fixture/20/1/0.0.2 b/fixture/20/1/0.0.2 deleted file mode 100644 index 1b7df776b6..0000000000 Binary files a/fixture/20/1/0.0.2 and /dev/null differ diff --git a/fixture/20/1/0.0.3 b/fixture/20/1/0.0.3 deleted file mode 100644 index a371944eea..0000000000 Binary files a/fixture/20/1/0.0.3 and /dev/null differ diff --git a/fixture/20/1/0.1.0 b/fixture/20/1/0.1.0 deleted file mode 100644 index bc9a97df71..0000000000 --- a/fixture/20/1/0.1.0 +++ /dev/null @@ -1,9 +0,0 @@ -x c_k̈YY9i9֒C89NCph'Iph'IơY8,94Nxϋ7:<@y$,f[, ,<H-q:nqw҇Tg i c8f}MiF&gB6o0K'9EpPK'A2ЛjְB>d#El@F‰Qa,5& 9RWTP9HI $R{ R-n#PFN:/3 &I {G)s2NPN=O'%n5\@A &2IcIO) (%^"vёGD3fo"JQE5RunpڑL# a(lm|DBӎ0"hI$1% &t^'9ee|M%L5jG"x$ҏd -VH;iH#BiN "hҙd0iL (8_RipщnWbY -J> nhE fCx^aOr/8NLvҁґ.a.XȟXreL5/׹ÏhG?30Hmb?qA$Leәf!9Y..qFϦы> Yfv ;iL;ҌiN4І2Y_pSӜrvt'y>,]Vlb3w@sý4 4c/1qLb2Sg([hG:3Н'f!3YrmB"l( b8ec8v ;?apцx= f%e᭴Q&8"ILf -ټLrO.U|yjB-ݣ,X* -zІL -He(/(FN)fRAQJdvDK;#t"d37aYB%稢o:7IR[H?ϑ`06P6>]fF-$ĒD2d1̣s[;HOD_џ"|@[6#4" Q& JStrR%e g? f \ No newline at end of file diff --git a/fixture/20/1/0.1.1 b/fixture/20/1/0.1.1 deleted file mode 100644 index dffa896ae2..0000000000 Binary files a/fixture/20/1/0.1.1 and /dev/null differ diff --git a/fixture/20/1/0.1.2 b/fixture/20/1/0.1.2 deleted file mode 100644 index a21b9f5179..0000000000 Binary files a/fixture/20/1/0.1.2 and /dev/null differ diff --git a/fixture/20/1/0.1.3 b/fixture/20/1/0.1.3 deleted file mode 100644 index 29c51b7bdf..0000000000 Binary files a/fixture/20/1/0.1.3 and /dev/null differ diff --git a/fixture/20/1/0.2.0 b/fixture/20/1/0.2.0 deleted file mode 100644 index fc0b00370c..0000000000 Binary files a/fixture/20/1/0.2.0 and /dev/null differ diff --git a/fixture/20/1/0.2.1 b/fixture/20/1/0.2.1 deleted file mode 100644 index 94b8d35554..0000000000 Binary files a/fixture/20/1/0.2.1 and /dev/null differ diff --git a/fixture/20/1/0.2.2 b/fixture/20/1/0.2.2 deleted file mode 100644 index bb22ea0ba8..0000000000 --- a/fixture/20/1/0.2.2 +++ /dev/null @@ -1,6 +0,0 @@ -x q|u_n133K%紖'Iph:4Nf,%ɱp,%ɱp,?v Iʳ" `%XM1ۂ Mf c5&4sC9Ag9G:AI 3,XI)&~a%HP1L^a4YL$a/|A*8N%Bړt"IfsG>ﱈVPepZ~&F΋ %QdRVQg즌s?H!amB,{.S jIC;R1!Sz']QDӒhK>g?jG4m8: :2)Le&by4g5~:="'5} b kYF&);hH;"h45҆qgo&3rS;.%R;ҝ_у^XMnܥћ eldg;^6͈Zӎh$^ Gf0ps\ -Wvvt'gEJVb>b%le͵>Q4)1"ьa,ټ$rtrT5',B;:t#gHeYB!"SMnDN$b(N&0,&nr v-=Z?M:vh26EюㄱuجYXhWB+Eq whwv[36=E`!F/]Evc{F{fL̂vhq|}wc[d BGxbl}D"ڝic_I"kݥ}c+ hwvG{|46KDa9If3\^>76/i̙,2e\56?i$h@{FcQ좈!N)L iF&8F9|E%Eb[kG t#/ uO>6B]&OCRIyt0ise4D=Ѵ=t+ ,d9,]rYZqp A.L -#xT2dE39)hJ f{( hY;ӌhN i `&#Yro8enp[|Z;zџ_3! elolg7{K.Qa4i4% d0dC(<Hl/#n$қ>%䱎gBvP ӐTFx^!)Le|A8MQ#v'.t%,"K.kX:q#A]eIa/X1좈|F19Q - \ No newline at end of file diff --git a/fixture/20/1/1.0.1 b/fixture/20/1/1.0.1 deleted file mode 100644 index 20317ff6b5..0000000000 Binary files a/fixture/20/1/1.0.1 and /dev/null differ diff --git a/fixture/20/1/1.0.2 b/fixture/20/1/1.0.2 deleted file mode 100644 index 5fc020f357..0000000000 Binary files a/fixture/20/1/1.0.2 and /dev/null differ diff --git a/fixture/20/1/1.0.3 b/fixture/20/1/1.0.3 deleted file mode 100644 index 22ca3074c3..0000000000 Binary files a/fixture/20/1/1.0.3 and /dev/null differ diff --git a/fixture/20/1/1.1.0 b/fixture/20/1/1.1.0 deleted file mode 100644 index 676f4d60ce..0000000000 Binary files a/fixture/20/1/1.1.0 and /dev/null differ diff --git a/fixture/20/1/1.1.1 b/fixture/20/1/1.1.1 deleted file mode 100644 index c06c4c4413..0000000000 Binary files a/fixture/20/1/1.1.1 and /dev/null differ diff --git a/fixture/20/1/1.1.2 b/fixture/20/1/1.1.2 deleted file mode 100644 index 9a602bf859..0000000000 Binary files a/fixture/20/1/1.1.2 and /dev/null differ diff --git a/fixture/20/1/1.1.3 b/fixture/20/1/1.1.3 deleted file mode 100644 index efe26bf741..0000000000 Binary files a/fixture/20/1/1.1.3 and /dev/null differ diff --git a/fixture/20/1/1.2.0 b/fixture/20/1/1.2.0 deleted file mode 100644 index 8c8e48b27e..0000000000 Binary files a/fixture/20/1/1.2.0 and /dev/null differ diff --git a/fixture/20/1/1.2.1 b/fixture/20/1/1.2.1 deleted file mode 100644 index 7a8b0b8c02..0000000000 --- a/fixture/20/1/1.2.1 +++ /dev/null @@ -1,5 +0,0 @@ -x |onIkIk9̒fFbf84KcY84KcY84¡Yrh '!'y}x^M7}Hy0lI)OA0qMsbd$y9C%p*s LRI/@:X6!(f%jQplc;!J] +L` -d*3yY NppK|eڵӎtYҋa9ﲂ5lb3[vԤ6x%F2?0ILf -9Cђ<`fP줔}^v'iD4͉aB.o0<1S |% :iGRI/@:X6!(f%JҎڄ1"XƑkL"r8.RI|ghK"H",YE].ʽ" 'a 'Cɦ}짌9B9' Y!ē@GcPJ -:7 %4P^$Q&bvP^>CnI͈&Vđ r[䱈sTr6w&%Y;Rs1! ld;QnNC%&4%,&t^'a.圤|pk$hG"t'?|@[vBzhG(uy0 h2d1iL)_RYqT툣=H+HfgQZֱܡ=?Pˏόr \ No newline at end of file diff --git a/fixture/20/1/1.2.2 b/fixture/20/1/1.2.2 deleted file mode 100644 index 39b2e3211a..0000000000 Binary files a/fixture/20/1/1.2.2 and /dev/null differ diff --git a/fixture/20/1/1.2.3 b/fixture/20/1/1.2.3 deleted file mode 100644 index a97d1c2770..0000000000 Binary files a/fixture/20/1/1.2.3 and /dev/null differ diff --git a/fixture/20/1/1.3.0 b/fixture/20/1/1.3.0 deleted file mode 100644 index cf472c33aa..0000000000 Binary files a/fixture/20/1/1.3.0 and /dev/null differ diff --git a/fixture/20/1/1.3.1 b/fixture/20/1/1.3.1 deleted file mode 100644 index 05047d37a3..0000000000 Binary files a/fixture/20/1/1.3.1 and /dev/null differ diff --git a/fixture/20/1/1.3.2 b/fixture/20/1/1.3.2 deleted file mode 100644 index 51c00060d4..0000000000 Binary files a/fixture/20/1/1.3.2 and /dev/null differ diff --git a/fixture/20/1/1.3.3 b/fixture/20/1/1.3.3 deleted file mode 100644 index 89f2759df3..0000000000 Binary files a/fixture/20/1/1.3.3 and /dev/null differ diff --git a/fixture/20/2/.zarray b/fixture/20/2/.zarray deleted file mode 100644 index 5738808845..0000000000 --- a/fixture/20/2/.zarray +++ /dev/null @@ -1,21 +0,0 @@ -{ - "chunks": [ - 100, - 3, - 3 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "0ɤ,f3|Gc>+Hc%X>3ֲlc;;.+v[8'8).s\W~:7x#/<#GI9'yC^Q(AIJ2P2TթAMjј&4iEKZE':Ӆtmޡ;= =3 c8L`"@"sˇ|<1f kY|z6]|Wf_ {qN=?pSO WƯun;p[< OyƿG@)H.r/#?(HIJ2P2PԤC(uiN -ZҊִ-toAOzћh3 c8`$q3La* $2$Lrr )@A -Q@(MR`B(O*RԢ6u.O hI+ZӆӎDIwzГ^&>%~ '$xF11%D1$LRHe Ybe,gigD:lf [^9Qq3E.q+]q<O&2d#;9I -RDRBy*PJT -UF(uG}h@Cј&%v'H:Б(:ћhЗџ dq3ьa,&D23I -b6s b5kH'la+v0G81;s\"~r>x#/,!z'ANyr@(BQQ(AIJQJT -UƫFujF҈4)hN "BWM \ No newline at end of file diff --git a/fixture/21/1/0.0.2 b/fixture/21/1/0.0.2 deleted file mode 100644 index 81a1664ada..0000000000 Binary files a/fixture/21/1/0.0.2 and /dev/null differ diff --git a/fixture/21/1/0.0.3 b/fixture/21/1/0.0.3 deleted file mode 100644 index e6715069bb..0000000000 Binary files a/fixture/21/1/0.0.3 and /dev/null differ diff --git a/fixture/21/1/0.1.0 b/fixture/21/1/0.1.0 deleted file mode 100644 index 1d7afd43be..0000000000 Binary files a/fixture/21/1/0.1.0 and /dev/null differ diff --git a/fixture/21/1/0.1.1 b/fixture/21/1/0.1.1 deleted file mode 100644 index eb6ba3e12d..0000000000 Binary files a/fixture/21/1/0.1.1 and /dev/null differ diff --git a/fixture/21/1/0.1.2 b/fixture/21/1/0.1.2 deleted file mode 100644 index ad7a0df808..0000000000 Binary files a/fixture/21/1/0.1.2 and /dev/null differ diff --git a/fixture/21/1/0.1.3 b/fixture/21/1/0.1.3 deleted file mode 100644 index 4e2f9caff1..0000000000 Binary files a/fixture/21/1/0.1.3 and /dev/null differ diff --git a/fixture/21/1/0.2.0 b/fixture/21/1/0.2.0 deleted file mode 100644 index 16b1382692..0000000000 Binary files a/fixture/21/1/0.2.0 and /dev/null differ diff --git a/fixture/21/1/0.2.1 b/fixture/21/1/0.2.1 deleted file mode 100644 index e76a1ed58f..0000000000 Binary files a/fixture/21/1/0.2.1 and /dev/null differ diff --git a/fixture/21/1/0.2.2 b/fixture/21/1/0.2.2 deleted file mode 100644 index 5028948a1c..0000000000 Binary files a/fixture/21/1/0.2.2 and /dev/null differ diff --git a/fixture/21/1/0.2.3 b/fixture/21/1/0.2.3 deleted file mode 100644 index cac892ae90..0000000000 Binary files a/fixture/21/1/0.2.3 and /dev/null differ diff --git a/fixture/21/1/0.3.0 b/fixture/21/1/0.3.0 deleted file mode 100644 index 5d16a9164b..0000000000 Binary files a/fixture/21/1/0.3.0 and /dev/null differ diff --git a/fixture/21/1/0.3.1 b/fixture/21/1/0.3.1 deleted file mode 100644 index b8fd44b7e1..0000000000 Binary files a/fixture/21/1/0.3.1 and /dev/null differ diff --git a/fixture/21/1/0.3.2 b/fixture/21/1/0.3.2 deleted file mode 100644 index 397044926b..0000000000 Binary files a/fixture/21/1/0.3.2 and /dev/null differ diff --git a/fixture/21/1/0.3.3 b/fixture/21/1/0.3.3 deleted file mode 100644 index 3c0d13d712..0000000000 --- a/fixture/21/1/0.3.3 +++ /dev/null @@ -1 +0,0 @@ -x%VP@GeXqKǥqt\:{{8)UPIPK4HʹJd蠓.顗>`!a1ƙ`)a9Y`%Ya5`-a=9#93ι+;y'y7/TRKPHŔŁ8@q ā8Yk \ No newline at end of file diff --git a/fixture/21/1/1.0.0 b/fixture/21/1/1.0.0 deleted file mode 100644 index 6d025242f1..0000000000 Binary files a/fixture/21/1/1.0.0 and /dev/null differ diff --git a/fixture/21/1/1.0.1 b/fixture/21/1/1.0.1 deleted file mode 100644 index f3514167c8..0000000000 Binary files a/fixture/21/1/1.0.1 and /dev/null differ diff --git a/fixture/21/1/1.0.2 b/fixture/21/1/1.0.2 deleted file mode 100644 index 77222f05cd..0000000000 Binary files a/fixture/21/1/1.0.2 and /dev/null differ diff --git a/fixture/21/1/1.0.3 b/fixture/21/1/1.0.3 deleted file mode 100644 index a0bb99cde7..0000000000 Binary files a/fixture/21/1/1.0.3 and /dev/null differ diff --git a/fixture/21/1/1.1.0 b/fixture/21/1/1.1.0 deleted file mode 100644 index cb1fb9c5a6..0000000000 Binary files a/fixture/21/1/1.1.0 and /dev/null differ diff --git a/fixture/21/1/1.1.1 b/fixture/21/1/1.1.1 deleted file mode 100644 index 02eaaa3bf6..0000000000 Binary files a/fixture/21/1/1.1.1 and /dev/null differ diff --git a/fixture/21/1/1.1.2 b/fixture/21/1/1.1.2 deleted file mode 100644 index eaad046efd..0000000000 Binary files a/fixture/21/1/1.1.2 and /dev/null differ diff --git a/fixture/21/1/1.1.3 b/fixture/21/1/1.1.3 deleted file mode 100644 index 51643ee8e5..0000000000 Binary files a/fixture/21/1/1.1.3 and /dev/null differ diff --git a/fixture/21/1/1.2.0 b/fixture/21/1/1.2.0 deleted file mode 100644 index b0ab313318..0000000000 Binary files a/fixture/21/1/1.2.0 and /dev/null differ diff --git a/fixture/21/1/1.2.1 b/fixture/21/1/1.2.1 deleted file mode 100644 index d95fa9baf9..0000000000 Binary files a/fixture/21/1/1.2.1 and /dev/null differ diff --git a/fixture/21/1/1.2.2 b/fixture/21/1/1.2.2 deleted file mode 100644 index 613b89e2f2..0000000000 Binary files a/fixture/21/1/1.2.2 and /dev/null differ diff --git a/fixture/21/1/1.2.3 b/fixture/21/1/1.2.3 deleted file mode 100644 index e8ebc253b6..0000000000 Binary files a/fixture/21/1/1.2.3 and /dev/null differ diff --git a/fixture/21/1/1.3.0 b/fixture/21/1/1.3.0 deleted file mode 100644 index a5ecc850a2..0000000000 Binary files a/fixture/21/1/1.3.0 and /dev/null differ diff --git a/fixture/21/1/1.3.1 b/fixture/21/1/1.3.1 deleted file mode 100644 index 3e220e32a4..0000000000 Binary files a/fixture/21/1/1.3.1 and /dev/null differ diff --git a/fixture/21/1/1.3.2 b/fixture/21/1/1.3.2 deleted file mode 100644 index 43c2585dbc..0000000000 Binary files a/fixture/21/1/1.3.2 and /dev/null differ diff --git a/fixture/21/1/1.3.3 b/fixture/21/1/1.3.3 deleted file mode 100644 index 77b54c165a..0000000000 Binary files a/fixture/21/1/1.3.3 and /dev/null differ diff --git a/fixture/21/2/.zarray b/fixture/21/2/.zarray deleted file mode 100644 index 9401811b6e..0000000000 --- a/fixture/21/2/.zarray +++ /dev/null @@ -1,21 +0,0 @@ -{ - "chunks": [ - 10, - 30, - 3 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "ƕعķ8MUw>oS]q9x#6oD6vq /Yk?ꎝQ\Sy|qwN@4㸈 43 diff --git a/fixture/22/1/0.0.1.0 b/fixture/22/1/0.0.1.0 deleted file mode 100644 index a25bc6a451..0000000000 Binary files a/fixture/22/1/0.0.1.0 and /dev/null differ diff --git a/fixture/22/1/0.0.1.1 b/fixture/22/1/0.0.1.1 deleted file mode 100644 index 42f4456bcc..0000000000 Binary files a/fixture/22/1/0.0.1.1 and /dev/null differ diff --git a/fixture/22/1/0.0.1.2 b/fixture/22/1/0.0.1.2 deleted file mode 100644 index f3d1405e2e..0000000000 Binary files a/fixture/22/1/0.0.1.2 and /dev/null differ diff --git a/fixture/22/1/0.0.1.3 b/fixture/22/1/0.0.1.3 deleted file mode 100644 index a64124fb02..0000000000 --- a/fixture/22/1/0.0.1.3 +++ /dev/null @@ -1,4 +0,0 @@ -xM1(auN!td0`$ L I&$d0$$Ip$`d1wz3L*Gt\p7 -Ohm .Eo=~L?.f -S|q-#~Ŏ،  >F -lglp-\3[by+x'=pOFgRlV:~ϸ3Kx:9/ Ptq3 \ No newline at end of file diff --git a/fixture/22/1/0.0.2.0 b/fixture/22/1/0.0.2.0 deleted file mode 100644 index b3a0fe53fb..0000000000 Binary files a/fixture/22/1/0.0.2.0 and /dev/null differ diff --git a/fixture/22/1/0.0.2.1 b/fixture/22/1/0.0.2.1 deleted file mode 100644 index ba298400be..0000000000 Binary files a/fixture/22/1/0.0.2.1 and /dev/null differ diff --git a/fixture/22/1/0.0.2.2 b/fixture/22/1/0.0.2.2 deleted file mode 100644 index 8ef05933d1..0000000000 Binary files a/fixture/22/1/0.0.2.2 and /dev/null differ diff --git a/fixture/22/1/0.0.2.3 b/fixture/22/1/0.0.2.3 deleted file mode 100644 index d653bb051c..0000000000 Binary files a/fixture/22/1/0.0.2.3 and /dev/null differ diff --git a/fixture/22/1/0.0.3.0 b/fixture/22/1/0.0.3.0 deleted file mode 100644 index b04cb5d95e..0000000000 Binary files a/fixture/22/1/0.0.3.0 and /dev/null differ diff --git a/fixture/22/1/0.0.3.1 b/fixture/22/1/0.0.3.1 deleted file mode 100644 index 6782af04ed..0000000000 Binary files a/fixture/22/1/0.0.3.1 and /dev/null differ diff --git a/fixture/22/1/0.0.3.2 b/fixture/22/1/0.0.3.2 deleted file mode 100644 index 3614c1ba78..0000000000 Binary files a/fixture/22/1/0.0.3.2 and /dev/null differ diff --git a/fixture/22/1/0.0.3.3 b/fixture/22/1/0.0.3.3 deleted file mode 100644 index de707a4ed7..0000000000 Binary files a/fixture/22/1/0.0.3.3 and /dev/null differ diff --git a/fixture/22/1/0.1.0.0 b/fixture/22/1/0.1.0.0 deleted file mode 100644 index a2eeb93a20..0000000000 Binary files a/fixture/22/1/0.1.0.0 and /dev/null differ diff --git a/fixture/22/1/0.1.0.1 b/fixture/22/1/0.1.0.1 deleted file mode 100644 index e56702b7dc..0000000000 Binary files a/fixture/22/1/0.1.0.1 and /dev/null differ diff --git a/fixture/22/1/0.1.0.2 b/fixture/22/1/0.1.0.2 deleted file mode 100644 index 705b3e4625..0000000000 Binary files a/fixture/22/1/0.1.0.2 and /dev/null differ diff --git a/fixture/22/1/0.1.0.3 b/fixture/22/1/0.1.0.3 deleted file mode 100644 index 428ce7748e..0000000000 --- a/fixture/22/1/0.1.0.3 +++ /dev/null @@ -1 +0,0 @@ -xM1(.]ʕ+W.An dn0H2I7$ L`L K&I%$eM_RK*q<?֚T85;?8_8x_q=~}㖎W,^FWt\FfO|UOG\0.M/z%"Ӎx 鏞}|/Nv*^ 7X!$$Z.g?03 \ No newline at end of file diff --git a/fixture/22/1/0.1.1.0 b/fixture/22/1/0.1.1.0 deleted file mode 100644 index 0de0db05a2..0000000000 Binary files a/fixture/22/1/0.1.1.0 and /dev/null differ diff --git a/fixture/22/1/0.1.1.1 b/fixture/22/1/0.1.1.1 deleted file mode 100644 index 6de9b36307..0000000000 Binary files a/fixture/22/1/0.1.1.1 and /dev/null differ diff --git a/fixture/22/1/0.1.1.2 b/fixture/22/1/0.1.1.2 deleted file mode 100644 index ddd0e73d69..0000000000 Binary files a/fixture/22/1/0.1.1.2 and /dev/null differ diff --git a/fixture/22/1/0.1.1.3 b/fixture/22/1/0.1.1.3 deleted file mode 100644 index 4560d14cf0..0000000000 Binary files a/fixture/22/1/0.1.1.3 and /dev/null differ diff --git a/fixture/22/1/0.1.2.0 b/fixture/22/1/0.1.2.0 deleted file mode 100644 index d6c4613ac8..0000000000 Binary files a/fixture/22/1/0.1.2.0 and /dev/null differ diff --git a/fixture/22/1/0.1.2.1 b/fixture/22/1/0.1.2.1 deleted file mode 100644 index d4971aa88b..0000000000 Binary files a/fixture/22/1/0.1.2.1 and /dev/null differ diff --git a/fixture/22/1/0.1.2.2 b/fixture/22/1/0.1.2.2 deleted file mode 100644 index f96c8d124d..0000000000 Binary files a/fixture/22/1/0.1.2.2 and /dev/null differ diff --git a/fixture/22/1/0.1.2.3 b/fixture/22/1/0.1.2.3 deleted file mode 100644 index abba8e8445..0000000000 Binary files a/fixture/22/1/0.1.2.3 and /dev/null differ diff --git a/fixture/22/1/0.1.3.0 b/fixture/22/1/0.1.3.0 deleted file mode 100644 index 96d51927fd..0000000000 Binary files a/fixture/22/1/0.1.3.0 and /dev/null differ diff --git a/fixture/22/1/0.1.3.1 b/fixture/22/1/0.1.3.1 deleted file mode 100644 index b00a9c93cc..0000000000 Binary files a/fixture/22/1/0.1.3.1 and /dev/null differ diff --git a/fixture/22/1/0.1.3.2 b/fixture/22/1/0.1.3.2 deleted file mode 100644 index 827cfb46de..0000000000 Binary files a/fixture/22/1/0.1.3.2 and /dev/null differ diff --git a/fixture/22/1/0.1.3.3 b/fixture/22/1/0.1.3.3 deleted file mode 100644 index 7a2b49c66a..0000000000 Binary files a/fixture/22/1/0.1.3.3 and /dev/null differ diff --git a/fixture/22/1/0.2.0.0 b/fixture/22/1/0.2.0.0 deleted file mode 100644 index 139772498c..0000000000 Binary files a/fixture/22/1/0.2.0.0 and /dev/null differ diff --git a/fixture/22/1/0.2.0.1 b/fixture/22/1/0.2.0.1 deleted file mode 100644 index 669013241e..0000000000 Binary files a/fixture/22/1/0.2.0.1 and /dev/null differ diff --git a/fixture/22/1/0.2.0.2 b/fixture/22/1/0.2.0.2 deleted file mode 100644 index 9f6313e3a0..0000000000 Binary files a/fixture/22/1/0.2.0.2 and /dev/null differ diff --git a/fixture/22/1/0.2.0.3 b/fixture/22/1/0.2.0.3 deleted file mode 100644 index f2268aed5a..0000000000 --- a/fixture/22/1/0.2.0.3 +++ /dev/null @@ -1,2 +0,0 @@ -xM1(P;ʅSGБp$ 7d0p 2d0`L I&$tAI2 IAyӧ}_)8+x=~£F&1p%6x ]y\h$U|O/Nَq?po]?7Nv_ie7-~/=}?<\g -C6: Of>>5| ù،;sq`l:q/5\pr8E\2 \ No newline at end of file diff --git a/fixture/22/1/0.2.1.0 b/fixture/22/1/0.2.1.0 deleted file mode 100644 index 0f9b103087..0000000000 Binary files a/fixture/22/1/0.2.1.0 and /dev/null differ diff --git a/fixture/22/1/0.2.1.1 b/fixture/22/1/0.2.1.1 deleted file mode 100644 index 3d86981a9e..0000000000 Binary files a/fixture/22/1/0.2.1.1 and /dev/null differ diff --git a/fixture/22/1/0.2.1.2 b/fixture/22/1/0.2.1.2 deleted file mode 100644 index 9411b4d7e4..0000000000 Binary files a/fixture/22/1/0.2.1.2 and /dev/null differ diff --git a/fixture/22/1/0.2.1.3 b/fixture/22/1/0.2.1.3 deleted file mode 100644 index 5c74d709aa..0000000000 --- a/fixture/22/1/0.2.1.3 +++ /dev/null @@ -1,2 +0,0 @@ -xM1(`qE!#Jp$`dL 2d 2n0I&$tAI $}+%jo*x.Y$2fo -~/x!:xbWq_[ܕNlsl<.c|pcK4q-#x>o 5 ?*쎿~p'xWc3s2D?،,Os/qj868.N4c \ No newline at end of file diff --git a/fixture/22/1/0.2.2.0 b/fixture/22/1/0.2.2.0 deleted file mode 100644 index cdfdde98bd..0000000000 Binary files a/fixture/22/1/0.2.2.0 and /dev/null differ diff --git a/fixture/22/1/0.2.2.1 b/fixture/22/1/0.2.2.1 deleted file mode 100644 index b244b2247e..0000000000 Binary files a/fixture/22/1/0.2.2.1 and /dev/null differ diff --git a/fixture/22/1/0.2.2.2 b/fixture/22/1/0.2.2.2 deleted file mode 100644 index bf7e5d7563..0000000000 Binary files a/fixture/22/1/0.2.2.2 and /dev/null differ diff --git a/fixture/22/1/0.2.2.3 b/fixture/22/1/0.2.2.3 deleted file mode 100644 index acec461696..0000000000 --- a/fixture/22/1/0.2.2.3 +++ /dev/null @@ -1 +0,0 @@ -xM1(`;B\\uԑIp$`dd0\2I&  2 2d0`L I&$};nH$x-\n>>5\Kt2.f*~ox<I<Yko=Φ3Gq36xs|ƽ8'3o gpom\Əw6íq_+\4',.S|+6c3 x,3 \ No newline at end of file diff --git a/fixture/22/1/0.2.3.0 b/fixture/22/1/0.2.3.0 deleted file mode 100644 index 0d7a005610..0000000000 Binary files a/fixture/22/1/0.2.3.0 and /dev/null differ diff --git a/fixture/22/1/0.2.3.1 b/fixture/22/1/0.2.3.1 deleted file mode 100644 index bcc1fa9083..0000000000 Binary files a/fixture/22/1/0.2.3.1 and /dev/null differ diff --git a/fixture/22/1/0.2.3.2 b/fixture/22/1/0.2.3.2 deleted file mode 100644 index 49c59ffb38..0000000000 Binary files a/fixture/22/1/0.2.3.2 and /dev/null differ diff --git a/fixture/22/1/0.2.3.3 b/fixture/22/1/0.2.3.3 deleted file mode 100644 index f9500e9127..0000000000 Binary files a/fixture/22/1/0.2.3.3 and /dev/null differ diff --git a/fixture/22/1/0.3.0.0 b/fixture/22/1/0.3.0.0 deleted file mode 100644 index e015279f47..0000000000 Binary files a/fixture/22/1/0.3.0.0 and /dev/null differ diff --git a/fixture/22/1/0.3.0.1 b/fixture/22/1/0.3.0.1 deleted file mode 100644 index 19b542b76b..0000000000 Binary files a/fixture/22/1/0.3.0.1 and /dev/null differ diff --git a/fixture/22/1/0.3.0.2 b/fixture/22/1/0.3.0.2 deleted file mode 100644 index 9ef533faf9..0000000000 Binary files a/fixture/22/1/0.3.0.2 and /dev/null differ diff --git a/fixture/22/1/0.3.0.3 b/fixture/22/1/0.3.0.3 deleted file mode 100644 index d9172b1b5c..0000000000 --- a/fixture/22/1/0.3.0.3 +++ /dev/null @@ -1 +0,0 @@ -x!AaFqp`ffccl77 ˂, ,Ȳ!˂, ,ش{NyOVEnQȴ;"Ӳ~%eL>JeL!\~#d+Sdz?L:g2E?w \ No newline at end of file diff --git a/fixture/22/1/0.3.1.0 b/fixture/22/1/0.3.1.0 deleted file mode 100644 index d0b2cb0772..0000000000 Binary files a/fixture/22/1/0.3.1.0 and /dev/null differ diff --git a/fixture/22/1/0.3.1.1 b/fixture/22/1/0.3.1.1 deleted file mode 100644 index 16c06afc64..0000000000 Binary files a/fixture/22/1/0.3.1.1 and /dev/null differ diff --git a/fixture/22/1/0.3.1.2 b/fixture/22/1/0.3.1.2 deleted file mode 100644 index 9fb7108772..0000000000 Binary files a/fixture/22/1/0.3.1.2 and /dev/null differ diff --git a/fixture/22/1/0.3.1.3 b/fixture/22/1/0.3.1.3 deleted file mode 100644 index 2571f6116c..0000000000 --- a/fixture/22/1/0.3.1.3 +++ /dev/null @@ -1 +0,0 @@ -xEff'i4 xy!8q\}^ {OB\<7"q$eL1FneL )2*ky#Sdksy)Sd*K)2}fO-ؿL` \ No newline at end of file diff --git a/fixture/22/1/0.3.2.0 b/fixture/22/1/0.3.2.0 deleted file mode 100644 index b66a99fe3c..0000000000 Binary files a/fixture/22/1/0.3.2.0 and /dev/null differ diff --git a/fixture/22/1/0.3.2.1 b/fixture/22/1/0.3.2.1 deleted file mode 100644 index 4d4dcf5f2a..0000000000 Binary files a/fixture/22/1/0.3.2.1 and /dev/null differ diff --git a/fixture/22/1/0.3.2.2 b/fixture/22/1/0.3.2.2 deleted file mode 100644 index 5f8d768b12..0000000000 Binary files a/fixture/22/1/0.3.2.2 and /dev/null differ diff --git a/fixture/22/1/0.3.2.3 b/fixture/22/1/0.3.2.3 deleted file mode 100644 index 8556a301a3..0000000000 --- a/fixture/22/1/0.3.2.3 +++ /dev/null @@ -1 +0,0 @@ -x!AaFq p 7˂, ,Ȳ ˂, CȊv7ygQ ĭw!Sd:w_LcO2Em~'eLZ>)2g\/L3a[Sw \ No newline at end of file diff --git a/fixture/22/1/0.3.3.0 b/fixture/22/1/0.3.3.0 deleted file mode 100644 index 1e599375bc..0000000000 Binary files a/fixture/22/1/0.3.3.0 and /dev/null differ diff --git a/fixture/22/1/0.3.3.1 b/fixture/22/1/0.3.3.1 deleted file mode 100644 index 5e89372761..0000000000 Binary files a/fixture/22/1/0.3.3.1 and /dev/null differ diff --git a/fixture/22/1/0.3.3.2 b/fixture/22/1/0.3.3.2 deleted file mode 100644 index 51df60bc2e..0000000000 Binary files a/fixture/22/1/0.3.3.2 and /dev/null differ diff --git a/fixture/22/1/0.3.3.3 b/fixture/22/1/0.3.3.3 deleted file mode 100644 index 1a463a4ac7..0000000000 Binary files a/fixture/22/1/0.3.3.3 and /dev/null differ diff --git a/fixture/22/1/1.0.0.0 b/fixture/22/1/1.0.0.0 deleted file mode 100644 index b4f4bec7da..0000000000 Binary files a/fixture/22/1/1.0.0.0 and /dev/null differ diff --git a/fixture/22/1/1.0.0.1 b/fixture/22/1/1.0.0.1 deleted file mode 100644 index 17d1f7dfe1..0000000000 Binary files a/fixture/22/1/1.0.0.1 and /dev/null differ diff --git a/fixture/22/1/1.0.0.2 b/fixture/22/1/1.0.0.2 deleted file mode 100644 index 8f7d154fc1..0000000000 Binary files a/fixture/22/1/1.0.0.2 and /dev/null differ diff --git a/fixture/22/1/1.0.0.3 b/fixture/22/1/1.0.0.3 deleted file mode 100644 index b005dca87a..0000000000 Binary files a/fixture/22/1/1.0.0.3 and /dev/null differ diff --git a/fixture/22/1/1.0.1.0 b/fixture/22/1/1.0.1.0 deleted file mode 100644 index 7ec4e434ba..0000000000 Binary files a/fixture/22/1/1.0.1.0 and /dev/null differ diff --git a/fixture/22/1/1.0.1.1 b/fixture/22/1/1.0.1.1 deleted file mode 100644 index d2794dd4b1..0000000000 Binary files a/fixture/22/1/1.0.1.1 and /dev/null differ diff --git a/fixture/22/1/1.0.1.2 b/fixture/22/1/1.0.1.2 deleted file mode 100644 index 8ac5ad017a..0000000000 Binary files a/fixture/22/1/1.0.1.2 and /dev/null differ diff --git a/fixture/22/1/1.0.1.3 b/fixture/22/1/1.0.1.3 deleted file mode 100644 index 604faaa2fa..0000000000 --- a/fixture/22/1/1.0.1.3 +++ /dev/null @@ -1 +0,0 @@ -xM1(P+%d2`8ĕ+W. LI $`dn0H2IA2I2.$ Lyӧ}_6h\U|Onj$m88+xwp/jW:~Ooܔ _;\ ll4>'_ E?\lzqo]\9×%vKx Wht!¿89og<.Չo-p љ%<7Eo=t1:8#A \ No newline at end of file diff --git a/fixture/22/1/1.0.2.0 b/fixture/22/1/1.0.2.0 deleted file mode 100644 index 9cd3ede04e..0000000000 Binary files a/fixture/22/1/1.0.2.0 and /dev/null differ diff --git a/fixture/22/1/1.0.2.1 b/fixture/22/1/1.0.2.1 deleted file mode 100644 index 575cd0da27..0000000000 Binary files a/fixture/22/1/1.0.2.1 and /dev/null differ diff --git a/fixture/22/1/1.0.2.2 b/fixture/22/1/1.0.2.2 deleted file mode 100644 index 9c50735597..0000000000 Binary files a/fixture/22/1/1.0.2.2 and /dev/null differ diff --git a/fixture/22/1/1.0.2.3 b/fixture/22/1/1.0.2.3 deleted file mode 100644 index e31319533e..0000000000 Binary files a/fixture/22/1/1.0.2.3 and /dev/null differ diff --git a/fixture/22/1/1.0.3.0 b/fixture/22/1/1.0.3.0 deleted file mode 100644 index 8cb94f9044..0000000000 Binary files a/fixture/22/1/1.0.3.0 and /dev/null differ diff --git a/fixture/22/1/1.0.3.1 b/fixture/22/1/1.0.3.1 deleted file mode 100644 index 338f0ff828..0000000000 Binary files a/fixture/22/1/1.0.3.1 and /dev/null differ diff --git a/fixture/22/1/1.0.3.2 b/fixture/22/1/1.0.3.2 deleted file mode 100644 index 716fd23788..0000000000 Binary files a/fixture/22/1/1.0.3.2 and /dev/null differ diff --git a/fixture/22/1/1.0.3.3 b/fixture/22/1/1.0.3.3 deleted file mode 100644 index 8ec8a7accc..0000000000 Binary files a/fixture/22/1/1.0.3.3 and /dev/null differ diff --git a/fixture/22/1/1.1.0.0 b/fixture/22/1/1.1.0.0 deleted file mode 100644 index 9c9955a5bd..0000000000 Binary files a/fixture/22/1/1.1.0.0 and /dev/null differ diff --git a/fixture/22/1/1.1.0.1 b/fixture/22/1/1.1.0.1 deleted file mode 100644 index 3836072634..0000000000 Binary files a/fixture/22/1/1.1.0.1 and /dev/null differ diff --git a/fixture/22/1/1.1.0.2 b/fixture/22/1/1.1.0.2 deleted file mode 100644 index b2f77aad18..0000000000 Binary files a/fixture/22/1/1.1.0.2 and /dev/null differ diff --git a/fixture/22/1/1.1.0.3 b/fixture/22/1/1.1.0.3 deleted file mode 100644 index 6e9d01f504..0000000000 Binary files a/fixture/22/1/1.1.0.3 and /dev/null differ diff --git a/fixture/22/1/1.1.1.0 b/fixture/22/1/1.1.1.0 deleted file mode 100644 index 2b676a23af..0000000000 Binary files a/fixture/22/1/1.1.1.0 and /dev/null differ diff --git a/fixture/22/1/1.1.1.1 b/fixture/22/1/1.1.1.1 deleted file mode 100644 index 05e34d40ae..0000000000 Binary files a/fixture/22/1/1.1.1.1 and /dev/null differ diff --git a/fixture/22/1/1.1.1.2 b/fixture/22/1/1.1.1.2 deleted file mode 100644 index 88e594c031..0000000000 Binary files a/fixture/22/1/1.1.1.2 and /dev/null differ diff --git a/fixture/22/1/1.1.1.3 b/fixture/22/1/1.1.1.3 deleted file mode 100644 index 3002f080b3..0000000000 Binary files a/fixture/22/1/1.1.1.3 and /dev/null differ diff --git a/fixture/22/1/1.1.2.0 b/fixture/22/1/1.1.2.0 deleted file mode 100644 index fc1e0fa17d..0000000000 Binary files a/fixture/22/1/1.1.2.0 and /dev/null differ diff --git a/fixture/22/1/1.1.2.1 b/fixture/22/1/1.1.2.1 deleted file mode 100644 index 663523a385..0000000000 Binary files a/fixture/22/1/1.1.2.1 and /dev/null differ diff --git a/fixture/22/1/1.1.2.2 b/fixture/22/1/1.1.2.2 deleted file mode 100644 index b4f75d972f..0000000000 Binary files a/fixture/22/1/1.1.2.2 and /dev/null differ diff --git a/fixture/22/1/1.1.2.3 b/fixture/22/1/1.1.2.3 deleted file mode 100644 index b1168f5205..0000000000 Binary files a/fixture/22/1/1.1.2.3 and /dev/null differ diff --git a/fixture/22/1/1.1.3.0 b/fixture/22/1/1.1.3.0 deleted file mode 100644 index 25f5b400af..0000000000 Binary files a/fixture/22/1/1.1.3.0 and /dev/null differ diff --git a/fixture/22/1/1.1.3.1 b/fixture/22/1/1.1.3.1 deleted file mode 100644 index 4c80fc02d5..0000000000 Binary files a/fixture/22/1/1.1.3.1 and /dev/null differ diff --git a/fixture/22/1/1.1.3.2 b/fixture/22/1/1.1.3.2 deleted file mode 100644 index 980b917e89..0000000000 Binary files a/fixture/22/1/1.1.3.2 and /dev/null differ diff --git a/fixture/22/1/1.1.3.3 b/fixture/22/1/1.1.3.3 deleted file mode 100644 index d28b3cc243..0000000000 Binary files a/fixture/22/1/1.1.3.3 and /dev/null differ diff --git a/fixture/22/1/1.2.0.0 b/fixture/22/1/1.2.0.0 deleted file mode 100644 index 288070000e..0000000000 Binary files a/fixture/22/1/1.2.0.0 and /dev/null differ diff --git a/fixture/22/1/1.2.0.1 b/fixture/22/1/1.2.0.1 deleted file mode 100644 index 896e6cec3f..0000000000 Binary files a/fixture/22/1/1.2.0.1 and /dev/null differ diff --git a/fixture/22/1/1.2.0.2 b/fixture/22/1/1.2.0.2 deleted file mode 100644 index 324d767d44..0000000000 Binary files a/fixture/22/1/1.2.0.2 and /dev/null differ diff --git a/fixture/22/1/1.2.0.3 b/fixture/22/1/1.2.0.3 deleted file mode 100644 index 1304af4cc0..0000000000 Binary files a/fixture/22/1/1.2.0.3 and /dev/null differ diff --git a/fixture/22/1/1.2.1.0 b/fixture/22/1/1.2.1.0 deleted file mode 100644 index cef5978395..0000000000 Binary files a/fixture/22/1/1.2.1.0 and /dev/null differ diff --git a/fixture/22/1/1.2.1.1 b/fixture/22/1/1.2.1.1 deleted file mode 100644 index 58e64613d0..0000000000 Binary files a/fixture/22/1/1.2.1.1 and /dev/null differ diff --git a/fixture/22/1/1.2.1.2 b/fixture/22/1/1.2.1.2 deleted file mode 100644 index 4240dc12bd..0000000000 Binary files a/fixture/22/1/1.2.1.2 and /dev/null differ diff --git a/fixture/22/1/1.2.1.3 b/fixture/22/1/1.2.1.3 deleted file mode 100644 index 9cac51028c..0000000000 Binary files a/fixture/22/1/1.2.1.3 and /dev/null differ diff --git a/fixture/22/1/1.2.2.0 b/fixture/22/1/1.2.2.0 deleted file mode 100644 index 770dfd64fe..0000000000 Binary files a/fixture/22/1/1.2.2.0 and /dev/null differ diff --git a/fixture/22/1/1.2.2.1 b/fixture/22/1/1.2.2.1 deleted file mode 100644 index 8144a94251..0000000000 Binary files a/fixture/22/1/1.2.2.1 and /dev/null differ diff --git a/fixture/22/1/1.2.2.2 b/fixture/22/1/1.2.2.2 deleted file mode 100644 index 7819de9e08..0000000000 Binary files a/fixture/22/1/1.2.2.2 and /dev/null differ diff --git a/fixture/22/1/1.2.2.3 b/fixture/22/1/1.2.2.3 deleted file mode 100644 index b4d098e108..0000000000 Binary files a/fixture/22/1/1.2.2.3 and /dev/null differ diff --git a/fixture/22/1/1.2.3.0 b/fixture/22/1/1.2.3.0 deleted file mode 100644 index 76254ea78b..0000000000 Binary files a/fixture/22/1/1.2.3.0 and /dev/null differ diff --git a/fixture/22/1/1.2.3.1 b/fixture/22/1/1.2.3.1 deleted file mode 100644 index 1b26cc2878..0000000000 Binary files a/fixture/22/1/1.2.3.1 and /dev/null differ diff --git a/fixture/22/1/1.2.3.2 b/fixture/22/1/1.2.3.2 deleted file mode 100644 index 3e9c7e2b75..0000000000 Binary files a/fixture/22/1/1.2.3.2 and /dev/null differ diff --git a/fixture/22/1/1.2.3.3 b/fixture/22/1/1.2.3.3 deleted file mode 100644 index dea4cd79c7..0000000000 Binary files a/fixture/22/1/1.2.3.3 and /dev/null differ diff --git a/fixture/22/1/1.3.0.0 b/fixture/22/1/1.3.0.0 deleted file mode 100644 index 1267f29539..0000000000 Binary files a/fixture/22/1/1.3.0.0 and /dev/null differ diff --git a/fixture/22/1/1.3.0.1 b/fixture/22/1/1.3.0.1 deleted file mode 100644 index f19f044789..0000000000 Binary files a/fixture/22/1/1.3.0.1 and /dev/null differ diff --git a/fixture/22/1/1.3.0.2 b/fixture/22/1/1.3.0.2 deleted file mode 100644 index 59772c916c..0000000000 Binary files a/fixture/22/1/1.3.0.2 and /dev/null differ diff --git a/fixture/22/1/1.3.0.3 b/fixture/22/1/1.3.0.3 deleted file mode 100644 index 85fc7f6e2b..0000000000 --- a/fixture/22/1/1.3.0.3 +++ /dev/null @@ -1 +0,0 @@ -x!AaFqUeAMl ˂, ,Ȳ!˂, 2oyNz*WACG$SdLig72Eѐ})/eL1BM%SdL9K~n w!Sd \ No newline at end of file diff --git a/fixture/22/1/1.3.1.0 b/fixture/22/1/1.3.1.0 deleted file mode 100644 index 022e119918..0000000000 Binary files a/fixture/22/1/1.3.1.0 and /dev/null differ diff --git a/fixture/22/1/1.3.1.1 b/fixture/22/1/1.3.1.1 deleted file mode 100644 index 76c7f19ece..0000000000 Binary files a/fixture/22/1/1.3.1.1 and /dev/null differ diff --git a/fixture/22/1/1.3.1.2 b/fixture/22/1/1.3.1.2 deleted file mode 100644 index 0def4f0cf4..0000000000 Binary files a/fixture/22/1/1.3.1.2 and /dev/null differ diff --git a/fixture/22/1/1.3.1.3 b/fixture/22/1/1.3.1.3 deleted file mode 100644 index 7d7d7bb695..0000000000 --- a/fixture/22/1/1.3.1.3 +++ /dev/null @@ -1 +0,0 @@ -x!AaFqEd&۵ndYeAYdهeA9yo} Z{/ B\G|p2EӘYS(SdoLi9g_k"Ӱ`?2Enɾ'eL}{;2LGGs \ No newline at end of file diff --git a/fixture/22/1/1.3.2.0 b/fixture/22/1/1.3.2.0 deleted file mode 100644 index 0ae0915622..0000000000 Binary files a/fixture/22/1/1.3.2.0 and /dev/null differ diff --git a/fixture/22/1/1.3.2.1 b/fixture/22/1/1.3.2.1 deleted file mode 100644 index d697b38934..0000000000 Binary files a/fixture/22/1/1.3.2.1 and /dev/null differ diff --git a/fixture/22/1/1.3.2.2 b/fixture/22/1/1.3.2.2 deleted file mode 100644 index 7f921ebdfc..0000000000 Binary files a/fixture/22/1/1.3.2.2 and /dev/null differ diff --git a/fixture/22/1/1.3.2.3 b/fixture/22/1/1.3.2.3 deleted file mode 100644 index 7f79d8c9cd..0000000000 --- a/fixture/22/1/1.3.2.3 +++ /dev/null @@ -1 +0,0 @@ -x!AFa,Ț 66r 7˂, , ,Ȳs' Z}!#L>ak"e*dL|)2U { Sd,r)SdLeߗ2E}Gϭ \ No newline at end of file diff --git a/fixture/22/1/1.3.3.0 b/fixture/22/1/1.3.3.0 deleted file mode 100644 index 4751c02b60..0000000000 Binary files a/fixture/22/1/1.3.3.0 and /dev/null differ diff --git a/fixture/22/1/1.3.3.1 b/fixture/22/1/1.3.3.1 deleted file mode 100644 index 4d6ffd8ce9..0000000000 Binary files a/fixture/22/1/1.3.3.1 and /dev/null differ diff --git a/fixture/22/1/1.3.3.2 b/fixture/22/1/1.3.3.2 deleted file mode 100644 index b3a45852a4..0000000000 Binary files a/fixture/22/1/1.3.3.2 and /dev/null differ diff --git a/fixture/22/1/1.3.3.3 b/fixture/22/1/1.3.3.3 deleted file mode 100644 index 376561b1e0..0000000000 Binary files a/fixture/22/1/1.3.3.3 and /dev/null differ diff --git a/fixture/22/2/.zarray b/fixture/22/2/.zarray deleted file mode 100644 index 88ed80efcc..0000000000 --- a/fixture/22/2/.zarray +++ /dev/null @@ -1,23 +0,0 @@ -{ - "chunks": [ - 10, - 3, - 3, - 3 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "ѵ>~#F?pǜpJ3IiY昧e*,]3\ppUDˬPg5ɥ@QgILD \ No newline at end of file diff --git a/fixture/23/1/0.3.0.2 b/fixture/23/1/0.3.0.2 deleted file mode 100644 index 2b2c4141cf..0000000000 --- a/fixture/23/1/0.3.0.2 +++ /dev/null @@ -1 +0,0 @@ -xM`϶X"(Xb6*m6v@=>{{OSZq\qMt+VYc 6b:;D7\L)1d"zd}8Uȑ@iJ!Wo'_|ӡKK0K*s̳"5rJ}3 CDwc6w#Oi] Hv8qƃ|c 3lJ#A 9 -tt \ No newline at end of file diff --git a/fixture/23/1/0.3.3.3 b/fixture/23/1/0.3.3.3 deleted file mode 100644 index 668895bf37..0000000000 Binary files a/fixture/23/1/0.3.3.3 and /dev/null differ diff --git a/fixture/23/1/1.0.0.0 b/fixture/23/1/1.0.0.0 deleted file mode 100644 index fc770b2b08..0000000000 Binary files a/fixture/23/1/1.0.0.0 and /dev/null differ diff --git a/fixture/23/1/1.0.0.1 b/fixture/23/1/1.0.0.1 deleted file mode 100644 index 35bde18a3c..0000000000 Binary files a/fixture/23/1/1.0.0.1 and /dev/null differ diff --git a/fixture/23/1/1.0.0.2 b/fixture/23/1/1.0.0.2 deleted file mode 100644 index ec2904ed32..0000000000 Binary files a/fixture/23/1/1.0.0.2 and /dev/null differ diff --git a/fixture/23/1/1.0.0.3 b/fixture/23/1/1.0.0.3 deleted file mode 100644 index 2b3c6b999d..0000000000 Binary files a/fixture/23/1/1.0.0.3 and /dev/null differ diff --git a/fixture/23/1/1.0.1.0 b/fixture/23/1/1.0.1.0 deleted file mode 100644 index bf299471af..0000000000 Binary files a/fixture/23/1/1.0.1.0 and /dev/null differ diff --git a/fixture/23/1/1.0.1.1 b/fixture/23/1/1.0.1.1 deleted file mode 100644 index 30167a5788..0000000000 Binary files a/fixture/23/1/1.0.1.1 and /dev/null differ diff --git a/fixture/23/1/1.0.1.2 b/fixture/23/1/1.0.1.2 deleted file mode 100644 index 862e662ca1..0000000000 Binary files a/fixture/23/1/1.0.1.2 and /dev/null differ diff --git a/fixture/23/1/1.0.1.3 b/fixture/23/1/1.0.1.3 deleted file mode 100644 index 8b83810e65..0000000000 Binary files a/fixture/23/1/1.0.1.3 and /dev/null differ diff --git a/fixture/23/1/1.0.2.0 b/fixture/23/1/1.0.2.0 deleted file mode 100644 index f12a0c7038..0000000000 Binary files a/fixture/23/1/1.0.2.0 and /dev/null differ diff --git a/fixture/23/1/1.0.2.1 b/fixture/23/1/1.0.2.1 deleted file mode 100644 index d65e47ec8f..0000000000 Binary files a/fixture/23/1/1.0.2.1 and /dev/null differ diff --git a/fixture/23/1/1.0.2.2 b/fixture/23/1/1.0.2.2 deleted file mode 100644 index aeeba5031d..0000000000 Binary files a/fixture/23/1/1.0.2.2 and /dev/null differ diff --git a/fixture/23/1/1.0.2.3 b/fixture/23/1/1.0.2.3 deleted file mode 100644 index 1b2fb6b86c..0000000000 Binary files a/fixture/23/1/1.0.2.3 and /dev/null differ diff --git a/fixture/23/1/1.0.3.0 b/fixture/23/1/1.0.3.0 deleted file mode 100644 index bb01a5b4f0..0000000000 Binary files a/fixture/23/1/1.0.3.0 and /dev/null differ diff --git a/fixture/23/1/1.0.3.1 b/fixture/23/1/1.0.3.1 deleted file mode 100644 index f7bdc6937a..0000000000 Binary files a/fixture/23/1/1.0.3.1 and /dev/null differ diff --git a/fixture/23/1/1.0.3.2 b/fixture/23/1/1.0.3.2 deleted file mode 100644 index 2e7553923f..0000000000 Binary files a/fixture/23/1/1.0.3.2 and /dev/null differ diff --git a/fixture/23/1/1.0.3.3 b/fixture/23/1/1.0.3.3 deleted file mode 100644 index 6a0ad4ffeb..0000000000 Binary files a/fixture/23/1/1.0.3.3 and /dev/null differ diff --git a/fixture/23/1/1.1.0.0 b/fixture/23/1/1.1.0.0 deleted file mode 100644 index 11919bdcff..0000000000 Binary files a/fixture/23/1/1.1.0.0 and /dev/null differ diff --git a/fixture/23/1/1.1.0.1 b/fixture/23/1/1.1.0.1 deleted file mode 100644 index 23388a5167..0000000000 Binary files a/fixture/23/1/1.1.0.1 and /dev/null differ diff --git a/fixture/23/1/1.1.0.2 b/fixture/23/1/1.1.0.2 deleted file mode 100644 index 05ac8f1ee8..0000000000 Binary files a/fixture/23/1/1.1.0.2 and /dev/null differ diff --git a/fixture/23/1/1.1.0.3 b/fixture/23/1/1.1.0.3 deleted file mode 100644 index c3a1a69eb4..0000000000 Binary files a/fixture/23/1/1.1.0.3 and /dev/null differ diff --git a/fixture/23/1/1.1.1.0 b/fixture/23/1/1.1.1.0 deleted file mode 100644 index 2efef30458..0000000000 Binary files a/fixture/23/1/1.1.1.0 and /dev/null differ diff --git a/fixture/23/1/1.1.1.1 b/fixture/23/1/1.1.1.1 deleted file mode 100644 index bd8dd14402..0000000000 Binary files a/fixture/23/1/1.1.1.1 and /dev/null differ diff --git a/fixture/23/1/1.1.1.2 b/fixture/23/1/1.1.1.2 deleted file mode 100644 index 085e779f5a..0000000000 Binary files a/fixture/23/1/1.1.1.2 and /dev/null differ diff --git a/fixture/23/1/1.1.1.3 b/fixture/23/1/1.1.1.3 deleted file mode 100644 index dd7567734d..0000000000 Binary files a/fixture/23/1/1.1.1.3 and /dev/null differ diff --git a/fixture/23/1/1.1.2.0 b/fixture/23/1/1.1.2.0 deleted file mode 100644 index 63dc924e74..0000000000 Binary files a/fixture/23/1/1.1.2.0 and /dev/null differ diff --git a/fixture/23/1/1.1.2.1 b/fixture/23/1/1.1.2.1 deleted file mode 100644 index 7c5445afac..0000000000 Binary files a/fixture/23/1/1.1.2.1 and /dev/null differ diff --git a/fixture/23/1/1.1.2.2 b/fixture/23/1/1.1.2.2 deleted file mode 100644 index 27fcd08b92..0000000000 Binary files a/fixture/23/1/1.1.2.2 and /dev/null differ diff --git a/fixture/23/1/1.1.2.3 b/fixture/23/1/1.1.2.3 deleted file mode 100644 index 0b92f63d18..0000000000 Binary files a/fixture/23/1/1.1.2.3 and /dev/null differ diff --git a/fixture/23/1/1.1.3.0 b/fixture/23/1/1.1.3.0 deleted file mode 100644 index b5d5d75829..0000000000 Binary files a/fixture/23/1/1.1.3.0 and /dev/null differ diff --git a/fixture/23/1/1.1.3.1 b/fixture/23/1/1.1.3.1 deleted file mode 100644 index b81b7bb917..0000000000 Binary files a/fixture/23/1/1.1.3.1 and /dev/null differ diff --git a/fixture/23/1/1.1.3.2 b/fixture/23/1/1.1.3.2 deleted file mode 100644 index 6acf2383f4..0000000000 Binary files a/fixture/23/1/1.1.3.2 and /dev/null differ diff --git a/fixture/23/1/1.1.3.3 b/fixture/23/1/1.1.3.3 deleted file mode 100644 index 3f1c37fd81..0000000000 Binary files a/fixture/23/1/1.1.3.3 and /dev/null differ diff --git a/fixture/23/1/1.2.0.0 b/fixture/23/1/1.2.0.0 deleted file mode 100644 index 21c3bba181..0000000000 Binary files a/fixture/23/1/1.2.0.0 and /dev/null differ diff --git a/fixture/23/1/1.2.0.1 b/fixture/23/1/1.2.0.1 deleted file mode 100644 index 730003bad1..0000000000 Binary files a/fixture/23/1/1.2.0.1 and /dev/null differ diff --git a/fixture/23/1/1.2.0.2 b/fixture/23/1/1.2.0.2 deleted file mode 100644 index bc46ae016c..0000000000 Binary files a/fixture/23/1/1.2.0.2 and /dev/null differ diff --git a/fixture/23/1/1.2.0.3 b/fixture/23/1/1.2.0.3 deleted file mode 100644 index 4fc192ae12..0000000000 Binary files a/fixture/23/1/1.2.0.3 and /dev/null differ diff --git a/fixture/23/1/1.2.1.0 b/fixture/23/1/1.2.1.0 deleted file mode 100644 index 00d512da18..0000000000 Binary files a/fixture/23/1/1.2.1.0 and /dev/null differ diff --git a/fixture/23/1/1.2.1.1 b/fixture/23/1/1.2.1.1 deleted file mode 100644 index 120c47c4c2..0000000000 Binary files a/fixture/23/1/1.2.1.1 and /dev/null differ diff --git a/fixture/23/1/1.2.1.2 b/fixture/23/1/1.2.1.2 deleted file mode 100644 index 6d368743d6..0000000000 Binary files a/fixture/23/1/1.2.1.2 and /dev/null differ diff --git a/fixture/23/1/1.2.1.3 b/fixture/23/1/1.2.1.3 deleted file mode 100644 index a3fee206b1..0000000000 Binary files a/fixture/23/1/1.2.1.3 and /dev/null differ diff --git a/fixture/23/1/1.2.2.0 b/fixture/23/1/1.2.2.0 deleted file mode 100644 index 216788e3ca..0000000000 Binary files a/fixture/23/1/1.2.2.0 and /dev/null differ diff --git a/fixture/23/1/1.2.2.1 b/fixture/23/1/1.2.2.1 deleted file mode 100644 index d04a4231f3..0000000000 Binary files a/fixture/23/1/1.2.2.1 and /dev/null differ diff --git a/fixture/23/1/1.2.2.2 b/fixture/23/1/1.2.2.2 deleted file mode 100644 index 33ae8e0153..0000000000 Binary files a/fixture/23/1/1.2.2.2 and /dev/null differ diff --git a/fixture/23/1/1.2.2.3 b/fixture/23/1/1.2.2.3 deleted file mode 100644 index 9b3dc9b7e6..0000000000 Binary files a/fixture/23/1/1.2.2.3 and /dev/null differ diff --git a/fixture/23/1/1.2.3.0 b/fixture/23/1/1.2.3.0 deleted file mode 100644 index 0de827b6c5..0000000000 Binary files a/fixture/23/1/1.2.3.0 and /dev/null differ diff --git a/fixture/23/1/1.2.3.1 b/fixture/23/1/1.2.3.1 deleted file mode 100644 index b687a3d431..0000000000 Binary files a/fixture/23/1/1.2.3.1 and /dev/null differ diff --git a/fixture/23/1/1.2.3.2 b/fixture/23/1/1.2.3.2 deleted file mode 100644 index e50f99a83a..0000000000 Binary files a/fixture/23/1/1.2.3.2 and /dev/null differ diff --git a/fixture/23/1/1.2.3.3 b/fixture/23/1/1.2.3.3 deleted file mode 100644 index 8014aa5179..0000000000 --- a/fixture/23/1/1.2.3.3 +++ /dev/null @@ -1 +0,0 @@ -x P@M~P#ʖid89•wz<x1f×YsH͉T:| \ No newline at end of file diff --git a/fixture/23/1/1.3.0.0 b/fixture/23/1/1.3.0.0 deleted file mode 100644 index 8b7c6e9377..0000000000 Binary files a/fixture/23/1/1.3.0.0 and /dev/null differ diff --git a/fixture/23/1/1.3.0.1 b/fixture/23/1/1.3.0.1 deleted file mode 100644 index e12851ec39..0000000000 Binary files a/fixture/23/1/1.3.0.1 and /dev/null differ diff --git a/fixture/23/1/1.3.0.2 b/fixture/23/1/1.3.0.2 deleted file mode 100644 index c3edd08ea3..0000000000 --- a/fixture/23/1/1.3.0.2 +++ /dev/null @@ -1 +0,0 @@ -xg2`ON$QR" v( nnp}f;so)rG{xDKi=s!Gt8&|JYrYdeV(P$NN8s.k+DUXB-40#2Fq dmaIfY'^y>b7Y46 \ No newline at end of file diff --git a/fixture/23/1/1.3.0.3 b/fixture/23/1/1.3.0.3 deleted file mode 100644 index 33e57a0ae4..0000000000 Binary files a/fixture/23/1/1.3.0.3 and /dev/null differ diff --git a/fixture/23/1/1.3.1.0 b/fixture/23/1/1.3.1.0 deleted file mode 100644 index e3fab0e36b..0000000000 --- a/fixture/23/1/1.3.1.0 +++ /dev/null @@ -1 +0,0 @@ -xW2`+qQF!Zd'!Nt lynlJnzK 0Ht6y^yulNimve}(sHt/|/DWǜpT ֔f3,H%%U['B?,*kA u41"Y8 \ No newline at end of file diff --git a/fixture/23/1/1.3.1.1 b/fixture/23/1/1.3.1.1 deleted file mode 100644 index 1c3d2c6049..0000000000 --- a/fixture/23/1/1.3.1.1 +++ /dev/null @@ -1 +0,0 @@ -xQPEW P0vc`ЍYȯg98cN8s.n)2+Rcu4 \JÌ0J,93Nt?l6;4i0$LQ4=|^y\fPeyX$}>~c@tjJ\q nN6R \ No newline at end of file diff --git a/fixture/23/1/1.3.1.2 b/fixture/23/1/1.3.1.2 deleted file mode 100644 index c686f9d5ee..0000000000 Binary files a/fixture/23/1/1.3.1.2 and /dev/null differ diff --git a/fixture/23/1/1.3.1.3 b/fixture/23/1/1.3.1.3 deleted file mode 100644 index d39734f63d..0000000000 Binary files a/fixture/23/1/1.3.1.3 and /dev/null differ diff --git a/fixture/23/1/1.3.2.0 b/fixture/23/1/1.3.2.0 deleted file mode 100644 index 0c7fd416e0..0000000000 --- a/fixture/23/1/1.3.2.0 +++ /dev/null @@ -1 +0,0 @@ -xQBQEkaNsF؍Y1t.{{fN!]C8SRRcYNe[L~@F%Xa5`-.aaFe9Dwgy^i!L2430KtmN>*)59\q D(~ \ No newline at end of file diff --git a/fixture/23/1/1.3.2.1 b/fixture/23/1/1.3.2.1 deleted file mode 100644 index d9ba37a9a8..0000000000 Binary files a/fixture/23/1/1.3.2.1 and /dev/null differ diff --git a/fixture/23/1/1.3.2.2 b/fixture/23/1/1.3.2.2 deleted file mode 100644 index e7c7cd1f2d..0000000000 Binary files a/fixture/23/1/1.3.2.2 and /dev/null differ diff --git a/fixture/23/1/1.3.2.3 b/fixture/23/1/1.3.2.3 deleted file mode 100644 index 5d7cec912f..0000000000 Binary files a/fixture/23/1/1.3.2.3 and /dev/null differ diff --git a/fixture/23/1/1.3.3.0 b/fixture/23/1/1.3.3.0 deleted file mode 100644 index 43e394eabf..0000000000 --- a/fixture/23/1/1.3.3.0 +++ /dev/null @@ -1,2 +0,0 @@ -x7PQRܐrzxߙc`WOڰ 7}A=0cv 1Ȑ@ -.ہ \ No newline at end of file diff --git a/fixture/23/1/1.3.3.1 b/fixture/23/1/1.3.3.1 deleted file mode 100644 index d22239bb91..0000000000 --- a/fixture/23/1/1.3.3.1 +++ /dev/null @@ -1 +0,0 @@ -x7P[J:Jro^wغc 3lR D RdX+iÎ'.xv \ No newline at end of file diff --git a/fixture/23/1/1.3.3.2 b/fixture/23/1/1.3.3.2 deleted file mode 100644 index 017470d76c..0000000000 --- a/fixture/23/1/1.3.3.2 +++ /dev/null @@ -1 +0,0 @@ -xз@Э t7-.y h ;p߁:c 3X~"H"C%*X~~ \ No newline at end of file diff --git a/fixture/23/1/1.3.3.3 b/fixture/23/1/1.3.3.3 deleted file mode 100644 index 784d171cf0..0000000000 Binary files a/fixture/23/1/1.3.3.3 and /dev/null differ diff --git a/fixture/23/2/.zarray b/fixture/23/2/.zarray deleted file mode 100644 index 06a917041e..0000000000 --- a/fixture/23/2/.zarray +++ /dev/null @@ -1,23 +0,0 @@ -{ - "chunks": [ - 10, - 3, - 3, - 3 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "ZoM_ b@B\K H*)mS,NQ \ No newline at end of file diff --git a/fixture/4/0/13 b/fixture/4/0/13 deleted file mode 100644 index f57b26d6c3..0000000000 --- a/fixture/4/0/13 +++ /dev/null @@ -1 +0,0 @@ -y.Q|5 rÔi9o -`QC [ \ No newline at end of file diff --git a/fixture/4/0/14 b/fixture/4/0/14 deleted file mode 100644 index f94817138a..0000000000 --- a/fixture/4/0/14 +++ /dev/null @@ -1 +0,0 @@ -%8l d2"|)]Mx4oxY`~%n>~:q1k:ŠY|ZIX9@9zC \ No newline at end of file diff --git a/fixture/4/0/18 b/fixture/4/0/18 deleted file mode 100644 index 4c620ed93e..0000000000 --- a/fixture/4/0/18 +++ /dev/null @@ -1 +0,0 @@ -U0y@  EM5t5\;~ djS94L;E @7kufTuTl; 1s]}Ni.gb;6.k#3! \ No newline at end of file diff --git a/fixture/4/0/19 b/fixture/4/0/19 deleted file mode 100644 index e92f942c4f..0000000000 Binary files a/fixture/4/0/19 and /dev/null differ diff --git a/fixture/4/0/2 b/fixture/4/0/2 deleted file mode 100644 index 883c1ade1c..0000000000 --- a/fixture/4/0/2 +++ /dev/null @@ -1,2 +0,0 @@ -"4;3 3 b+MY]4 -;S)[=np[=(~CL"a,8Pdg?B1.g]_LOl-#5;5i4T5 \ No newline at end of file diff --git a/fixture/4/0/20 b/fixture/4/0/20 deleted file mode 100644 index 6898070345..0000000000 --- a/fixture/4/0/20 +++ /dev/null @@ -1 +0,0 @@ -b=}u/¶XN.BtGe r4Fr8%DBo\jk7 KLk?![ (3UJmW$D . \ No newline at end of file diff --git a/fixture/4/0/21 b/fixture/4/0/21 deleted file mode 100644 index c51a1f4cd4..0000000000 Binary files a/fixture/4/0/21 and /dev/null differ diff --git a/fixture/4/0/22 b/fixture/4/0/22 deleted file mode 100644 index 766adbc56d..0000000000 Binary files a/fixture/4/0/22 and /dev/null differ diff --git a/fixture/4/0/3 b/fixture/4/0/3 deleted file mode 100644 index 346d3e3007..0000000000 Binary files a/fixture/4/0/3 and /dev/null differ diff --git a/fixture/4/0/4 b/fixture/4/0/4 deleted file mode 100644 index 1cffb0e029..0000000000 Binary files a/fixture/4/0/4 and /dev/null differ diff --git a/fixture/4/0/5 b/fixture/4/0/5 deleted file mode 100644 index e1cc79cb85..0000000000 Binary files a/fixture/4/0/5 and /dev/null differ diff --git a/fixture/4/0/6 b/fixture/4/0/6 deleted file mode 100644 index 333191f0cc..0000000000 Binary files a/fixture/4/0/6 and /dev/null differ diff --git a/fixture/4/0/7 b/fixture/4/0/7 deleted file mode 100644 index 670ef1274e..0000000000 Binary files a/fixture/4/0/7 and /dev/null differ diff --git a/fixture/4/0/8 b/fixture/4/0/8 deleted file mode 100644 index 12b9e4c69a..0000000000 Binary files a/fixture/4/0/8 and /dev/null differ diff --git a/fixture/4/0/9 b/fixture/4/0/9 deleted file mode 100644 index 7d5922cd9d..0000000000 --- a/fixture/4/0/9 +++ /dev/null @@ -1 +0,0 @@ - p9i5- Vv_UGVXdS=}B%4TL}rSfU=C·^z<1=bl% '^Dexsa \ No newline at end of file diff --git a/fixture/4/1/.zarray b/fixture/4/1/.zarray deleted file mode 100644 index 6a051baada..0000000000 --- a/fixture/4/1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "|u1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 2222 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/4/1/.zattrs b/fixture/4/1/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/4/1/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/4/1/0 b/fixture/4/1/0 deleted file mode 100644 index e9f97dbccf..0000000000 Binary files a/fixture/4/1/0 and /dev/null differ diff --git a/fixture/4/1/1 b/fixture/4/1/1 deleted file mode 100644 index f0f6c39dd9..0000000000 Binary files a/fixture/4/1/1 and /dev/null differ diff --git a/fixture/4/1/10 b/fixture/4/1/10 deleted file mode 100644 index 1f36297a0a..0000000000 Binary files a/fixture/4/1/10 and /dev/null differ diff --git a/fixture/4/1/11 b/fixture/4/1/11 deleted file mode 100644 index c86df746da..0000000000 Binary files a/fixture/4/1/11 and /dev/null differ diff --git a/fixture/4/1/12 b/fixture/4/1/12 deleted file mode 100644 index 6fa13f610d..0000000000 Binary files a/fixture/4/1/12 and /dev/null differ diff --git a/fixture/4/1/13 b/fixture/4/1/13 deleted file mode 100644 index 248ae49efb..0000000000 Binary files a/fixture/4/1/13 and /dev/null differ diff --git a/fixture/4/1/14 b/fixture/4/1/14 deleted file mode 100644 index e81a6f9f17..0000000000 Binary files a/fixture/4/1/14 and /dev/null differ diff --git a/fixture/4/1/15 b/fixture/4/1/15 deleted file mode 100644 index d376431198..0000000000 Binary files a/fixture/4/1/15 and /dev/null differ diff --git a/fixture/4/1/16 b/fixture/4/1/16 deleted file mode 100644 index a677008367..0000000000 Binary files a/fixture/4/1/16 and /dev/null differ diff --git a/fixture/4/1/17 b/fixture/4/1/17 deleted file mode 100644 index ce8b23a238..0000000000 Binary files a/fixture/4/1/17 and /dev/null differ diff --git a/fixture/4/1/18 b/fixture/4/1/18 deleted file mode 100644 index a7f3d248d9..0000000000 Binary files a/fixture/4/1/18 and /dev/null differ diff --git a/fixture/4/1/19 b/fixture/4/1/19 deleted file mode 100644 index 7312311c52..0000000000 Binary files a/fixture/4/1/19 and /dev/null differ diff --git a/fixture/4/1/2 b/fixture/4/1/2 deleted file mode 100644 index b7920e4704..0000000000 Binary files a/fixture/4/1/2 and /dev/null differ diff --git a/fixture/4/1/20 b/fixture/4/1/20 deleted file mode 100644 index e4f8c115e3..0000000000 Binary files a/fixture/4/1/20 and /dev/null differ diff --git a/fixture/4/1/21 b/fixture/4/1/21 deleted file mode 100644 index 35225d4218..0000000000 Binary files a/fixture/4/1/21 and /dev/null differ diff --git a/fixture/4/1/22 b/fixture/4/1/22 deleted file mode 100644 index b8b5e078b7..0000000000 Binary files a/fixture/4/1/22 and /dev/null differ diff --git a/fixture/4/1/3 b/fixture/4/1/3 deleted file mode 100644 index 666dedbbf4..0000000000 Binary files a/fixture/4/1/3 and /dev/null differ diff --git a/fixture/4/1/4 b/fixture/4/1/4 deleted file mode 100644 index cfee6f71cc..0000000000 Binary files a/fixture/4/1/4 and /dev/null differ diff --git a/fixture/4/1/5 b/fixture/4/1/5 deleted file mode 100644 index d949a1f1d8..0000000000 Binary files a/fixture/4/1/5 and /dev/null differ diff --git a/fixture/4/1/6 b/fixture/4/1/6 deleted file mode 100644 index ff98a66fe2..0000000000 Binary files a/fixture/4/1/6 and /dev/null differ diff --git a/fixture/4/1/7 b/fixture/4/1/7 deleted file mode 100644 index 1890e57b24..0000000000 Binary files a/fixture/4/1/7 and /dev/null differ diff --git a/fixture/4/1/8 b/fixture/4/1/8 deleted file mode 100644 index 65040779fd..0000000000 Binary files a/fixture/4/1/8 and /dev/null differ diff --git a/fixture/4/1/9 b/fixture/4/1/9 deleted file mode 100644 index 4afeef2025..0000000000 Binary files a/fixture/4/1/9 and /dev/null differ diff --git a/fixture/4/2/.zarray b/fixture/4/2/.zarray deleted file mode 100644 index eb1c0d802a..0000000000 --- a/fixture/4/2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "|u1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 2222 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/4/2/.zattrs b/fixture/4/2/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/4/2/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/4/2/0 b/fixture/4/2/0 deleted file mode 100644 index 6267e2504e..0000000000 Binary files a/fixture/4/2/0 and /dev/null differ diff --git a/fixture/4/2/1 b/fixture/4/2/1 deleted file mode 100644 index f8ae66dafa..0000000000 Binary files a/fixture/4/2/1 and /dev/null differ diff --git a/fixture/4/2/10 b/fixture/4/2/10 deleted file mode 100644 index f812d0bf37..0000000000 Binary files a/fixture/4/2/10 and /dev/null differ diff --git a/fixture/4/2/11 b/fixture/4/2/11 deleted file mode 100644 index 340e17990f..0000000000 Binary files a/fixture/4/2/11 and /dev/null differ diff --git a/fixture/4/2/12 b/fixture/4/2/12 deleted file mode 100644 index 28e7a19856..0000000000 Binary files a/fixture/4/2/12 and /dev/null differ diff --git a/fixture/4/2/13 b/fixture/4/2/13 deleted file mode 100644 index 410e633554..0000000000 Binary files a/fixture/4/2/13 and /dev/null differ diff --git a/fixture/4/2/14 b/fixture/4/2/14 deleted file mode 100644 index e50e9078b5..0000000000 Binary files a/fixture/4/2/14 and /dev/null differ diff --git a/fixture/4/2/15 b/fixture/4/2/15 deleted file mode 100644 index 522c940b87..0000000000 Binary files a/fixture/4/2/15 and /dev/null differ diff --git a/fixture/4/2/16 b/fixture/4/2/16 deleted file mode 100644 index cd18162164..0000000000 Binary files a/fixture/4/2/16 and /dev/null differ diff --git a/fixture/4/2/17 b/fixture/4/2/17 deleted file mode 100644 index c529a3217d..0000000000 Binary files a/fixture/4/2/17 and /dev/null differ diff --git a/fixture/4/2/18 b/fixture/4/2/18 deleted file mode 100644 index aecf6d89ed..0000000000 Binary files a/fixture/4/2/18 and /dev/null differ diff --git a/fixture/4/2/19 b/fixture/4/2/19 deleted file mode 100644 index 918edeb112..0000000000 Binary files a/fixture/4/2/19 and /dev/null differ diff --git a/fixture/4/2/2 b/fixture/4/2/2 deleted file mode 100644 index 5d1954653b..0000000000 Binary files a/fixture/4/2/2 and /dev/null differ diff --git a/fixture/4/2/20 b/fixture/4/2/20 deleted file mode 100644 index bc3cc5965e..0000000000 Binary files a/fixture/4/2/20 and /dev/null differ diff --git a/fixture/4/2/21 b/fixture/4/2/21 deleted file mode 100644 index 72e6bf012c..0000000000 Binary files a/fixture/4/2/21 and /dev/null differ diff --git a/fixture/4/2/22 b/fixture/4/2/22 deleted file mode 100644 index 317f843a59..0000000000 Binary files a/fixture/4/2/22 and /dev/null differ diff --git a/fixture/4/2/3 b/fixture/4/2/3 deleted file mode 100644 index 2900b3e95a..0000000000 Binary files a/fixture/4/2/3 and /dev/null differ diff --git a/fixture/4/2/4 b/fixture/4/2/4 deleted file mode 100644 index 1f799131df..0000000000 Binary files a/fixture/4/2/4 and /dev/null differ diff --git a/fixture/4/2/5 b/fixture/4/2/5 deleted file mode 100644 index 8810cf3345..0000000000 Binary files a/fixture/4/2/5 and /dev/null differ diff --git a/fixture/4/2/6 b/fixture/4/2/6 deleted file mode 100644 index f69c6e4cb6..0000000000 Binary files a/fixture/4/2/6 and /dev/null differ diff --git a/fixture/4/2/7 b/fixture/4/2/7 deleted file mode 100644 index 01083e08c7..0000000000 Binary files a/fixture/4/2/7 and /dev/null differ diff --git a/fixture/4/2/8 b/fixture/4/2/8 deleted file mode 100644 index 5ef12531ba..0000000000 Binary files a/fixture/4/2/8 and /dev/null differ diff --git a/fixture/4/2/9 b/fixture/4/2/9 deleted file mode 100644 index 227a54a717..0000000000 Binary files a/fixture/4/2/9 and /dev/null differ diff --git a/fixture/4/3/.zarray b/fixture/4/3/.zarray deleted file mode 100644 index cd1f621720..0000000000 --- a/fixture/4/3/.zarray +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 0 - }, - "dtype": "|u1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 2222 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/4/3/.zattrs b/fixture/4/3/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/4/3/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/4/3/0 b/fixture/4/3/0 deleted file mode 100644 index f540c771ad..0000000000 Binary files a/fixture/4/3/0 and /dev/null differ diff --git a/fixture/4/3/1 b/fixture/4/3/1 deleted file mode 100644 index 79f1509b01..0000000000 Binary files a/fixture/4/3/1 and /dev/null differ diff --git a/fixture/4/3/10 b/fixture/4/3/10 deleted file mode 100644 index 367520fe2a..0000000000 Binary files a/fixture/4/3/10 and /dev/null differ diff --git a/fixture/4/3/11 b/fixture/4/3/11 deleted file mode 100644 index 512026eee9..0000000000 Binary files a/fixture/4/3/11 and /dev/null differ diff --git a/fixture/4/3/12 b/fixture/4/3/12 deleted file mode 100644 index 87ad9c1094..0000000000 Binary files a/fixture/4/3/12 and /dev/null differ diff --git a/fixture/4/3/13 b/fixture/4/3/13 deleted file mode 100644 index de77dc821c..0000000000 Binary files a/fixture/4/3/13 and /dev/null differ diff --git a/fixture/4/3/14 b/fixture/4/3/14 deleted file mode 100644 index 9d53c1a1c8..0000000000 Binary files a/fixture/4/3/14 and /dev/null differ diff --git a/fixture/4/3/15 b/fixture/4/3/15 deleted file mode 100644 index 1ab24bde60..0000000000 Binary files a/fixture/4/3/15 and /dev/null differ diff --git a/fixture/4/3/16 b/fixture/4/3/16 deleted file mode 100644 index a58272c3f6..0000000000 Binary files a/fixture/4/3/16 and /dev/null differ diff --git a/fixture/4/3/17 b/fixture/4/3/17 deleted file mode 100644 index 7826a65021..0000000000 Binary files a/fixture/4/3/17 and /dev/null differ diff --git a/fixture/4/3/18 b/fixture/4/3/18 deleted file mode 100644 index e1b745de02..0000000000 Binary files a/fixture/4/3/18 and /dev/null differ diff --git a/fixture/4/3/19 b/fixture/4/3/19 deleted file mode 100644 index b04e74d383..0000000000 Binary files a/fixture/4/3/19 and /dev/null differ diff --git a/fixture/4/3/2 b/fixture/4/3/2 deleted file mode 100644 index 9e2e6718e0..0000000000 Binary files a/fixture/4/3/2 and /dev/null differ diff --git a/fixture/4/3/20 b/fixture/4/3/20 deleted file mode 100644 index 426218fcea..0000000000 Binary files a/fixture/4/3/20 and /dev/null differ diff --git a/fixture/4/3/21 b/fixture/4/3/21 deleted file mode 100644 index 817a22bedc..0000000000 Binary files a/fixture/4/3/21 and /dev/null differ diff --git a/fixture/4/3/22 b/fixture/4/3/22 deleted file mode 100644 index 9b28ee38a0..0000000000 Binary files a/fixture/4/3/22 and /dev/null differ diff --git a/fixture/4/3/3 b/fixture/4/3/3 deleted file mode 100644 index e53f4402ba..0000000000 Binary files a/fixture/4/3/3 and /dev/null differ diff --git a/fixture/4/3/4 b/fixture/4/3/4 deleted file mode 100644 index 29149ad662..0000000000 Binary files a/fixture/4/3/4 and /dev/null differ diff --git a/fixture/4/3/5 b/fixture/4/3/5 deleted file mode 100644 index 3c7239e8f9..0000000000 Binary files a/fixture/4/3/5 and /dev/null differ diff --git a/fixture/4/3/6 b/fixture/4/3/6 deleted file mode 100644 index b02e381c2b..0000000000 Binary files a/fixture/4/3/6 and /dev/null differ diff --git a/fixture/4/3/7 b/fixture/4/3/7 deleted file mode 100644 index a8fc37a9df..0000000000 Binary files a/fixture/4/3/7 and /dev/null differ diff --git a/fixture/4/3/8 b/fixture/4/3/8 deleted file mode 100644 index a13db20a57..0000000000 Binary files a/fixture/4/3/8 and /dev/null differ diff --git a/fixture/4/3/9 b/fixture/4/3/9 deleted file mode 100644 index 4f2162de29..0000000000 Binary files a/fixture/4/3/9 and /dev/null differ diff --git a/fixture/4/4/.zarray b/fixture/4/4/.zarray deleted file mode 100644 index d89a6b2a47..0000000000 --- a/fixture/4/4/.zarray +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "|u1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 2222 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/4/4/.zattrs b/fixture/4/4/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/4/4/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/4/4/0 b/fixture/4/4/0 deleted file mode 100644 index 0cac20d7fc..0000000000 Binary files a/fixture/4/4/0 and /dev/null differ diff --git a/fixture/4/4/1 b/fixture/4/4/1 deleted file mode 100644 index a5f4ed368e..0000000000 Binary files a/fixture/4/4/1 and /dev/null differ diff --git a/fixture/4/4/10 b/fixture/4/4/10 deleted file mode 100644 index 1290670917..0000000000 Binary files a/fixture/4/4/10 and /dev/null differ diff --git a/fixture/4/4/11 b/fixture/4/4/11 deleted file mode 100644 index e2fea879dd..0000000000 Binary files a/fixture/4/4/11 and /dev/null differ diff --git a/fixture/4/4/12 b/fixture/4/4/12 deleted file mode 100644 index ab92dea000..0000000000 Binary files a/fixture/4/4/12 and /dev/null differ diff --git a/fixture/4/4/13 b/fixture/4/4/13 deleted file mode 100644 index a4ceee43eb..0000000000 Binary files a/fixture/4/4/13 and /dev/null differ diff --git a/fixture/4/4/14 b/fixture/4/4/14 deleted file mode 100644 index c75c32350d..0000000000 Binary files a/fixture/4/4/14 and /dev/null differ diff --git a/fixture/4/4/15 b/fixture/4/4/15 deleted file mode 100644 index 00f5a189c2..0000000000 Binary files a/fixture/4/4/15 and /dev/null differ diff --git a/fixture/4/4/16 b/fixture/4/4/16 deleted file mode 100644 index d9e34816ff..0000000000 Binary files a/fixture/4/4/16 and /dev/null differ diff --git a/fixture/4/4/17 b/fixture/4/4/17 deleted file mode 100644 index 8ed0d466f8..0000000000 Binary files a/fixture/4/4/17 and /dev/null differ diff --git a/fixture/4/4/18 b/fixture/4/4/18 deleted file mode 100644 index 94c28d9cb7..0000000000 Binary files a/fixture/4/4/18 and /dev/null differ diff --git a/fixture/4/4/19 b/fixture/4/4/19 deleted file mode 100644 index 3a772ffd25..0000000000 Binary files a/fixture/4/4/19 and /dev/null differ diff --git a/fixture/4/4/2 b/fixture/4/4/2 deleted file mode 100644 index f6280fe613..0000000000 Binary files a/fixture/4/4/2 and /dev/null differ diff --git a/fixture/4/4/20 b/fixture/4/4/20 deleted file mode 100644 index 5266e2804f..0000000000 Binary files a/fixture/4/4/20 and /dev/null differ diff --git a/fixture/4/4/21 b/fixture/4/4/21 deleted file mode 100644 index 1197e78ef2..0000000000 Binary files a/fixture/4/4/21 and /dev/null differ diff --git a/fixture/4/4/22 b/fixture/4/4/22 deleted file mode 100644 index 54cb6fb413..0000000000 Binary files a/fixture/4/4/22 and /dev/null differ diff --git a/fixture/4/4/3 b/fixture/4/4/3 deleted file mode 100644 index 67875d77bf..0000000000 Binary files a/fixture/4/4/3 and /dev/null differ diff --git a/fixture/4/4/4 b/fixture/4/4/4 deleted file mode 100644 index 79161bdd5a..0000000000 Binary files a/fixture/4/4/4 and /dev/null differ diff --git a/fixture/4/4/5 b/fixture/4/4/5 deleted file mode 100644 index ac780b8917..0000000000 Binary files a/fixture/4/4/5 and /dev/null differ diff --git a/fixture/4/4/6 b/fixture/4/4/6 deleted file mode 100644 index e1c7827214..0000000000 Binary files a/fixture/4/4/6 and /dev/null differ diff --git a/fixture/4/4/7 b/fixture/4/4/7 deleted file mode 100644 index eea6cedd1d..0000000000 Binary files a/fixture/4/4/7 and /dev/null differ diff --git a/fixture/4/4/8 b/fixture/4/4/8 deleted file mode 100644 index 633dc42bcd..0000000000 Binary files a/fixture/4/4/8 and /dev/null differ diff --git a/fixture/4/4/9 b/fixture/4/4/9 deleted file mode 100644 index d788950673..0000000000 Binary files a/fixture/4/4/9 and /dev/null differ diff --git a/fixture/4/5/.zarray b/fixture/4/5/.zarray deleted file mode 100644 index 683afb9240..0000000000 --- a/fixture/4/5/.zarray +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 2 - }, - "dtype": "|u1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 2222 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/4/5/.zattrs b/fixture/4/5/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/4/5/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/4/5/0 b/fixture/4/5/0 deleted file mode 100644 index 662a0586a0..0000000000 Binary files a/fixture/4/5/0 and /dev/null differ diff --git a/fixture/4/5/1 b/fixture/4/5/1 deleted file mode 100644 index b248541fb4..0000000000 Binary files a/fixture/4/5/1 and /dev/null differ diff --git a/fixture/4/5/10 b/fixture/4/5/10 deleted file mode 100644 index a8b0e6d582..0000000000 Binary files a/fixture/4/5/10 and /dev/null differ diff --git a/fixture/4/5/11 b/fixture/4/5/11 deleted file mode 100644 index 52abc642ab..0000000000 Binary files a/fixture/4/5/11 and /dev/null differ diff --git a/fixture/4/5/12 b/fixture/4/5/12 deleted file mode 100644 index ed36238f59..0000000000 Binary files a/fixture/4/5/12 and /dev/null differ diff --git a/fixture/4/5/13 b/fixture/4/5/13 deleted file mode 100644 index de4dad839c..0000000000 Binary files a/fixture/4/5/13 and /dev/null differ diff --git a/fixture/4/5/14 b/fixture/4/5/14 deleted file mode 100644 index e228940b9b..0000000000 Binary files a/fixture/4/5/14 and /dev/null differ diff --git a/fixture/4/5/15 b/fixture/4/5/15 deleted file mode 100644 index 4d6eaa3bc2..0000000000 Binary files a/fixture/4/5/15 and /dev/null differ diff --git a/fixture/4/5/16 b/fixture/4/5/16 deleted file mode 100644 index 6ccf089153..0000000000 Binary files a/fixture/4/5/16 and /dev/null differ diff --git a/fixture/4/5/17 b/fixture/4/5/17 deleted file mode 100644 index 4717299646..0000000000 Binary files a/fixture/4/5/17 and /dev/null differ diff --git a/fixture/4/5/18 b/fixture/4/5/18 deleted file mode 100644 index c06d67649d..0000000000 Binary files a/fixture/4/5/18 and /dev/null differ diff --git a/fixture/4/5/19 b/fixture/4/5/19 deleted file mode 100644 index 53cde70394..0000000000 Binary files a/fixture/4/5/19 and /dev/null differ diff --git a/fixture/4/5/2 b/fixture/4/5/2 deleted file mode 100644 index 6283cd6544..0000000000 Binary files a/fixture/4/5/2 and /dev/null differ diff --git a/fixture/4/5/20 b/fixture/4/5/20 deleted file mode 100644 index 9f0339c8a2..0000000000 Binary files a/fixture/4/5/20 and /dev/null differ diff --git a/fixture/4/5/21 b/fixture/4/5/21 deleted file mode 100644 index a5bd479d51..0000000000 Binary files a/fixture/4/5/21 and /dev/null differ diff --git a/fixture/4/5/22 b/fixture/4/5/22 deleted file mode 100644 index bf6d3f0788..0000000000 Binary files a/fixture/4/5/22 and /dev/null differ diff --git a/fixture/4/5/3 b/fixture/4/5/3 deleted file mode 100644 index 9e7a719eb0..0000000000 Binary files a/fixture/4/5/3 and /dev/null differ diff --git a/fixture/4/5/4 b/fixture/4/5/4 deleted file mode 100644 index 81dee86fe9..0000000000 Binary files a/fixture/4/5/4 and /dev/null differ diff --git a/fixture/4/5/5 b/fixture/4/5/5 deleted file mode 100644 index feb0e18f90..0000000000 Binary files a/fixture/4/5/5 and /dev/null differ diff --git a/fixture/4/5/6 b/fixture/4/5/6 deleted file mode 100644 index 08e2f1edcc..0000000000 Binary files a/fixture/4/5/6 and /dev/null differ diff --git a/fixture/4/5/7 b/fixture/4/5/7 deleted file mode 100644 index 2ac54b8369..0000000000 Binary files a/fixture/4/5/7 and /dev/null differ diff --git a/fixture/4/5/8 b/fixture/4/5/8 deleted file mode 100644 index af53e544f8..0000000000 Binary files a/fixture/4/5/8 and /dev/null differ diff --git a/fixture/4/5/9 b/fixture/4/5/9 deleted file mode 100644 index 6ef6b159b8..0000000000 Binary files a/fixture/4/5/9 and /dev/null differ diff --git a/fixture/4/6/.zarray b/fixture/4/6/.zarray deleted file mode 100644 index 8489e95909..0000000000 --- a/fixture/4/6/.zarray +++ /dev/null @@ -1,19 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "clevel": 1, - "cname": "lz4", - "id": "blosc", - "shuffle": 0 - }, - "dtype": "|u1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 2222 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/4/6/.zattrs b/fixture/4/6/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/4/6/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/4/6/0 b/fixture/4/6/0 deleted file mode 100644 index 3a1d0df7a2..0000000000 Binary files a/fixture/4/6/0 and /dev/null differ diff --git a/fixture/4/6/1 b/fixture/4/6/1 deleted file mode 100644 index 790c8f8b15..0000000000 Binary files a/fixture/4/6/1 and /dev/null differ diff --git a/fixture/4/6/10 b/fixture/4/6/10 deleted file mode 100644 index 839dd2d2d0..0000000000 Binary files a/fixture/4/6/10 and /dev/null differ diff --git a/fixture/4/6/11 b/fixture/4/6/11 deleted file mode 100644 index 30b1abffc6..0000000000 Binary files a/fixture/4/6/11 and /dev/null differ diff --git a/fixture/4/6/12 b/fixture/4/6/12 deleted file mode 100644 index 70a9c95a8b..0000000000 Binary files a/fixture/4/6/12 and /dev/null differ diff --git a/fixture/4/6/13 b/fixture/4/6/13 deleted file mode 100644 index 1e0c2a7a9a..0000000000 Binary files a/fixture/4/6/13 and /dev/null differ diff --git a/fixture/4/6/14 b/fixture/4/6/14 deleted file mode 100644 index 00deb5e1cc..0000000000 Binary files a/fixture/4/6/14 and /dev/null differ diff --git a/fixture/4/6/15 b/fixture/4/6/15 deleted file mode 100644 index fad2921266..0000000000 Binary files a/fixture/4/6/15 and /dev/null differ diff --git a/fixture/4/6/16 b/fixture/4/6/16 deleted file mode 100644 index 7997cda633..0000000000 Binary files a/fixture/4/6/16 and /dev/null differ diff --git a/fixture/4/6/17 b/fixture/4/6/17 deleted file mode 100644 index b0fbdbeacb..0000000000 Binary files a/fixture/4/6/17 and /dev/null differ diff --git a/fixture/4/6/18 b/fixture/4/6/18 deleted file mode 100644 index f012ed4200..0000000000 Binary files a/fixture/4/6/18 and /dev/null differ diff --git a/fixture/4/6/19 b/fixture/4/6/19 deleted file mode 100644 index f9cb235bf3..0000000000 Binary files a/fixture/4/6/19 and /dev/null differ diff --git a/fixture/4/6/2 b/fixture/4/6/2 deleted file mode 100644 index 040c088e5c..0000000000 Binary files a/fixture/4/6/2 and /dev/null differ diff --git a/fixture/4/6/20 b/fixture/4/6/20 deleted file mode 100644 index af3e2f1bba..0000000000 Binary files a/fixture/4/6/20 and /dev/null differ diff --git a/fixture/4/6/21 b/fixture/4/6/21 deleted file mode 100644 index f7a1749aec..0000000000 Binary files a/fixture/4/6/21 and /dev/null differ diff --git a/fixture/4/6/22 b/fixture/4/6/22 deleted file mode 100644 index 265130bbd9..0000000000 Binary files a/fixture/4/6/22 and /dev/null differ diff --git a/fixture/4/6/3 b/fixture/4/6/3 deleted file mode 100644 index 597b7c2787..0000000000 Binary files a/fixture/4/6/3 and /dev/null differ diff --git a/fixture/4/6/4 b/fixture/4/6/4 deleted file mode 100644 index 8500f6a21a..0000000000 Binary files a/fixture/4/6/4 and /dev/null differ diff --git a/fixture/4/6/5 b/fixture/4/6/5 deleted file mode 100644 index 3b747cce85..0000000000 Binary files a/fixture/4/6/5 and /dev/null differ diff --git a/fixture/4/6/6 b/fixture/4/6/6 deleted file mode 100644 index 51e8466415..0000000000 Binary files a/fixture/4/6/6 and /dev/null differ diff --git a/fixture/4/6/7 b/fixture/4/6/7 deleted file mode 100644 index 17752969b6..0000000000 Binary files a/fixture/4/6/7 and /dev/null differ diff --git a/fixture/4/6/8 b/fixture/4/6/8 deleted file mode 100644 index a8f7fd328a..0000000000 Binary files a/fixture/4/6/8 and /dev/null differ diff --git a/fixture/4/6/9 b/fixture/4/6/9 deleted file mode 100644 index 8bbde74ea2..0000000000 Binary files a/fixture/4/6/9 and /dev/null differ diff --git a/fixture/5/.zattrs b/fixture/5/.zattrs deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/fixture/5/.zattrs +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/fixture/5/.zgroup b/fixture/5/.zgroup deleted file mode 100644 index 3b7daf227c..0000000000 --- a/fixture/5/.zgroup +++ /dev/null @@ -1,3 +0,0 @@ -{ - "zarr_format": 2 -} \ No newline at end of file diff --git a/fixture/5/0/.zarray b/fixture/5/0/.zarray deleted file mode 100644 index 8b3e48ffe7..0000000000 --- a/fixture/5/0/.zarray +++ /dev/null @@ -1,14 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": null, - "dtype": "o?^"~{OňN_D>'#^ -[uoyO^ү[3@9KSx_6= %\q7z5e/iūTeY f,[tƺlZɇtxO',׻7\Yο_=<;+܅?s4 \ No newline at end of file diff --git a/fixture/6/1/15 b/fixture/6/1/15 deleted file mode 100644 index d816848355..0000000000 Binary files a/fixture/6/1/15 and /dev/null differ diff --git a/fixture/6/1/16 b/fixture/6/1/16 deleted file mode 100644 index 69f3f7e6b8..0000000000 --- a/fixture/6/1/16 +++ /dev/null @@ -1,2 +0,0 @@ -x?(q_`E,7n`ԕA1[OpA d+$a2] )w~><;Q-z"t艘ވG.5nnNݔ5=6ln_IgBߚlX+Oڋj33sc -$رK5 Qܦj:?gžQ~ߜ:ꂷ|FΟ\]yl} ;e7|E"ege-vqx6 \ No newline at end of file diff --git a/fixture/6/1/17 b/fixture/6/1/17 deleted file mode 100644 index 5bb69635fd..0000000000 --- a/fixture/6/1/17 +++ /dev/null @@ -1,2 +0,0 @@ -xο+q:I70 bX -#7 ]6n%)dJoW|ߟO%Qi'ѭl1~ޣywr“ֈ):m7X|sg{[4_~Z3&9:sNkfW=5[s]I9wffZJاuQ7s[;OSu+yQasw½%sx?n&2 \ No newline at end of file diff --git a/fixture/6/1/18 b/fixture/6/1/18 deleted file mode 100644 index 2fa507b536..0000000000 --- a/fixture/6/1/18 +++ /dev/null @@ -1 +0,0 @@ -xб+qq-In_ @LAreA6,),])$餔Y~çy#b/%ߙ\|7לd#6f>b-w?_-区/Ig-J4%BmOx׽]NkuRFnh#1we߫#QxJvo{kmbu\e=I]γCSu+w~xW2@ \ No newline at end of file diff --git a/fixture/6/1/19 b/fixture/6/1/19 deleted file mode 100644 index 82b6ae9cd4..0000000000 --- a/fixture/6/1/19 +++ /dev/null @@ -1 +0,0 @@ -x?(q3J &p.`"R ddpu]ARIJl7<=?˹ψXŞxD͕z_5UhD;^7O:|n0;C{,=Œ~/d.WoЩ';3wvkKٜ9ʘ/Sosj?Y%sU^ncgy;)+8nbi\w{5Fnkv}՝~Q2 \ No newline at end of file diff --git a/fixture/6/1/2 b/fixture/6/1/2 deleted file mode 100644 index 563994f7ce..0000000000 --- a/fixture/6/1/2 +++ /dev/null @@ -1,2 +0,0 @@ -xο+qG -nQO7Q&,vErw)l-NAȮ 2=N2`?y 94z":P#}:mDL F˶L~J7=jU/ޛPm/~e~=+uڲItau!aC^Oۘнﺽ"KAK?_2 z(ԕ-FftSv/<ɗ= _Wl<;v:n35V \ No newline at end of file diff --git a/fixture/6/1/20 b/fixture/6/1/20 deleted file mode 100644 index e876fabebc..0000000000 --- a/fixture/6/1/20 +++ /dev/null @@ -1,3 +0,0 @@ -x%1(qa2 \,w WLW3rLr B]b0LhP[%gxz}_boc>"1q2"e:f^[_D"`݁Y=rSe~6yS{maoKLp_'s%|]3y+x -eHk -lA3e]czgz`Z"7" xKfn$g߮NWm1c \ No newline at end of file diff --git a/fixture/6/1/21 b/fixture/6/1/21 deleted file mode 100644 index 56f0e9e2a4..0000000000 --- a/fixture/6/1/21 +++ /dev/null @@ -1,3 +0,0 @@ -x;,PT,f J $*L4Ju&VQ 1HMt$$b|w89?2q4qEKD"&)w -kO*)˹65wC~xϼk'S+93e~ d7:e^>qS׶ŋxMӫl]1n/߸GRɿt{ -oKmG!iq2 \ No newline at end of file diff --git a/fixture/6/1/22 b/fixture/6/1/22 deleted file mode 100644 index 1f8433ac7b..0000000000 Binary files a/fixture/6/1/22 and /dev/null differ diff --git a/fixture/6/1/3 b/fixture/6/1/3 deleted file mode 100644 index 008050d082..0000000000 --- a/fixture/6/1/3 +++ /dev/null @@ -1 +0,0 @@ -x!Haaj2"nA-&,ʂ6a "eM0MFA"ð}]6xDL3('k#=EGt`Krn?9sm7| .ʼ<@~<G}4o7:e/ޱU\[oe%s5;<;>"#Zxn胇QaS17oCy j^+nTF{Kx?_k[#9}cO>|!w)7t]vNoQ7/Y[#8У)N׼3/COj=~1ʟ / \ No newline at end of file diff --git a/fixture/6/1/5 b/fixture/6/1/5 deleted file mode 100644 index e690d646fa..0000000000 Binary files a/fixture/6/1/5 and /dev/null differ diff --git a/fixture/6/1/6 b/fixture/6/1/6 deleted file mode 100644 index 06d044458c..0000000000 --- a/fixture/6/1/6 +++ /dev/null @@ -1 +0,0 @@ -x/HCqS'aiAm W`6( f  0=WDLMDH[D*cX_iђ\v4bJ[^CF7n*GxoQ۝ݮDz|M_OV{P@W{Սgwsb?SH&ߴ޷͹g6R3UݞMwdX{6/'mΝOJ/y \ No newline at end of file diff --git a/fixture/6/1/7 b/fixture/6/1/7 deleted file mode 100644 index a2546fc6f9..0000000000 Binary files a/fixture/6/1/7 and /dev/null differ diff --git a/fixture/6/1/8 b/fixture/6/1/8 deleted file mode 100644 index 8438ac02d1..0000000000 Binary files a/fixture/6/1/8 and /dev/null differ diff --git a/fixture/6/1/9 b/fixture/6/1/9 deleted file mode 100644 index 7a69fc9334..0000000000 --- a/fixture/6/1/9 +++ /dev/null @@ -1,2 +0,0 @@ -x?(q߷Kw Hd &`e w.)ztJI].V&2߾{~YOL_HW.FuS^_Go`MߤS!s1Fq+K~1@[&;ڷ-;^s {ҫAݜyZne3#^iīnWh?Z07 \ No newline at end of file diff --git a/fixture/6/2/.zarray b/fixture/6/2/.zarray deleted file mode 100644 index e251bae8a8..0000000000 --- a/fixture/6/2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": " |+8+'c~wݡʿBOï`OJ0O-w`#m{~Ÿaƞu={=3c/7km{*|r܈6_N}\S䟴/y>gNa3z~ȼD͗޷1-{3 \ No newline at end of file diff --git a/fixture/7/1/10 b/fixture/7/1/10 deleted file mode 100644 index 3dec0e1b48..0000000000 Binary files a/fixture/7/1/10 and /dev/null differ diff --git a/fixture/7/1/11 b/fixture/7/1/11 deleted file mode 100644 index 4acad22bd2..0000000000 --- a/fixture/7/1/11 +++ /dev/null @@ -1 +0,0 @@ -x5ҿ+qs+]., SW,wbd`ێ?@IR̗RGLSﳼ|{β|0-t|:5_=Pxلg~~DϘn8Z z7`oCys_[yV^)>>öx]{OG/'sv*k/Wi"\kF^>y:g__߃ l-韗W/ \ No newline at end of file diff --git a/fixture/7/1/12 b/fixture/7/1/12 deleted file mode 100644 index 0ed495820d..0000000000 --- a/fixture/7/1/12 +++ /dev/null @@ -1,2 +0,0 @@ -x5/HCq?[0@ f""bj$K"cmED,a, -8,2u~|˹sM['CtKQ/di\=JR%a:we~Cn`4n̯䟗[5<Ӿ'ez˾9#c%^cS+ݓsB7JӉ1y3rGF gw#8nWl/U*S|x,ΚW/'?V^z>`GB5U \ No newline at end of file diff --git a/fixture/7/1/13 b/fixture/7/1/13 deleted file mode 100644 index 643f9ebe60..0000000000 --- a/fixture/7/1/13 +++ /dev/null @@ -1 +0,0 @@ -x5+q/۬@YNn&%(r!FN.jw?.NVJ,a9Hy\=_yG/0}> | s?!A~VEzaVMύy]O?|#E؍ݚ |>kkӖ3ɝ~/pj6}^2?72 \ No newline at end of file diff --git a/fixture/7/1/14 b/fixture/7/1/14 deleted file mode 100644 index 8910adf351..0000000000 Binary files a/fixture/7/1/14 and /dev/null differ diff --git a/fixture/7/1/15 b/fixture/7/1/15 deleted file mode 100644 index bb4bfaba31..0000000000 --- a/fixture/7/1/15 +++ /dev/null @@ -1 +0,0 @@ -x5ѯKCqoVDʬ D . g ȶ `eA ’M'(AWλsp\ #a>] "\X_6n` rI8}`צYW 95#ضr~{wbOޙy6%}V3ȯ+L{?ji}va.KN]S=UU}N{=_l~q|01 \ No newline at end of file diff --git a/fixture/7/1/16 b/fixture/7/1/16 deleted file mode 100644 index c093f63a14..0000000000 --- a/fixture/7/1/16 +++ /dev/null @@ -1,2 +0,0 @@ -x5;HapR7&4N$MQCRSP46E8Hh=ګ͆qOx p0_-&bYM 6\3@_hg -|?KzeXгʷeywO?'ʷ%Ws慨*;j>\'^˗I?n9ܻ}c0r/Sz23x7~/II~O3V \ No newline at end of file diff --git a/fixture/7/1/17 b/fixture/7/1/17 deleted file mode 100644 index ea4559a237..0000000000 --- a/fixture/7/1/17 +++ /dev/null @@ -1 +0,0 @@ -x5п+q\2[(-. R}78"+ Yd$L۟ bcA ߯OݦŸL`Os});Qe|Nһ1վyX\Q澊~o//{_:VW|r6?%?Ȑɛsׁ<3cmJn}EU+M9H,_]||U'ݩw$u">!}{ϵ 1 \ No newline at end of file diff --git a/fixture/7/1/18 b/fixture/7/1/18 deleted file mode 100644 index e55581b69a..0000000000 --- a/fixture/7/1/18 +++ /dev/null @@ -1,3 +0,0 @@ -x]?(q;wDJ2lPԱ(ut͟$c`]YlMdgydfP.ny>^-vLoaLr~|I -~]~~ʜq=݅{7noy3Gmٯg':pDOo/=|G;i 4$݆ -~ Et+r>'=*ݐw[-s=s]ώW=tsq!_q_ދ;0_0? \ No newline at end of file diff --git a/fixture/7/1/19 b/fixture/7/1/19 deleted file mode 100644 index 1a5759b0d8..0000000000 --- a/fixture/7/1/19 +++ /dev/null @@ -1 +0,0 @@ -x5!HCa=6^hrA Nb(YVua%diL6A%U2?sϽ+F/58\L^=fBm0M%o2 '6#{8K,6w+wMON߸]'ޱ!g^E㸼:>{(lMi/f |M|icWsuk]g]1jeψOqFH}y= b. \ No newline at end of file diff --git a/fixture/7/1/2 b/fixture/7/1/2 deleted file mode 100644 index 003395bec4..0000000000 --- a/fixture/7/1/2 +++ /dev/null @@ -1 +0,0 @@ -x5б+qs$%E&ŀlNF%ɩRG"nq)QPn+QR_ E[>|ߟ繮 :ɯe$U3w lx(y3oC.8f+SXC߃3[ \ No newline at end of file diff --git a/fixture/7/1/21 b/fixture/7/1/21 deleted file mode 100644 index 212e3514d2..0000000000 --- a/fixture/7/1/21 +++ /dev/null @@ -1 +0,0 @@ -x5ЯKCaPa it0aa,65- ‚m^APll-_ssx"mӁ }&+d~UEXhϚX}S ߗ;5 7MwW_҇rԯuz(K|UWiaDϨ8$? |5 6yy|O[m✹sOO}۞)s^Ww}=6ms|Y/& \ No newline at end of file diff --git a/fixture/7/1/22 b/fixture/7/1/22 deleted file mode 100644 index e05e70333d..0000000000 Binary files a/fixture/7/1/22 and /dev/null differ diff --git a/fixture/7/1/3 b/fixture/7/1/3 deleted file mode 100644 index e098767a58..0000000000 --- a/fixture/7/1/3 +++ /dev/null @@ -1 +0,0 @@ -x5?(q;E1EE)ȟLtn+FEnbN)aL7\XXlӥ|_[yy(\tDm5F2Lcs&! _?Կk};<,%ݱBW wʜ^=gtӃ|]|9oqy]{ cew\-<wדo9ȗt Xtߩ5j~i?K{}O]Cd_}sMw"ܔ0* \ No newline at end of file diff --git a/fixture/7/1/4 b/fixture/7/1/4 deleted file mode 100644 index 04ea1fa394..0000000000 --- a/fixture/7/1/4 +++ /dev/null @@ -1,2 +0,0 @@ -x5?KQ?oI-M"Q AQKmVb4SC*AAkC:aM } tyssν"\v4  -w5~v"9ّo_cy|Oo9^溽g5XߛU{Jz'|Is ~s 5=&7#o^ᬽs"=wG[|wK |3r3r&2ߑvo _'}}$!g?{~U- \ No newline at end of file diff --git a/fixture/7/1/5 b/fixture/7/1/5 deleted file mode 100644 index fac8730e26..0000000000 --- a/fixture/7/1/5 +++ /dev/null @@ -1,6 +0,0 @@ -x]-Has:dU,d&6-0&b4(hYšC8,6 - -C@D1dL0i|~ɷ{'D`5/C{ -??7|I -},AO5[/kwO^O}ѯϩ;|ܧ{ -MmE/Ex-w@wuY)l`}[re]sz}i{߭{V1c/kaz39Kcr2|r~S;Gꊹ~s]_))2 \ No newline at end of file diff --git a/fixture/7/1/6 b/fixture/7/1/6 deleted file mode 100644 index e1d4851359..0000000000 --- a/fixture/7/1/6 +++ /dev/null @@ -1,2 +0,0 @@ -x51/CqG` 1b!D:H Md$A(Q4ED"i :,&@={vDO3e[k況F]eC.aO-%aüaOł0Oe=2)=;~J^$W^^C7/*6[pc/ݛ#ͷ+wOo}kc[uNO{!  -1V \ No newline at end of file diff --git a/fixture/7/1/7 b/fixture/7/1/7 deleted file mode 100644 index 265611e887..0000000000 --- a/fixture/7/1/7 +++ /dev/null @@ -1 +0,0 @@ -x5б+qqeBd`EJpF:70tGRn[,YQR"E s>>~8e8 | rq~s[ΖwզgZ}[gkїWO ʯ;3'n=8dD)LS1] Jw k n^?WW9Ox]쀋|rg[ lCg 'ū̙c &ڽc1 \ No newline at end of file diff --git a/fixture/7/1/8 b/fixture/7/1/8 deleted file mode 100644 index dc459ca766..0000000000 --- a/fixture/7/1/8 +++ /dev/null @@ -1 +0,0 @@ -x51(qȠn0u2Ā1tI7\r%ҭtR rX.}}v|2'xkC|pn=vh/w?_Yz|k._%xD{`KnSn|nG.ɩU0no;rw??'wf?_QqOs)g=ߕ1{}.k;ْ3e9p#%6q \ No newline at end of file diff --git a/fixture/7/1/9 b/fixture/7/1/9 deleted file mode 100644 index 2d92fded36..0000000000 --- a/fixture/7/1/9 +++ /dev/null @@ -1,2 +0,0 @@ -x5+KaM@:0LeILd -F`d]W< jh a&o}&H-zf(S Nq aEܤ9LOL0zgO}m~|_ߐeYtaǞ:M?ͻs;Ưo;r>|m®ww'-ǁEwwOz^2W[#w'+iN-- \ No newline at end of file diff --git a/fixture/7/2/.zarray b/fixture/7/2/.zarray deleted file mode 100644 index d1f1669345..0000000000 --- a/fixture/7/2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "6?6A6B6C6D6E6G6H6I6J6L6M6N6O6Q6R6S6T6U6W6X6Y6Z6\6]6^6_6`6b6c6d6e6g6h6i6j6l6m6n6o6p6r6s6t6u6w6x6y6z6|6}6~666666666666666666666666666666666666666666666666 \ No newline at end of file diff --git a/fixture/8/0/14 b/fixture/8/0/14 deleted file mode 100644 index e5354ceffc..0000000000 Binary files a/fixture/8/0/14 and /dev/null differ diff --git a/fixture/8/0/15 b/fixture/8/0/15 deleted file mode 100644 index 1f210be817..0000000000 --- a/fixture/8/0/15 +++ /dev/null @@ -1 +0,0 @@ -4757678797:7;7=7>7?7@7A7C7D7E7F7H7I7J7K7M7N7O7P7Q7S7T7U7V7X7Y7Z7[7]7^7_7`7a7c7d7e7f7h7i7j7k7l7n7o7p7q7s7t7u7v7x7y7z7{7|7~7777777777777777777777777777777777777777 \ No newline at end of file diff --git a/fixture/8/0/16 b/fixture/8/0/16 deleted file mode 100644 index 8234a8d60f..0000000000 Binary files a/fixture/8/0/16 and /dev/null differ diff --git a/fixture/8/0/17 b/fixture/8/0/17 deleted file mode 100644 index 1a679108a0..0000000000 --- a/fixture/8/0/17 +++ /dev/null @@ -1 +0,0 @@ -888888888888888888 8!8!8"8"8#8$8$8%8%8&8'8'8(8)8)8*8*8+8,8,8-8-8.8/8/808181828283848485858687878889898:8:8;8<8<8=8=8>8?8?8@8A8A8B8B8C8D8D8E8E8F8G8G8H8I8I8J8J8K8L8L8M8M8N8O8O8P8Q8Q8R8 \ No newline at end of file diff --git a/fixture/8/0/18 b/fixture/8/0/18 deleted file mode 100644 index 3a621a92df..0000000000 --- a/fixture/8/0/18 +++ /dev/null @@ -1 +0,0 @@ -R8S8T8T8U8U8V8W8W8X8Y8Y8Z8Z8[8\8\8]8]8^8_8_8`8`8a8b8b8c8d8d8e8e8f8g8g8h8h8i8j8j8k8l8l8m8m8n8o8o8p8p8q8r8r8s8t8t8u8u8v8w8w8x8x8y8z8z8{8|8|8}8}8~88888888888888888888888888888 \ No newline at end of file diff --git a/fixture/8/0/19 b/fixture/8/0/19 deleted file mode 100644 index bec158bd6f..0000000000 --- a/fixture/8/0/19 +++ /dev/null @@ -1 +0,0 @@ -8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 \ No newline at end of file diff --git a/fixture/8/0/2 b/fixture/8/0/2 deleted file mode 100644 index abbbc5a38b..0000000000 --- a/fixture/8/0/2 +++ /dev/null @@ -1 +0,0 @@ -+++++++++, ,,,,,!,&,+,0,5,:,?,D,I,M,R,W,\,a,f,k,p,u,z,,,,,,,,,,,,,,,,,,,,,,,,,,,,-- ----!-&-+-0-5-9->-C-H-M-R-W-\-a-f-k-p-t-y-~-------------- \ No newline at end of file diff --git a/fixture/8/0/20 b/fixture/8/0/20 deleted file mode 100644 index 40fefc09d4..0000000000 Binary files a/fixture/8/0/20 and /dev/null differ diff --git a/fixture/8/0/21 b/fixture/8/0/21 deleted file mode 100644 index 6b33e50e5f..0000000000 --- a/fixture/8/0/21 +++ /dev/null @@ -1 +0,0 @@ - 9 9 9 9 999999999999999999999999999999 9 9!9!9"9#9#9$9%9%9&9&9'9(9(9)9)9*9+9+9,9-9-9.9.9/90909191929393949595969697989899999:9;9;9<9=9=9>9>9?9@9@9A9A9B9C9C9D9E9E9F9F9G9H9 \ No newline at end of file diff --git a/fixture/8/0/22 b/fixture/8/0/22 deleted file mode 100644 index 292d67cb35..0000000000 --- a/fixture/8/0/22 +++ /dev/null @@ -1 +0,0 @@ -H9I9I9J9K9K9L9M9M9N9N9O9P9P9Q9Q9R9S9S9T9U9U9V9V9W9X9X9Y9Y9Z9[9[9\9]9]9^9^9_9`9`9a9a9b9c9c9d9d9e9f9f9g9h9h9i9i9j9k9k9l9l9m9n9n9o9p9p9q9q9r9s9s9t9t9u9v9v9w9x9x9y9y9z9{9{9|9|9}9~9~999999999999 \ No newline at end of file diff --git a/fixture/8/0/23 b/fixture/8/0/23 deleted file mode 100644 index 1da80b239c..0000000000 --- a/fixture/8/0/23 +++ /dev/null @@ -1 +0,0 @@ -9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 \ No newline at end of file diff --git a/fixture/8/0/24 b/fixture/8/0/24 deleted file mode 100644 index 475577ef79..0000000000 Binary files a/fixture/8/0/24 and /dev/null differ diff --git a/fixture/8/0/25 b/fixture/8/0/25 deleted file mode 100644 index 0c5c43a6ae..0000000000 --- a/fixture/8/0/25 +++ /dev/null @@ -1,3 +0,0 @@ -::::::::::::: : : -: -: : : : ::::::::::::::::::::::::::::::: :!:!:":":#:$:$:%:%:&:':':(:):):*:*:+:,:,:-:-:.:/:/:0:1:1:2:2:3:4:4:5:5:6:7:7:8:9:9:::::;:<:<:=:=: \ No newline at end of file diff --git a/fixture/8/0/26 b/fixture/8/0/26 deleted file mode 100644 index 2dea2cd590..0000000000 --- a/fixture/8/0/26 +++ /dev/null @@ -1 +0,0 @@ ->:?:?:@:A:A:B:B:C:D:D:E:E:F:G:G:H:I:I:J:J:K:L:L:M:M:N:O:O:P:Q:Q:R:R:S:T:T:U:U:V:W:W:X:Y:Y:Z:Z:[:\:\:]:]:^:_:_:`:`:a:b:b:c:d:d:e:e:f:g:g:h:h:i:j:j:k:l:l:m:m:n:o:o:p:p:q:r:r:s:t:t:u:u:v:w:w:x:x:y:z:z:{: \ No newline at end of file diff --git a/fixture/8/0/27 b/fixture/8/0/27 deleted file mode 100644 index a09b792d77..0000000000 --- a/fixture/8/0/27 +++ /dev/null @@ -1 +0,0 @@ -|:|:}:}:~:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: \ No newline at end of file diff --git a/fixture/8/0/28 b/fixture/8/0/28 deleted file mode 100644 index ae9c041253..0000000000 --- a/fixture/8/0/28 +++ /dev/null @@ -1 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: \ No newline at end of file diff --git a/fixture/8/0/29 b/fixture/8/0/29 deleted file mode 100644 index 1fcfd3e335..0000000000 Binary files a/fixture/8/0/29 and /dev/null differ diff --git a/fixture/8/0/3 b/fixture/8/0/3 deleted file mode 100644 index 8189ad0a17..0000000000 --- a/fixture/8/0/3 +++ /dev/null @@ -1 +0,0 @@ --------------.. ....!.%.*./.4.9.>.C.H.M.R.W.\.`.e.j.o.t.y.~...........................// //// /%/*///4/9/>/C/H/M/Q/V/[/`/e/j/o/t/y/~////////// \ No newline at end of file diff --git a/fixture/8/0/30 b/fixture/8/0/30 deleted file mode 100644 index 3b4843c8ee..0000000000 --- a/fixture/8/0/30 +++ /dev/null @@ -1 +0,0 @@ -4;5;5;6;6;7;8;8;9;9;:;;;;;<;=;=;>;>;?;@;@;A;A;B;C;C;D;E;E;F;F;G;H;H;I;I;J;K;K;L;M;M;N;N;O;P;P;Q;Q;R;S;S;T;U;U;V;V;W;X;X;Y;Y;Z;[;[;\;];];^;^;_;`;`;a;a;b;c;c;d;d;e;f;f;g;h;h;i;i;j;k;k;l;l;m;n;n;o;p;p;q; \ No newline at end of file diff --git a/fixture/8/0/31 b/fixture/8/0/31 deleted file mode 100644 index 69ddd32101..0000000000 --- a/fixture/8/0/31 +++ /dev/null @@ -1 +0,0 @@ -q;r;s;s;t;t;u;v;v;w;x;x;y;y;z;{;{;|;|;};~;~;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; \ No newline at end of file diff --git a/fixture/8/0/32 b/fixture/8/0/32 deleted file mode 100644 index 36eb92395c..0000000000 --- a/fixture/8/0/32 +++ /dev/null @@ -1 +0,0 @@ -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; \ No newline at end of file diff --git a/fixture/8/0/33 b/fixture/8/0/33 deleted file mode 100644 index db0396d24c..0000000000 Binary files a/fixture/8/0/33 and /dev/null differ diff --git a/fixture/8/0/4 b/fixture/8/0/4 deleted file mode 100644 index 849f1a60e9..0000000000 --- a/fixture/8/0/4 +++ /dev/null @@ -1 +0,0 @@ -/////////////////000 0 000000000!0$0&0)0+0-000205070:0<0?0A0D0F0I0K0M0P0R0U0W0Z0\0_0a0d0f0h0k0m0p0r0u0w0z0|000000000000000000000000000000000 \ No newline at end of file diff --git a/fixture/8/0/5 b/fixture/8/0/5 deleted file mode 100644 index 3e2f539f26..0000000000 --- a/fixture/8/0/5 +++ /dev/null @@ -1 +0,0 @@ -0000000000000000000001111 1 11111111!1#1&1(1+1-10121517191<1>1A1C1F1H1K1M1P1R1U1W1Y1\1^1a1c1f1h1k1m1p1r1t1w1y1|1~1111111111111111111111111111 \ No newline at end of file diff --git a/fixture/8/0/6 b/fixture/8/0/6 deleted file mode 100644 index 1e0bb36f03..0000000000 --- a/fixture/8/0/6 +++ /dev/null @@ -1,2 +0,0 @@ -11111111111111111111111112222 -2 22222222!2#2%2(2*2-2/222427292<2>2A2C2E2H2J2M2O2R2T2W2Y2\2^2`2c2e2h2j2m2o2r2t2w2y2|2~222222222222222222222222 \ No newline at end of file diff --git a/fixture/8/0/7 b/fixture/8/0/7 deleted file mode 100644 index e86f0d5f21..0000000000 Binary files a/fixture/8/0/7 and /dev/null differ diff --git a/fixture/8/0/8 b/fixture/8/0/8 deleted file mode 100644 index eaf53fe592..0000000000 Binary files a/fixture/8/0/8 and /dev/null differ diff --git a/fixture/8/0/9 b/fixture/8/0/9 deleted file mode 100644 index 323029cf6d..0000000000 --- a/fixture/8/0/9 +++ /dev/null @@ -1 +0,0 @@ -R4T4U4V4W4Y4Z4[4\4]4_4`4a4b4d4e4f4g4h4j4k4l4m4o4p4q4r4t4u4v4w4x4z4{4|4}44444444444444444444444444444444444444444444444444444444444444444 \ No newline at end of file diff --git a/fixture/8/1/.zarray b/fixture/8/1/.zarray deleted file mode 100644 index d53b2f7091..0000000000 --- a/fixture/8/1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "=?=@=d:B=uC=ʯD=}E=0%G=_H=I=HJ=L=JM=aN=O=P=y5R=,pS=ߪT=U=E W=ZX=Y=]Z= \=E]=v^=)_=`=0b=Akc=d=e=Zg= Vh=i=sj=%l=@m={n=>o=p=+r=Wfs= -t= \ No newline at end of file diff --git a/fixture/9/0/10 b/fixture/9/0/10 deleted file mode 100644 index 6b31d5ffd8..0000000000 --- a/fixture/9/0/10 +++ /dev/null @@ -1,3 +0,0 @@ -V>Й>>Y>F>n>\>> >_ >2> Z>b>>Л>e>>F>hm>>>k> ->2>nY>ŀ>>qϝ>>>tE>l>!>x>>$ ->{1>X>'>~>Ο>*>>D>-l>>ں>1> >0>4X>>ড>7Ρ>>>:D>k>撢>=>>>@0>W>~>C>ͣ>>F>C>j>I>>>L>/>V>O~>>̥>R>>B>Uj>>>Y>>/>\V>}>>_̧>> >bB>i>>e>ߨ> \ No newline at end of file diff --git a/fixture/9/0/11 b/fixture/9/0/11 deleted file mode 100644 index e3d7e05c61..0000000000 --- a/fixture/9/0/11 +++ /dev/null @@ -1 +0,0 @@ ->h.>U>}>k>˩>>n>A>i>q>Ƿ>ߪ>t>->!U>w|>Σ>$˫>z>>'A>}h>ԏ>*>ެ>>-->T>{>0>ʭ>>3>@>g>6>>ݮ>9>,>S>={>>ɯ>@>>?>Cg>>ﵰ>Fݰ>>+>IS>z>>Lɱ>>>O?>f>>R>ܲ>>U+>R>z>X>ȳ>>[>>>f>^>> ܴ>a>*>R>dy>>ȵ>h>>>>ke>>>n۶>>*>qQ>x>>tǷ>>!>w=> \ No newline at end of file diff --git a/fixture/9/0/12 b/fixture/9/0/12 deleted file mode 100644 index 27afc77e92..0000000000 Binary files a/fixture/9/0/12 and /dev/null differ diff --git a/fixture/9/0/13 b/fixture/9/0/13 deleted file mode 100644 index ecedf338ba..0000000000 --- a/fixture/9/0/13 +++ /dev/null @@ -1,2 +0,0 @@ ->>6>8>_>9>>><>$>K>?s>>>B>>7>E_>>>H>>#>KK>r>>N>>>Q7>^>>U>>>X#>J>r>[>>>^>6> -^>a>>>d>">J>gq>>>j>>6>m]>Ä>>p>>">sI>p> >v>>#>y5>\>&>|>>)>!>H>,p>>پ>/> >4>2\>>ߪ>5>> >9H>o>><>> >?4>[>>B>>> \ No newline at end of file diff --git a/fixture/9/0/14 b/fixture/9/0/14 deleted file mode 100644 index d6f8a1a8c9..0000000000 --- a/fixture/9/0/14 +++ /dev/null @@ -1,3 +0,0 @@ -E >G>n>H>>>K >3>Z>N>>>Q>>F>Tn>>>W> >3>ZZ>>>]>> ->`F>m> >d>> >g2>Y>>j>>>m>E>m>p>ƻ>>s ->1> Y>v>̧>#>y>>&E>|l>ӓ>)>> >,1>X>>/>>>2>D>k>5>>>8 >0>W>;>>>>>>C>Ak>>>E>>/>HW>~>>K>>>NC>j>>Q>>>T/>V> \ No newline at end of file diff --git a/fixture/9/0/15 b/fixture/9/0/15 deleted file mode 100644 index e79caa0e9b..0000000000 Binary files a/fixture/9/0/15 and /dev/null differ diff --git a/fixture/9/0/16 b/fixture/9/0/16 deleted file mode 100644 index 72ead90e6a..0000000000 Binary files a/fixture/9/0/16 and /dev/null differ diff --git a/fixture/9/0/17 b/fixture/9/0/17 deleted file mode 100644 index ee3be49fe5..0000000000 --- a/fixture/9/0/17 +++ /dev/null @@ -1,4 +0,0 @@ -?g???i???j&?:?M?la?u?ˆ?m???o???p?&?9?rM?a?t?t??ʯ?u? ??w?"?%?x9?#M?`?zt?%?Л?{?&??}?(??~%?)9?L?`?+t?և??-???.???0%?8?L?1`?s??3?ޮ??4???6?$?8?7L?_?s?9???:???<? ?$ ?=8 ?K ?_ ??s ? ? ?A ? ? ?B ? ? -?D$ -?7 -? \ No newline at end of file diff --git a/fixture/9/0/18 b/fixture/9/0/18 deleted file mode 100644 index 6703c4ff6a..0000000000 Binary files a/fixture/9/0/18 and /dev/null differ diff --git a/fixture/9/0/19 b/fixture/9/0/19 deleted file mode 100644 index c291600c42..0000000000 --- a/fixture/9/0/19 +++ /dev/null @@ -1 +0,0 @@ -x?#?!?y5?%I?\?{p?&?ї?|?(??~?)? ?!?+5?H?\?,p?׃??.?پ??/?? ?1!?4?H?2\?o??4?ߪ??5???7 ? ?4?9H?[?o?:???<???=? ? ??4?G?[?@o???B???C?? ?E ?3?G?F[?n??H???I???K ??3?MG?Z?n?N???P???Q? ??S3?F?Z?Tn??? \ No newline at end of file diff --git a/fixture/9/0/2 b/fixture/9/0/2 deleted file mode 100644 index 3e1667f0a2..0000000000 Binary files a/fixture/9/0/2 and /dev/null differ diff --git a/fixture/9/0/20 b/fixture/9/0/20 deleted file mode 100644 index 6de2757838..0000000000 --- a/fixture/9/0/20 +++ /dev/null @@ -1,5 +0,0 @@ -V???W?? ?Y?3?F?ZZ?n??\???]? ??_ ? -?2?`F? Z?m?b? ??d???e? ??g2?F?Y?hm???j???k?? -?m?2?E?nY?m?ŀ?p??ƻ?q???s -??1?tE? Y?l?v?!?̧?x?#??y?$ -??{1?&E?X?|l?'?ӓ?~?)???*? ? ?,1 ?D ?X ?-l ? ? ?/ ?ں ? ?1 ? ? !?2!?0!?D!? \ No newline at end of file diff --git a/fixture/9/0/21 b/fixture/9/0/21 deleted file mode 100644 index 0678831305..0000000000 --- a/fixture/9/0/21 +++ /dev/null @@ -1,2 +0,0 @@ -4X!?k!?!?5!?!?!?7!?!?!?8 "?"?0"?:D"?W"?k"?;"?"?"?="?"?"?>"?#?#?@0#?C#?W#?Ak#?~#?#?C#?#?#?E#?#?$?F$?/$?C$?HW$?j$?~$?I$?$?$?K$?$?$?L%?%?/%?NC%?V%?j%?O~%?%?%?Q%?%?%?R%?&?&?T/&?B&?V&?Uj&?~&?&?W&?&?&?Y&?&?'?Z'?/'?B'?\V'?j'?}'?]'?'?'?_'? -'?'?`(? (?.(?bB(? V(?i(?c}(?(?(?e(?(?(?f(? \ No newline at end of file diff --git a/fixture/9/0/22 b/fixture/9/0/22 deleted file mode 100644 index 655e07fc0a..0000000000 --- a/fixture/9/0/22 +++ /dev/null @@ -1 +0,0 @@ -)?)?h.)?B)?U)?ii)?})?)?k)?)?)?l)?)?*?n*?.*?A*?pU*?i*?|*?q*?*?Ƿ*?s*?*?*?t+?+?-+?vA+?!U+?h+?w|+?"+?Σ+?y+?$+?+?z+?%,?,?|-,?'A,?T,?}h,?)|,?ԏ,?,?*,?,?,?,,?-?-?---?@-?T-?/h-?{-?-?0-?۶-?-?2-?-?.?3.?,.?@.?5T.?g.?{.?6.?.?.?8.?.?.?9/?/?,/?;@/?S/?g/?={/?/?/?>/?/?/?@/?0?0?A,0??0?S0?Cg0?z0?0?D0? \ No newline at end of file diff --git a/fixture/9/0/23 b/fixture/9/0/23 deleted file mode 100644 index f6633eae3e..0000000000 Binary files a/fixture/9/0/23 and /dev/null differ diff --git a/fixture/9/0/24 b/fixture/9/0/24 deleted file mode 100644 index 9216c84f16..0000000000 Binary files a/fixture/9/0/24 and /dev/null differ diff --git a/fixture/9/0/25 b/fixture/9/0/25 deleted file mode 100644 index 4164afac15..0000000000 --- a/fixture/9/0/25 +++ /dev/null @@ -1 +0,0 @@ -@?V'@?;@?N@?Xb@?v@?@?Y@?@?@?[@?@?@?]A?'A?:A?^NA? bA?uA?`A? A?A?aA? A?A?cA?B?&B?d:B?NB?aB?fuB?B?B?gB?B?B?iB?B?C?j&C?:C?MC?laC?uC?ˆC?mC?C?C?oC?C?C?pD?&D?9D?rMD?aD?tD?tD?D?ʯD?uD? D?D?wD?"E?%E?x9E?#ME?`E?ztE?%E?ЛE?{E?&E?E?}E?(E?F?~%F?)9F?LF?`F?+tF?ևF?F?-F?F?F?.F?F?G?0%G?8G?LG?1`G?sG?G?3G?ޮG? \ No newline at end of file diff --git a/fixture/9/0/26 b/fixture/9/0/26 deleted file mode 100644 index 2bd41c73ca..0000000000 Binary files a/fixture/9/0/26 and /dev/null differ diff --git a/fixture/9/0/27 b/fixture/9/0/27 deleted file mode 100644 index 06eaf83458..0000000000 --- a/fixture/9/0/27 +++ /dev/null @@ -1 +0,0 @@ -gqO?O?O?hO?O?O?jO?O?P?l"P?6P?IP?m]P?qP?ÄP?oP?P?ſP?pP?P?P?rQ?"Q?5Q?sIQ?]Q?pQ?uQ? Q?˫Q?vQ?!Q?Q?xQ?#R?!R?y5R?%IR?\R?{pR?&R?їR?|R?(R?R?~R?)R? S?!S?+5S?HS?\S?,pS?׃S?S?.S?پS?S?/S?S? T?1!T?4T?HT?2\T?oT?T?4T?ߪT?T?5T?T?T?7 U? U?4U?9HU?[U?oU?:U?U?U?b?c?c?@0c?Cc?Wc?Akc?~c?c?Cc?c?c?Ec?c?d?Fd?/d?Cd?HWd?jd?~d?Id?d?d?Kd?d?d?Le?e?/e?NCe?Ve?je?O~e?e?e?Qe?e?e?Re?f?f?T/f?Bf?Vf?Ujf? \ No newline at end of file diff --git a/fixture/9/0/3 b/fixture/9/0/3 deleted file mode 100644 index b2cf6ffa43..0000000000 --- a/fixture/9/0/3 +++ /dev/null @@ -1,2 +0,0 @@ -d='==<=3ں=w==?=O==L='=Ŀ=Xb== =d:==u=p=ʯ=#M=}=և=0%==_=<==7=H=r==U=J==a="==m]== =y5==,p= =ߪ=9H===E ==Z=Q==3=]=m= =j=E==v==)=X==5=0==Ak===NC==~=Z== V=f==.=s=h=%==@=2={==>=S==J=+==Wf== -=c>= \ No newline at end of file diff --git a/fixture/9/0/30 b/fixture/9/0/30 deleted file mode 100644 index a30e3ee074..0000000000 --- a/fixture/9/0/30 +++ /dev/null @@ -1,2 +0,0 @@ -~f?f?Wf?f?f?Yf?f?g?Zg?/g?Bg?\Vg?jg?}g?]g?g?g?_g? -g?g?`h? h?.h?bBh? Vh?ih?c}h?h?h?eh?h?h?fh?i?i?h.i?Bi?Ui?iii?}i?i?ki?i?i?li?i?j?nj?.j?Aj?pUj?ij?|j?qj?j?Ƿj?sj?j?j?tk?k?-k?vAk?!Uk?hk?w|k?"k?Σk?yk?$k?k?zk?%l?l?|-l?'Al?Tl?}hl?)|l?ԏl?l?*l?l?l?,l?m?m?--m?@m?Tm?/hm?{m?m?0m?۶m?m?2m?m?n?3n? \ No newline at end of file diff --git a/fixture/9/0/31 b/fixture/9/0/31 deleted file mode 100644 index 120d96eef5..0000000000 Binary files a/fixture/9/0/31 and /dev/null differ diff --git a/fixture/9/0/32 b/fixture/9/0/32 deleted file mode 100644 index 115984f147..0000000000 Binary files a/fixture/9/0/32 and /dev/null differ diff --git a/fixture/9/0/33 b/fixture/9/0/33 deleted file mode 100644 index cdef8db668..0000000000 Binary files a/fixture/9/0/33 and /dev/null differ diff --git a/fixture/9/0/4 b/fixture/9/0/4 deleted file mode 100644 index acc26ee39f..0000000000 Binary files a/fixture/9/0/4 and /dev/null differ diff --git a/fixture/9/0/5 b/fixture/9/0/5 deleted file mode 100644 index 15ea08a157..0000000000 --- a/fixture/9/0/5 +++ /dev/null @@ -1,2 +0,0 @@ -V>>F>\> >2>b>>>hm>> ->nY>>>tE>!>>{1>'>> >-l >ں > !>4X!>!>!>:D">">">@0#>~#>#>F$>j$>$>L%>V%>%>R%>B&>&>Y&>/'>}'>_'> (>i(>e(>)>U)>k)>)>A*>q*>*>-+>w|+>$+>,>}h,>*,>->T->0->->@.>6.>.>,/>={/>/>0>Cg0>0>1>IS1>1>1>O?2>2>2>U+3>z3>3>[4>f4>4>a5>R5>5>h5>>6>6>n6>*7>x7>t7>!8> \ No newline at end of file diff --git a/fixture/9/0/6 b/fixture/9/0/6 deleted file mode 100644 index 745cd11269..0000000000 Binary files a/fixture/9/0/6 and /dev/null differ diff --git a/fixture/9/0/7 b/fixture/9/0/7 deleted file mode 100644 index ffa66cdf47..0000000000 --- a/fixture/9/0/7 +++ /dev/null @@ -1,3 +0,0 @@ -E W>nW>W>K X>ZX>X>QX>FY>Y>WY>3Z>Z>]Z> -[>m[>d[> \>Y\>j\>\>E]>p]>]>1^>v^>#^>_>|l_>)_> `>X`>/`>`>Da>5a>a>0b>;b>b>c>Akc>c>d>HWd>d>d>NCe>e>e>T/f>~f>f>Zg>jg>g>`h> Vh>h>fh>Bi>i>li>.j>|j>sj>k>hk>yk>%l>Tl>l>,l>@m>m>2m>,n>{n>8n>o>go>>o>p>Sp>Dp>p>?q>Jq>q>+r>Qzr>r>s>Wfs>s>t>]Rt> -t>t>c>u>u> \ No newline at end of file diff --git a/fixture/9/0/8 b/fixture/9/0/8 deleted file mode 100644 index 9f9eac9f83..0000000000 Binary files a/fixture/9/0/8 and /dev/null differ diff --git a/fixture/9/0/9 b/fixture/9/0/9 deleted file mode 100644 index 85846b4239..0000000000 Binary files a/fixture/9/0/9 and /dev/null differ diff --git a/fixture/9/1/.zarray b/fixture/9/1/.zarray deleted file mode 100644 index 015e37f3cb..0000000000 --- a/fixture/9/1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "G|&'}>Y3?XjF/M'pHk"lȭk!;##"U#E|_țC1Lܝsd-|@)?+ 5IK9627RSNp>a1#dgvDODm^IAdQm\iETMtC]t~AtSth==ݔ33z`tss5/捎.4Rftc^c}Vzlb?շu \ No newline at end of file diff --git a/fixture/9/1/17 b/fixture/9/1/17 deleted file mode 100644 index 6a17f1833a..0000000000 Binary files a/fixture/9/1/17 and /dev/null differ diff --git a/fixture/9/1/18 b/fixture/9/1/18 deleted file mode 100644 index 5726659bdb..0000000000 --- a/fixture/9/1/18 +++ /dev/null @@ -1,3 +0,0 @@ -xJa$1zG"Qj(P+EQD{#y $Fn(FQl+B):ں(p+|VryDtY1QU6_?;"_iͳ&5e͟f4/:=MW iõod، ÿe0U5hZ鎡ge&m9lGҟXve@ղ[̵,:%c?㘏*v7Sv8nxv:D7)NF6/-E]IWB)o gByBRIQDQ&;?&DJQۢ( - -m]IEUVdWvòW/5QC9VZ]q_9$9|<-zΖ=ߖ=js g9H* O. ˁKb+1s9XP \ No newline at end of file diff --git a/fixture/9/1/21 b/fixture/9/1/21 deleted file mode 100644 index d90f3b75ca..0000000000 Binary files a/fixture/9/1/21 and /dev/null differ diff --git a/fixture/9/1/22 b/fixture/9/1/22 deleted file mode 100644 index a91a34ec37..0000000000 --- a/fixture/9/1/22 +++ /dev/null @@ -1 +0,0 @@ -xaQ'hݛ99'"mnW+o$n{ 'DDwmڼ9Ü N aqb85x`x242 OӖ9K[,ʖ5|bյ[. -'͙bڱ'RxYu\n8%WǕc~x=v, iMVh - U4w-a#ؾ~ \ 2>լ|+k%%TOuZKsO>PTz&Y{<_*uks92|yST{&H`1Q lTꁟvx/k3pgODf"!r9Y]oFN#]& \ No newline at end of file diff --git a/fixture/9/1/23 b/fixture/9/1/23 deleted file mode 100644 index 050dbeba9e..0000000000 Binary files a/fixture/9/1/23 and /dev/null differ diff --git a/fixture/9/1/24 b/fixture/9/1/24 deleted file mode 100644 index b0010d7be5..0000000000 --- a/fixture/9/1/24 +++ /dev/null @@ -1 +0,0 @@ -x JQ߼;P6D )I6oII(M(|y nb-")|yRdUśmSqviE4s}ӼOh4Wy|IӿX,45W;zȰ1 bXJSA0T2V 5õmX Y#%/SҲhyXsòֶؐ%,k\x|F8 *l4m86Îјc[qͤ[ڱsˎ'UxqrD#)Yϭg'*x=UEGŸ;bw^` a3-ʬ \ No newline at end of file diff --git a/fixture/9/1/25 b/fixture/9/1/25 deleted file mode 100644 index 5f4e313e36..0000000000 Binary files a/fixture/9/1/25 and /dev/null differ diff --git a/fixture/9/1/26 b/fixture/9/1/26 deleted file mode 100644 index 42ba1629c5..0000000000 --- a/fixture/9/1/26 +++ /dev/null @@ -1 +0,0 @@ -xkaq"˒_p'M,><ϗE7IDDEDDv'm;ew""FDŽ~µQBq°U0\/bg.n,^14 7[cO]íajdYn,KZ-,Y>,w-˗^2ݷ|Z焓[,4T8UW5!i?ö`Ϟh(HNWNJ;ST_SԕM%YO9=Tyq&v"s8U/뎙cO9ˑ\޳/s>x^W=Mϛu\3x(p0xW %0_,TˁN`e7pq#))6!םy \ No newline at end of file diff --git a/fixture/9/1/27 b/fixture/9/1/27 deleted file mode 100644 index c808c1d370..0000000000 Binary files a/fixture/9/1/27 and /dev/null differ diff --git a/fixture/9/1/28 b/fixture/9/1/28 deleted file mode 100644 index a16b311194..0000000000 --- a/fixture/9/1/28 +++ /dev/null @@ -1 +0,0 @@ -xqHdEt6xEDD}rxau?@6ѝǣV6g-1 džSB^n"bZ,""on?úv \ No newline at end of file diff --git a/fixture/9/1/29 b/fixture/9/1/29 deleted file mode 100644 index 36f5c3ac36..0000000000 Binary files a/fixture/9/1/29 and /dev/null differ diff --git a/fixture/9/1/3 b/fixture/9/1/3 deleted file mode 100644 index 84bc05c39a..0000000000 Binary files a/fixture/9/1/3 and /dev/null differ diff --git a/fixture/9/1/30 b/fixture/9/1/30 deleted file mode 100644 index e41bc98ddb..0000000000 --- a/fixture/9/1/30 +++ /dev/null @@ -1,3 +0,0 @@ -xJaaD-ĈS"Q?")-# -v HB7(6O݋بFLG lG\ً/b+a6 k5<)W  OmY0FV & -ea* sMp[xp$N)U7EU9VVVWWvK]x_ySd>gYdU[>6,[]N2wJ9ZǝtݒcTs,4-c8|IzJٜ넧,s%Ϸ5rs\y~L&rD`Lk+KXd*s \ No newline at end of file diff --git a/fixture/9/1/31 b/fixture/9/1/31 deleted file mode 100644 index e2df800f93..0000000000 Binary files a/fixture/9/1/31 and /dev/null differ diff --git a/fixture/9/1/32 b/fixture/9/1/32 deleted file mode 100644 index 0b06160e7d..0000000000 --- a/fixture/9/1/32 +++ /dev/null @@ -1,3 +0,0 @@ -xJamaHm pGwc0'Qk#>FFQ(FEQ$JNz٭0;pUp+kHL^ s35noN0m滆a;.I B+/- -3B[{5)o #]C_ĕ3)cF2ZT>UeTζݎʹe/i2t`yPdjY[7-_[Ŏeos,%G'X.8.]qT jqs -mh{"'"n#7Kܨg \ No newline at end of file diff --git a/fixture/9/1/33 b/fixture/9/1/33 deleted file mode 100644 index 3319b494b1..0000000000 --- a/fixture/9/1/33 +++ /dev/null @@ -1,2 +0,0 @@ -x `ov=<' -@:(BuݡvB[x44Ss<3ZuhG56I[fr[XWFģ&6X ]&xľJ87;|< \ No newline at end of file diff --git a/fixture/9/1/4 b/fixture/9/1/4 deleted file mode 100644 index 255ddf6807..0000000000 --- a/fixture/9/1/4 +++ /dev/null @@ -1,4 +0,0 @@ -x[HS϶ME (HìHŒ!YIQ]у$Z !4.=IPFD^}C:9'k^KvAN.E)-%ږ%TrٗeIYͼ=Kv/4 -A' -y}NZ~9*\f]d&\Z17_Ipꑇ;tdl-}LsQ?&*s*=5= _dץ BԵp<q_hد}1Ѵ#L;0uF>NEHj$B[8Jacw628`V&>lZk&g1ʴ8Źv7,ƙI[t]T%[Vk=Ir-? -< \ No newline at end of file diff --git a/fixture/9/1/5 b/fixture/9/1/5 deleted file mode 100644 index db4aaad720..0000000000 Binary files a/fixture/9/1/5 and /dev/null differ diff --git a/fixture/9/1/6 b/fixture/9/1/6 deleted file mode 100644 index dc9705c136..0000000000 Binary files a/fixture/9/1/6 and /dev/null differ diff --git a/fixture/9/1/7 b/fixture/9/1/7 deleted file mode 100644 index 0ecddaf5b3..0000000000 Binary files a/fixture/9/1/7 and /dev/null differ diff --git a/fixture/9/1/8 b/fixture/9/1/8 deleted file mode 100644 index 85231fc36b..0000000000 Binary files a/fixture/9/1/8 and /dev/null differ diff --git a/fixture/9/1/9 b/fixture/9/1/9 deleted file mode 100644 index aeb17db367..0000000000 Binary files a/fixture/9/1/9 and /dev/null differ diff --git a/fixture/9/2/.zarray b/fixture/9/2/.zarray deleted file mode 100644 index be1b7e3f28..0000000000 --- a/fixture/9/2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 100 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": " 1\n", - "\n", - "Slices with step > 1 are supported via ``get/set_basic_selection()``, ``get/set_orthogonal_selection()``, ``__getitem__`` and ``.oindex[]``. Negative steps are not supported." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "a = np.arange(10)\n", - "za = zarr.array(a, chunks=2)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 3, 5, 7, 9])" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "za[1::2]" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 0, 10, 2, 30, 4, 50, 6, 70, 8, 90])" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.oindex[1::2] = a[1::2] * 10\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Orthogonal (outer) indexing of multi-dimensional arrays\n", - "\n", - "Orthogonal (a.k.a. outer) indexing is supported with either Boolean or integer arrays, in combination with integers and slices. This functionality is provided via the ``get/set_orthogonal_selection()`` methods. For convenience, this functionality is also available via the ``.oindex[]`` property." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [ 3, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 11],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(15).reshape(5, 3)\n", - "za = zarr.array(a, chunks=(3, 2))\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# orthogonal indexing with Boolean arrays\n", - "ix0 = [False, True, False, True, False]\n", - "ix1 = [True, False, True]\n", - "za.get_orthogonal_selection((ix0, ix1))" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# orthogonal indexing with integer arrays\n", - "ix0 = [1, 3]\n", - "ix1 = [0, 2]\n", - "za.get_orthogonal_selection((ix0, ix1))" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 5],\n", - " [ 9, 11]])" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 3, 4, 5],\n", - " [ 9, 10, 11]])" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# combine with slice\n", - "za.oindex[[1, 3], :]" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 2],\n", - " [ 3, 5],\n", - " [ 6, 8],\n", - " [ 9, 11],\n", - " [12, 14]])" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# combine with slice\n", - "za.oindex[:, [0, 2]]" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [42, 4, 42],\n", - " [ 6, 7, 8],\n", - " [42, 10, 42],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items via Boolean selection\n", - "ix0 = [False, True, False, True, False]\n", - "ix1 = [True, False, True]\n", - "selection = ix0, ix1\n", - "value = 42\n", - "za.set_orthogonal_selection(selection, value)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [44, 4, 44],\n", - " [ 6, 7, 8],\n", - " [44, 10, 44],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1] = 44\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [46, 4, 46],\n", - " [ 6, 7, 8],\n", - " [46, 10, 46],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items via integer selection\n", - "ix0 = [1, 3]\n", - "ix1 = [0, 2]\n", - "selection = ix0, ix1\n", - "value = 46\n", - "za.set_orthogonal_selection(selection, value)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [48, 4, 48],\n", - " [ 6, 7, 8],\n", - " [48, 10, 48],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.oindex[ix0, ix1] = 48\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Coordinate indexing of multi-dimensional arrays\n", - "\n", - "Selecting arbitrary points from a multi-dimensional array by indexing with integer (coordinate) arrays is supported. This functionality is provided via the ``get/set_coordinate_selection()`` methods. For convenience, this functionality is also available via the ``.vindex[]`` property." - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [ 3, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 11],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(15).reshape(5, 3)\n", - "za = zarr.array(a, chunks=(3, 2))\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# get items\n", - "ix0 = [1, 3]\n", - "ix1 = [0, 2]\n", - "za.get_coordinate_selection((ix0, ix1))" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.vindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [42, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 42],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# set items\n", - "za.set_coordinate_selection((ix0, ix1), 42)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [44, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 44],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# alternative API\n", - "za.vindex[ix0, ix1] = 44\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Mask indexing of multi-dimensional arrays\n", - "\n", - "Selecting arbitrary points from a multi-dimensional array by a Boolean array is supported. This functionality is provided via the ``get/set_mask_selection()`` methods. For convenience, this functionality is also available via the ``.vindex[]`` property." - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [ 3, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 11],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(15).reshape(5, 3)\n", - "za = zarr.array(a, chunks=(3, 2))\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ix = np.zeros_like(a, dtype=bool)\n", - "ix[1, 0] = True\n", - "ix[3, 2] = True\n", - "za.get_mask_selection(ix)" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([ 3, 11])" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "za.vindex[ix]" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [42, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 42],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "za.set_mask_selection(ix, 42)\n", - "za[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0, 1, 2],\n", - " [44, 4, 5],\n", - " [ 6, 7, 8],\n", - " [ 9, 10, 44],\n", - " [12, 13, 14]])" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "za.vindex[ix] = 44\n", - "za[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Selecting fields from arrays with a structured dtype\n", - "\n", - "All ``get/set_selection_...()`` methods support a ``fields`` argument which allows retrieving/replacing data for a specific field or fields. Also h5py-like API is supported where fields can be provided within ``__getitem__``, ``.oindex[]`` and ``.vindex[]``." - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([(b'aaa', 1, 4.2), (b'bbb', 2, 8.4), (b'ccc', 3, 12.6)],\n", - " dtype=[('foo', 'S3'), ('bar', '\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ma\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'foo'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'baz'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m: only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid indices" - ] - } - ], - "source": [ - "a['foo', 'baz']" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([(b'aaa', 4.2), (b'bbb', 8.4), (b'ccc', 12.6)],\n", - " dtype=[('foo', 'S3'), ('baz', '", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mza\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'foo'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'baz'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 537\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 538\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpop_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 539\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 540\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 541\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mEllipsis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36mget_basic_selection\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 661\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 662\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 663\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_basic_selection_nd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 664\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 665\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_get_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_get_basic_selection_nd\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 701\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 702\u001b[0m \u001b[0;31m# setup indexer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 703\u001b[0;31m \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasicIndexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 704\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 705\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/indexing.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, selection, array)\u001b[0m\n\u001b[1;32m 275\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 276\u001b[0m raise IndexError('unsupported selection item for basic indexing; expected integer '\n\u001b[0;32m--> 277\u001b[0;31m 'or slice, got {!r}'.format(type(dim_sel)))\n\u001b[0m\u001b[1;32m 278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 279\u001b[0m \u001b[0mdim_indexers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim_indexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mIndexError\u001b[0m: unsupported selection item for basic indexing; expected integer or slice, got " - ] - } - ], - "source": [ - "za[['foo', 'baz']]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1D Benchmarking" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "800000000" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c = np.arange(100000000)\n", - "c.nbytes" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 480 ms, sys: 16 ms, total: 496 ms\n", - "Wall time: 141 ms\n" - ] - }, - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typeint64
Shape(100000000,)
Chunk shape(97657,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes800000000 (762.9M)
No. bytes stored11854081 (11.3M)
Storage ratio67.5
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (100000000,)\n", - "Chunk shape : (97657,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 800000000 (762.9M)\n", - "No. bytes stored : 11854081 (11.3M)\n", - "Storage ratio : 67.5\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time zc = zarr.array(c)\n", - "zc.info" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "121 ms ± 1.49 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit c.copy()" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "254 ms ± 942 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc[:]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### bool dense selection" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "9997476" - ] - }, - "execution_count": 57, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# relatively dense selection - 10%\n", - "ix_dense_bool = np.random.binomial(1, 0.1, size=c.shape[0]).astype(bool)\n", - "np.count_nonzero(ix_dense_bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "243 ms ± 5.8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "433 ms ± 6.49 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "548 ms ± 5.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [], - "source": [ - "import tempfile\n", - "import cProfile\n", - "import pstats\n", - "\n", - "def profile(statement, sort='time', restrictions=(7,)):\n", - " with tempfile.NamedTemporaryFile() as f:\n", - " cProfile.run(statement, filename=f.name)\n", - " pstats.Stats(f.name).sort_stats(sort).print_stats(*restrictions)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:17:48 2017 /tmp/tmpruua2rs_\n", - "\n", - " 98386 function calls in 0.483 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 83 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1025 0.197 0.000 0.197 0.000 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1024 0.149 0.000 0.159 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.044 0.000 0.231 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1024 0.009 0.000 0.009 0.000 {built-in method numpy.core.multiarray.count_nonzero}\n", - " 1025 0.007 0.000 0.238 0.000 ../zarr/indexing.py:541(__iter__)\n", - " 1024 0.006 0.000 0.207 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/index_tricks.py:26(ix_)\n", - " 2048 0.005 0.000 0.005 0.000 ../zarr/core.py:337()\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_dense_bool]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Method ``nonzero`` is being called internally within numpy to convert bool to int selections, no way to avoid." - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:18:06 2017 /tmp/tmp7_bautep\n", - "\n", - " 52382 function calls in 0.592 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 88 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 2 0.219 0.110 0.219 0.110 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1024 0.096 0.000 0.101 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2 0.094 0.047 0.094 0.047 ../zarr/indexing.py:630()\n", - " 1024 0.044 0.000 0.167 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1 0.029 0.029 0.029 0.029 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - " 1 0.021 0.021 0.181 0.181 ../zarr/indexing.py:603(__init__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_dense_bool]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "``.vindex[]`` is a bit slower, possibly because internally it converts to a coordinate array first." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### int dense selection" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "10000000" - ] - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ix_dense_int = np.random.choice(c.shape[0], size=c.shape[0]//10, replace=True)\n", - "ix_dense_int_sorted = ix_dense_int.copy()\n", - "ix_dense_int_sorted.sort()\n", - "len(ix_dense_int)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "62.2 ms ± 2.36 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_dense_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "355 ms ± 3.53 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_dense_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "351 ms ± 3.51 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_dense_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "128 ms ± 137 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_dense_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.71 s ± 5.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_dense_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.68 s ± 3.87 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_dense_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:09 2017 /tmp/tmpgmu5btr_\n", - "\n", - " 95338 function calls in 0.424 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 89 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.141 0.141 0.184 0.184 ../zarr/indexing.py:369(__init__)\n", - " 1024 0.099 0.000 0.106 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.046 0.000 0.175 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1025 0.027 0.000 0.027 0.000 ../zarr/indexing.py:424(__iter__)\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - " 1 0.010 0.010 0.010 0.010 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/function_base.py:1848(diff)\n", - " 1025 0.006 0.000 0.059 0.000 ../zarr/indexing.py:541(__iter__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_dense_int_sorted]')" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:13 2017 /tmp/tmpay1gvnx8\n", - "\n", - " 52362 function calls in 0.398 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 85 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 2 0.107 0.054 0.107 0.054 ../zarr/indexing.py:630()\n", - " 1024 0.091 0.000 0.096 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.041 0.000 0.160 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1 0.040 0.040 0.213 0.213 ../zarr/indexing.py:603(__init__)\n", - " 1 0.029 0.029 0.029 0.029 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - " 2048 0.011 0.000 0.011 0.000 ../zarr/indexing.py:695()\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_dense_int_sorted]')" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:20 2017 /tmp/tmpngsf6zpp\n", - "\n", - " 120946 function calls in 1.793 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 92 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 1.128 1.128 1.128 1.128 {method 'argsort' of 'numpy.ndarray' objects}\n", - " 1024 0.139 0.000 0.285 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1 0.132 0.132 1.422 1.422 ../zarr/indexing.py:369(__init__)\n", - " 1 0.120 0.120 0.120 0.120 {method 'take' of 'numpy.ndarray' objects}\n", - " 1024 0.116 0.000 0.123 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1025 0.034 0.000 0.034 0.000 ../zarr/indexing.py:424(__iter__)\n", - " 1 0.023 0.023 0.023 0.023 {built-in method numpy.core.multiarray.bincount}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_dense_int]')" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:19:22 2017 /tmp/tmpbskhj8de\n", - "\n", - " 50320 function calls in 1.730 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 86 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 1.116 1.116 1.116 1.116 {method 'argsort' of 'numpy.ndarray' objects}\n", - " 1024 0.133 0.000 0.275 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2 0.121 0.060 0.121 0.060 ../zarr/indexing.py:654()\n", - " 1024 0.113 0.000 0.119 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2 0.100 0.050 0.100 0.050 ../zarr/indexing.py:630()\n", - " 1 0.030 0.030 0.030 0.030 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.024 0.024 1.427 1.427 ../zarr/indexing.py:603(__init__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_dense_int]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When indices are not sorted, zarr needs to partially sort them so the occur in chunk order, so we only have to visit each chunk once. This sorting dominates the processing time and is unavoidable AFAIK." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### bool sparse selection" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "9932" - ] - }, - "execution_count": 75, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# relatively sparse selection\n", - "ix_sparse_bool = np.random.binomial(1, 0.0001, size=c.shape[0]).astype(bool)\n", - "np.count_nonzero(ix_sparse_bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "15.7 ms ± 38.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "156 ms ± 2.1 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "133 ms ± 2.76 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:20:09 2017 /tmp/tmpb7nqc9ax\n", - "\n", - " 98386 function calls in 0.191 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 83 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.093 0.000 0.098 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1025 0.017 0.000 0.017 0.000 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1024 0.007 0.000 0.007 0.000 {built-in method numpy.core.multiarray.count_nonzero}\n", - " 1024 0.007 0.000 0.129 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1025 0.005 0.000 0.052 0.000 ../zarr/indexing.py:541(__iter__)\n", - " 1024 0.005 0.000 0.025 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/index_tricks.py:26(ix_)\n", - " 2048 0.004 0.000 0.004 0.000 ../zarr/core.py:337()\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_sparse_bool]')" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:20:09 2017 /tmp/tmphsko8nvh\n", - "\n", - " 52382 function calls in 0.160 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 88 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.093 0.000 0.098 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2 0.017 0.008 0.017 0.008 {method 'nonzero' of 'numpy.ndarray' objects}\n", - " 1025 0.008 0.000 0.014 0.000 ../zarr/indexing.py:674(__iter__)\n", - " 1024 0.006 0.000 0.127 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2048 0.004 0.000 0.004 0.000 ../zarr/indexing.py:695()\n", - " 2054 0.003 0.000 0.003 0.000 ../zarr/core.py:337()\n", - " 1024 0.002 0.000 0.005 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/core/arrayprint.py:381(wrapper)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_sparse_bool]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### int sparse selection" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "10000" - ] - }, - "execution_count": 81, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ix_sparse_int = np.random.choice(c.shape[0], size=c.shape[0]//10000, replace=True)\n", - "ix_sparse_int_sorted = ix_sparse_int.copy()\n", - "ix_sparse_int_sorted.sort()\n", - "len(ix_sparse_int)" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "18.9 µs ± 392 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_sparse_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "20.3 µs ± 155 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n" - ] - } - ], - "source": [ - "%timeit c[ix_sparse_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "125 ms ± 296 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_sparse_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 85, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "109 ms ± 428 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_sparse_int_sorted]" - ] - }, - { - "cell_type": "code", - "execution_count": 86, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "132 ms ± 489 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[ix_sparse_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "108 ms ± 579 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc.vindex[ix_sparse_int]" - ] - }, - { - "cell_type": "code", - "execution_count": 88, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:21:12 2017 /tmp/tmp0b0o2quo\n", - "\n", - " 120946 function calls in 0.196 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 92 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.105 0.000 0.111 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 2048 0.006 0.000 0.013 0.000 /home/aliman/pyenv/zarr_20171023/lib/python3.6/site-packages/numpy/lib/index_tricks.py:26(ix_)\n", - " 1025 0.006 0.000 0.051 0.000 ../zarr/indexing.py:541(__iter__)\n", - " 1024 0.006 0.000 0.141 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2048 0.005 0.000 0.005 0.000 ../zarr/core.py:337()\n", - " 15373 0.004 0.000 0.010 0.000 {built-in method builtins.isinstance}\n", - " 1025 0.004 0.000 0.005 0.000 ../zarr/indexing.py:424(__iter__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.oindex[ix_sparse_int]')" - ] - }, - { - "cell_type": "code", - "execution_count": 89, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:21:19 2017 /tmp/tmpdwju98kn\n", - "\n", - " 50320 function calls in 0.167 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 86 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.105 0.000 0.111 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1025 0.009 0.000 0.017 0.000 ../zarr/indexing.py:674(__iter__)\n", - " 1024 0.006 0.000 0.142 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 2048 0.005 0.000 0.005 0.000 ../zarr/indexing.py:695()\n", - " 2054 0.004 0.000 0.004 0.000 ../zarr/core.py:337()\n", - " 1 0.003 0.003 0.162 0.162 ../zarr/core.py:591(_get_selection)\n", - " 1027 0.003 0.000 0.003 0.000 {method 'reshape' of 'numpy.ndarray' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc.vindex[ix_sparse_int]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For sparse selections, processing time is dominated by decompression, so we can't do any better." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### sparse bool selection as zarr array" - ] - }, - { - "cell_type": "code", - "execution_count": 90, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typebool
Shape(100000000,)
Chunk shape(390625,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes100000000 (95.4M)
No. bytes stored507131 (495.2K)
Storage ratio197.2
Chunks initialized256/256
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : bool\n", - "Shape : (100000000,)\n", - "Chunk shape : (390625,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 100000000 (95.4M)\n", - "No. bytes stored : 507131 (495.2K)\n", - "Storage ratio : 197.2\n", - "Chunks initialized : 256/256" - ] - }, - "execution_count": 90, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zix_sparse_bool = zarr.array(ix_sparse_bool)\n", - "zix_sparse_bool.info" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "387 ms ± 5.47 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zc.oindex[zix_sparse_bool]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### slice with step" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "80.3 ms ± 377 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit np.array(c[::2])" - ] - }, - { - "cell_type": "code", - "execution_count": 93, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "168 ms ± 837 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::2]" - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "136 ms ± 1.56 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::10]" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "104 ms ± 1.86 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::100]" - ] - }, - { - "cell_type": "code", - "execution_count": 96, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "100 ms ± 1.47 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit zc[::1000]" - ] - }, - { - "cell_type": "code", - "execution_count": 97, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:22:44 2017 /tmp/tmpg9dxqcpg\n", - "\n", - " 49193 function calls in 0.211 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 55 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1024 0.104 0.000 0.110 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1024 0.067 0.000 0.195 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1025 0.005 0.000 0.013 0.000 ../zarr/indexing.py:278(__iter__)\n", - " 2048 0.004 0.000 0.004 0.000 ../zarr/core.py:337()\n", - " 2050 0.003 0.000 0.003 0.000 ../zarr/indexing.py:90(ceildiv)\n", - " 1025 0.003 0.000 0.006 0.000 ../zarr/indexing.py:109(__iter__)\n", - " 1024 0.003 0.000 0.003 0.000 {method 'reshape' of 'numpy.ndarray' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zc[::2]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2D Benchmarking" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(100000000,)" - ] - }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 100, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(100000, 1000)" - ] - }, - "execution_count": 100, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "d = c.reshape(-1, 1000)\n", - "d.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typeint64
Shape(100000, 1000)
Chunk shape(3125, 32)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes800000000 (762.9M)
No. bytes stored39228864 (37.4M)
Storage ratio20.4
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (100000, 1000)\n", - "Chunk shape : (3125, 32)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 800000000 (762.9M)\n", - "No. bytes stored : 39228864 (37.4M)\n", - "Storage ratio : 20.4\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 101, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zd = zarr.array(d)\n", - "zd.info" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### bool orthogonal selection" - ] - }, - { - "cell_type": "code", - "execution_count": 102, - "metadata": {}, - "outputs": [], - "source": [ - "ix0 = np.random.binomial(1, 0.5, size=d.shape[0]).astype(bool)\n", - "ix1 = np.random.binomial(1, 0.5, size=d.shape[1]).astype(bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "101 ms ± 577 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit d[np.ix_(ix0, ix1)]" - ] - }, - { - "cell_type": "code", - "execution_count": 104, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "373 ms ± 5.45 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zd.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### int orthogonal selection" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [], - "source": [ - "ix0 = np.random.choice(d.shape[0], size=int(d.shape[0] * .5), replace=True)\n", - "ix1 = np.random.choice(d.shape[1], size=int(d.shape[1] * .5), replace=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "174 ms ± 4.13 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit d[np.ix_(ix0, ix1)]" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "566 ms ± 12.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zd.oindex[ix0, ix1]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### coordinate (point) selection" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "10000000" - ] - }, - "execution_count": 108, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "n = int(d.size * .1)\n", - "ix0 = np.random.choice(d.shape[0], size=n, replace=True)\n", - "ix1 = np.random.choice(d.shape[1], size=n, replace=True)\n", - "n" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "243 ms ± 3.37 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit d[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 110, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2.03 s ± 17 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit zd.vindex[ix0, ix1]" - ] - }, - { - "cell_type": "code", - "execution_count": 111, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Wed Nov 8 17:24:31 2017 /tmp/tmp7c68z70p\n", - "\n", - " 62673 function calls in 2.065 seconds\n", - "\n", - " Ordered by: internal time\n", - " List reduced from 88 to 7 due to restriction <7>\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 1.112 1.112 1.112 1.112 {method 'argsort' of 'numpy.ndarray' objects}\n", - " 3 0.244 0.081 0.244 0.081 ../zarr/indexing.py:654()\n", - " 3 0.193 0.064 0.193 0.064 ../zarr/indexing.py:630()\n", - " 1024 0.170 0.000 0.350 0.000 ../zarr/core.py:849(_chunk_getitem)\n", - " 1024 0.142 0.000 0.151 0.000 ../zarr/core.py:1028(_decode_chunk)\n", - " 1 0.044 0.044 0.044 0.044 {built-in method numpy.core.multiarray.ravel_multi_index}\n", - " 1 0.043 0.043 1.676 1.676 ../zarr/indexing.py:603(__init__)\n", - "\n", - "\n" - ] - } - ], - "source": [ - "profile('zd.vindex[ix0, ix1]')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Points need to be partially sorted so all points in the same chunk are grouped and processed together. This requires ``argsort`` which dominates time." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## h5py comparison\n", - "\n", - "N.B., not really fair because using slower compressor, but for interest..." - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [], - "source": [ - "import h5py\n", - "import tempfile" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [], - "source": [ - "h5f = h5py.File(tempfile.mktemp(), driver='core', backing_store=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 79, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "hc = h5f.create_dataset('c', data=c, compression='gzip', compression_opts=1, chunks=zc.chunks, shuffle=True)\n", - "hc" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.16 s, sys: 172 ms, total: 1.33 s\n", - "Wall time: 1.32 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([ 0, 1, 2, ..., 99999997, 99999998, 99999999])" - ] - }, - "execution_count": 80, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time hc[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.11 s, sys: 0 ns, total: 1.11 s\n", - "Wall time: 1.11 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([ 1063, 28396, 37229, ..., 99955875, 99979354, 99995791])" - ] - }, - "execution_count": 81, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time hc[ix_sparse_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [], - "source": [ - "# # this is pathological, takes minutes \n", - "# %time hc[ix_dense_bool]" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 38.3 s, sys: 136 ms, total: 38.4 s\n", - "Wall time: 38.1 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([ 0, 1000, 2000, ..., 99997000, 99998000, 99999000])" - ] - }, - "execution_count": 83, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# this is pretty slow\n", - "%time hc[::1000]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/blosc_microbench.ipynb b/notebooks/blosc_microbench.ipynb deleted file mode 100644 index 9361d8e95b..0000000000 --- a/notebooks/blosc_microbench.ipynb +++ /dev/null @@ -1,200 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.0.1'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numpy as np\n", - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10 loops, best of 3: 110 ms per loop\n", - "1 loop, best of 3: 235 ms per loop\n", - "Array((100000000,), int64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 11.2M; ratio: 67.8; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='i8')\n", - "data = np.arange(100000000, dtype='i8')\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 loop, best of 3: 331 ms per loop\n", - "1 loop, best of 3: 246 ms per loop\n", - "Array((100000000,), float64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 724.8M; ratio: 1.1; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='f8')\n", - "data = np.random.normal(size=100000000)\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.0.2.dev0+dirty'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numpy as np\n", - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10 loops, best of 3: 92.7 ms per loop\n", - "1 loop, best of 3: 230 ms per loop\n", - "Array((100000000,), int64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 11.2M; ratio: 67.8; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='i8')\n", - "data = np.arange(100000000, dtype='i8')\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 loop, best of 3: 338 ms per loop\n", - "1 loop, best of 3: 253 ms per loop\n", - "Array((100000000,), float64, chunks=(200000,), order=C)\n", - " nbytes: 762.9M; nbytes_stored: 724.8M; ratio: 1.1; initialized: 500/500\n", - " compressor: Blosc(cname='lz4', clevel=5, shuffle=1)\n", - " store: dict\n" - ] - } - ], - "source": [ - "z = zarr.empty(shape=100000000, chunks=200000, dtype='f8')\n", - "data = np.random.normal(size=100000000)\n", - "%timeit z[:] = data\n", - "%timeit z[:]\n", - "print(z)\n", - "assert np.all(z[:] == data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/notebooks/dask_2d_subset.ipynb b/notebooks/dask_2d_subset.ipynb deleted file mode 100644 index 6e88b510d5..0000000000 --- a/notebooks/dask_2d_subset.ipynb +++ /dev/null @@ -1,869 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook has some profiling of Dask used to make a selection along both first and second axes of a large-ish multidimensional array. The use case is making selections of genotype data, e.g., as required for making a web-browser for genotype data as in www.malariagen.net/apps/ag1000g." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 2.1.1\n", - "dask 0.11.0\n" - ] - } - ], - "source": [ - "import zarr; print('zarr', zarr.__version__)\n", - "import dask; print('dask', dask.__version__)\n", - "import dask.array as da\n", - "import numpy as np" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Real data" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Group(/, 8)\n", - " arrays: 1; samples\n", - " groups: 7; 2L, 2R, 3L, 3R, UNKN, X, Y_unplaced\n", - " store: DirectoryStore" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# here's the real data\n", - "callset = zarr.open_group('/kwiat/2/coluzzi/ag1000g/data/phase1/release/AR3.1/variation/main/zarr2/zstd/ag1000g.phase1.ar3',\n", - " mode='r')\n", - "callset" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array(/3R/calldata/genotype, (22632425, 765, 2), int8, chunks=(13107, 40, 2), order=C)\n", - " nbytes: 32.2G; nbytes_stored: 1.0G; ratio: 31.8; initialized: 34540/34540\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: DirectoryStore" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# here's the array we're going to work with\n", - "g = callset['3R/calldata/genotype']\n", - "g" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 4 ms, sys: 0 ns, total: 4 ms\n", - "Wall time: 5.13 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# wrap as dask array with very simple chunking of first dim only\n", - "%time gd = da.from_array(g, chunks=(g.chunks[0], None, None))\n", - "gd" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "((22632425,), dtype('bool'), 13167162)" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# load condition used to make selection on first axis\n", - "dim0_condition = callset['3R/variants/FILTER_PASS'][:]\n", - "dim0_condition.shape, dim0_condition.dtype, np.count_nonzero(dim0_condition)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "# invent a random selection for second axis\n", - "dim1_indices = sorted(np.random.choice(765, size=100, replace=False))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.3 s, sys: 256 ms, total: 15.5 s\n", - "Wall time: 15.5 s\n" - ] - }, - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# setup the 2D selection - this is the slow bit\n", - "%time gd_sel = gd[dim0_condition][:, dim1_indices]\n", - "gd_sel" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.21 s, sys: 152 ms, total: 1.36 s\n", - "Wall time: 316 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " ..., \n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 1],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]],\n", - "\n", - " [[0, 0],\n", - " [0, 0],\n", - " [0, 0],\n", - " ..., \n", - " [0, 0],\n", - " [0, 0],\n", - " [0, 0]]], dtype=int8)" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# now load a slice from this new selection - quick!\n", - "%time gd_sel[1000000:1100000].compute(optimize_graph=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 105406881 function calls (79072145 primitive calls) in 26.182 seconds\n", - "\n", - " Ordered by: internal time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - "13167268/6 6.807 0.000 9.038 1.506 slicing.py:623(check_index)\n", - " 2 4.713 2.356 5.831 2.916 slicing.py:398(partition_by_size)\n", - "13167270/2 4.470 0.000 8.763 4.382 slicing.py:540(posify_index)\n", - " 52669338 4.118 0.000 4.119 0.000 {built-in method builtins.isinstance}\n", - " 2 2.406 1.203 8.763 4.382 slicing.py:563()\n", - " 1 0.875 0.875 0.875 0.875 slicing.py:44()\n", - " 13182474 0.600 0.000 0.600 0.000 {built-in method builtins.len}\n", - " 2 0.527 0.264 0.527 0.264 slicing.py:420(issorted)\n", - " 13189168 0.520 0.000 0.520 0.000 {method 'append' of 'list' objects}\n", - " 2 0.271 0.136 0.271 0.136 slicing.py:479()\n", - " 2 0.220 0.110 0.220 0.110 {built-in method builtins.sorted}\n", - " 1 0.162 0.162 0.162 0.162 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 2 0.113 0.056 26.071 13.035 core.py:1024(__getitem__)\n", - " 2 0.112 0.056 6.435 3.217 slicing.py:441(take_sorted)\n", - " 1 0.111 0.111 26.182 26.182 :1()\n", - " 2 0.060 0.030 24.843 12.422 slicing.py:142(slice_with_newaxes)\n", - " 106/3 0.039 0.000 1.077 0.359 slicing.py:15(sanitize_index)\n", - " 3 0.037 0.012 0.037 0.012 {built-in method _hashlib.openssl_md5}\n", - " 6726 0.012 0.000 0.017 0.000 slicing.py:567(insert_many)\n", - " 3364 0.004 0.000 0.021 0.000 slicing.py:156()\n", - " 20178 0.003 0.000 0.003 0.000 {method 'pop' of 'list' objects}\n", - " 8 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n", - " 2 0.000 0.000 25.920 12.960 slicing.py:60(slice_array)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 106/4 0.000 0.000 0.037 0.009 utils.py:502(__call__)\n", - " 100 0.000 0.000 0.000 0.000 arrayprint.py:340(array2string)\n", - " 2 0.000 0.000 0.037 0.019 base.py:343(tokenize)\n", - " 100 0.000 0.000 0.000 0.000 {built-in method builtins.repr}\n", - " 2 0.000 0.000 24.763 12.381 slicing.py:170(slice_wrap_lists)\n", - " 108 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 2 0.000 0.000 6.962 3.481 slicing.py:487(take)\n", - " 1 0.000 0.000 26.182 26.182 {built-in method builtins.exec}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 1 0.000 0.000 0.037 0.037 base.py:314(normalize_array)\n", - " 2/1 0.000 0.000 0.000 0.000 base.py:270(normalize_seq)\n", - " 116 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 100 0.000 0.000 0.000 0.000 numeric.py:1835(array_str)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:47()\n", - " 6 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 2 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n", - " 100 0.000 0.000 0.000 0.000 inspect.py:441(getmro)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n", - " 4 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 100 0.000 0.000 0.000 0.000 {method 'item' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 8 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 5 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 7 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 5 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 3 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 2 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 1 0.000 0.000 0.000 0.000 functoolz.py:11(identity)\n", - " 4 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "# what's taking so long?\n", - "import cProfile\n", - "cProfile.run('gd[dim0_condition][:, dim1_indices]', sort='time')" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 105406881 function calls (79072145 primitive calls) in 25.630 seconds\n", - "\n", - " Ordered by: cumulative time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.000 0.000 25.630 25.630 {built-in method builtins.exec}\n", - " 1 0.107 0.107 25.630 25.630 :1()\n", - " 2 0.102 0.051 25.523 12.761 core.py:1024(__getitem__)\n", - " 2 0.001 0.000 25.381 12.691 slicing.py:60(slice_array)\n", - " 2 0.049 0.024 24.214 12.107 slicing.py:142(slice_with_newaxes)\n", - " 2 0.000 0.000 24.147 12.073 slicing.py:170(slice_wrap_lists)\n", - "13167268/6 6.664 0.000 8.855 1.476 slicing.py:623(check_index)\n", - "13167270/2 4.354 0.000 8.466 4.233 slicing.py:540(posify_index)\n", - " 2 2.277 1.139 8.465 4.233 slicing.py:563()\n", - " 2 0.000 0.000 6.826 3.413 slicing.py:487(take)\n", - " 2 0.111 0.056 6.331 3.165 slicing.py:441(take_sorted)\n", - " 2 4.628 2.314 5.704 2.852 slicing.py:398(partition_by_size)\n", - " 52669338 4.026 0.000 4.026 0.000 {built-in method builtins.isinstance}\n", - " 106/3 0.071 0.001 1.167 0.389 slicing.py:15(sanitize_index)\n", - " 1 0.943 0.943 0.943 0.943 slicing.py:44()\n", - " 13182474 0.581 0.000 0.581 0.000 {built-in method builtins.len}\n", - " 13189168 0.497 0.000 0.497 0.000 {method 'append' of 'list' objects}\n", - " 2 0.495 0.248 0.495 0.248 slicing.py:420(issorted)\n", - " 2 0.281 0.141 0.281 0.141 slicing.py:479()\n", - " 2 0.234 0.117 0.234 0.117 {built-in method builtins.sorted}\n", - " 1 0.152 0.152 0.152 0.152 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.039 0.020 base.py:343(tokenize)\n", - " 106/4 0.000 0.000 0.039 0.010 utils.py:502(__call__)\n", - " 1 0.000 0.000 0.039 0.039 base.py:314(normalize_array)\n", - " 3 0.039 0.013 0.039 0.013 {built-in method _hashlib.openssl_md5}\n", - " 3364 0.003 0.000 0.019 0.000 slicing.py:156()\n", - " 6726 0.012 0.000 0.016 0.000 slicing.py:567(insert_many)\n", - " 20178 0.003 0.000 0.003 0.000 {method 'pop' of 'list' objects}\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n", - " 8 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n", - " 4 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n", - " 2 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n", - " 2/1 0.000 0.000 0.000 0.000 base.py:270(normalize_seq)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 100 0.000 0.000 0.000 0.000 {built-in method builtins.repr}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:47()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 100 0.000 0.000 0.000 0.000 numeric.py:1835(array_str)\n", - " 100 0.000 0.000 0.000 0.000 arrayprint.py:340(array2string)\n", - " 108 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 8 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 2 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 116 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 6 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 100 0.000 0.000 0.000 0.000 {method 'item' of 'numpy.ndarray' objects}\n", - " 100 0.000 0.000 0.000 0.000 inspect.py:441(getmro)\n", - " 2 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 5 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 3 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 5 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 7 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 2 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 4 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 2 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - " 1 0.000 0.000 0.000 0.000 functoolz.py:11(identity)\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "cProfile.run('gd[dim0_condition][:, dim1_indices]', sort='cumtime')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Synthetic data" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array((20000000, 200, 2), int8, chunks=(10000, 100, 2), order=C)\n", - " nbytes: 7.5G; nbytes_stored: 2.7G; ratio: 2.8; initialized: 4000/4000\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: dict" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# create a synthetic dataset for profiling\n", - "a = zarr.array(np.random.randint(-1, 4, size=(20000000, 200, 2), dtype='i1'),\n", - " chunks=(10000, 100, 2), compressor=zarr.Blosc(cname='zstd', clevel=1, shuffle=2))\n", - "a" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# create a synthetic selection for first axis\n", - "c = np.random.randint(0, 2, size=a.shape[0], dtype=bool)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# create a synthetic selection for second axis\n", - "s = sorted(np.random.choice(a.shape[1], size=100, replace=False))" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 208 ms, sys: 0 ns, total: 208 ms\n", - "Wall time: 206 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time d = da.from_array(a, chunks=(a.chunks[0], None, None))\n", - "d" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 12 s, sys: 200 ms, total: 12.2 s\n", - "Wall time: 12.2 s\n" - ] - } - ], - "source": [ - "%time ds = d[c][:, s]" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 80095589 function calls (60091843 primitive calls) in 19.467 seconds\n", - "\n", - " Ordered by: internal time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - "10001773/6 4.872 0.000 6.456 1.076 slicing.py:623(check_index)\n", - " 2 3.517 1.758 4.357 2.179 slicing.py:398(partition_by_size)\n", - "10001775/2 3.354 0.000 6.484 3.242 slicing.py:540(posify_index)\n", - " 40007358 2.965 0.000 2.965 0.000 {built-in method builtins.isinstance}\n", - " 2 1.749 0.875 6.484 3.242 slicing.py:563()\n", - " 1 0.878 0.878 0.878 0.878 slicing.py:44()\n", - " 10019804 0.451 0.000 0.451 0.000 {built-in method builtins.len}\n", - " 10027774 0.392 0.000 0.392 0.000 {method 'append' of 'list' objects}\n", - " 2 0.363 0.181 0.363 0.181 slicing.py:420(issorted)\n", - " 2 0.270 0.135 4.786 2.393 slicing.py:441(take_sorted)\n", - " 1 0.207 0.207 0.207 0.207 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 2 0.158 0.079 0.158 0.079 {built-in method builtins.sorted}\n", - " 1 0.094 0.094 19.467 19.467 :1()\n", - " 2 0.079 0.040 19.373 9.686 core.py:1024(__getitem__)\n", - " 2 0.035 0.017 18.147 9.074 slicing.py:142(slice_with_newaxes)\n", - " 3 0.033 0.011 0.033 0.011 {built-in method _hashlib.openssl_md5}\n", - " 106/3 0.028 0.000 1.112 0.371 slicing.py:15(sanitize_index)\n", - " 8002 0.015 0.000 0.020 0.000 slicing.py:567(insert_many)\n", - " 4002 0.004 0.000 0.023 0.000 slicing.py:156()\n", - " 24006 0.003 0.000 0.003 0.000 {method 'pop' of 'list' objects}\n", - " 8 0.001 0.000 0.001 0.000 {method 'update' of 'dict' objects}\n", - " 2 0.001 0.000 0.001 0.000 slicing.py:479()\n", - " 2 0.000 0.000 19.259 9.630 slicing.py:60(slice_array)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 106/4 0.000 0.000 0.034 0.008 utils.py:502(__call__)\n", - " 2 0.000 0.000 18.089 9.044 slicing.py:170(slice_wrap_lists)\n", - " 100 0.000 0.000 0.000 0.000 arrayprint.py:340(array2string)\n", - " 100 0.000 0.000 0.000 0.000 {built-in method builtins.repr}\n", - " 108 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 2 0.000 0.000 5.149 2.574 slicing.py:487(take)\n", - " 2 0.000 0.000 0.034 0.017 base.py:343(tokenize)\n", - " 1 0.000 0.000 0.033 0.033 base.py:314(normalize_array)\n", - " 116 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 2/1 0.000 0.000 0.000 0.000 base.py:270(normalize_seq)\n", - " 6 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 100 0.000 0.000 0.000 0.000 numeric.py:1835(array_str)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:47()\n", - " 1 0.000 0.000 19.467 19.467 {built-in method builtins.exec}\n", - " 100 0.000 0.000 0.000 0.000 inspect.py:441(getmro)\n", - " 8 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 4 0.000 0.000 0.001 0.000 dicttoolz.py:19(merge)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 100 0.000 0.000 0.000 0.000 {method 'item' of 'numpy.ndarray' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 2 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 3 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 2 0.000 0.000 0.001 0.000 exceptions.py:15(merge)\n", - " 7 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 4 0.000 0.000 0.001 0.000 functoolz.py:217(__call__)\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 4 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 5 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 2 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 5 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 8 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 2 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.000 0.000 functoolz.py:11(identity)\n", - " 4 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 2 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "cProfile.run('d[c][:, s]', sort='time')" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 452 ms, sys: 8 ms, total: 460 ms\n", - "Wall time: 148 ms\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[[ 2, -1],\n", - " [ 2, 3],\n", - " [ 3, 0],\n", - " ..., \n", - " [ 1, 3],\n", - " [-1, -1],\n", - " [ 1, 1]],\n", - "\n", - " [[ 1, -1],\n", - " [ 2, 2],\n", - " [-1, 2],\n", - " ..., \n", - " [ 2, -1],\n", - " [ 1, 3],\n", - " [-1, -1]],\n", - "\n", - " [[ 1, -1],\n", - " [ 2, 0],\n", - " [ 0, 3],\n", - " ..., \n", - " [ 2, 2],\n", - " [ 3, 2],\n", - " [ 0, 2]],\n", - "\n", - " ..., \n", - " [[ 1, 2],\n", - " [ 3, -1],\n", - " [ 2, 1],\n", - " ..., \n", - " [ 1, 2],\n", - " [ 1, 0],\n", - " [ 2, 0]],\n", - "\n", - " [[ 1, 2],\n", - " [ 1, 0],\n", - " [ 2, 3],\n", - " ..., \n", - " [-1, 2],\n", - " [ 3, 3],\n", - " [ 1, -1]],\n", - "\n", - " [[-1, 3],\n", - " [ 2, 2],\n", - " [ 1, 1],\n", - " ..., \n", - " [ 3, 3],\n", - " [ 0, 0],\n", - " [ 0, 2]]], dtype=int8)" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time ds[1000000:1100000].compute(optimize_graph=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 80055494 function calls (60052157 primitive calls) in 19.425 seconds\n", - "\n", - " Ordered by: internal time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - "10001670/3 5.032 0.000 6.671 2.224 slicing.py:623(check_index)\n", - " 1 3.459 3.459 4.272 4.272 slicing.py:398(partition_by_size)\n", - "10001671/1 3.287 0.000 6.378 6.378 slicing.py:540(posify_index)\n", - " 40006704 2.999 0.000 2.999 0.000 {built-in method builtins.isinstance}\n", - " 1 1.731 1.731 6.378 6.378 slicing.py:563()\n", - " 1 0.849 0.849 0.849 0.849 slicing.py:44()\n", - " 10011685 0.433 0.000 0.433 0.000 {built-in method builtins.len}\n", - " 10015670 0.381 0.000 0.381 0.000 {method 'append' of 'list' objects}\n", - " 1 0.355 0.355 0.355 0.355 slicing.py:420(issorted)\n", - " 1 0.196 0.196 0.196 0.196 {method 'tolist' of 'numpy.ndarray' objects}\n", - " 1 0.193 0.193 0.193 0.193 slicing.py:479()\n", - " 1 0.157 0.157 0.157 0.157 {built-in method builtins.sorted}\n", - " 1 0.085 0.085 4.707 4.707 slicing.py:441(take_sorted)\n", - " 1 0.085 0.085 19.425 19.425 :1()\n", - " 1 0.079 0.079 19.341 19.341 core.py:1024(__getitem__)\n", - " 1 0.034 0.034 18.157 18.157 slicing.py:142(slice_with_newaxes)\n", - " 2 0.033 0.017 0.033 0.017 {built-in method _hashlib.openssl_md5}\n", - " 1 0.026 0.026 1.071 1.071 slicing.py:15(sanitize_index)\n", - " 4001 0.007 0.000 0.009 0.000 slicing.py:567(insert_many)\n", - " 2001 0.002 0.000 0.011 0.000 slicing.py:156()\n", - " 12003 0.001 0.000 0.001 0.000 {method 'pop' of 'list' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n", - " 1 0.000 0.000 19.228 19.228 slicing.py:60(slice_array)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:464()\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:162()\n", - " 1 0.000 0.000 0.033 0.033 base.py:314(normalize_array)\n", - " 1 0.000 0.000 18.111 18.111 slicing.py:170(slice_wrap_lists)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:465()\n", - " 1 0.000 0.000 5.062 5.062 slicing.py:487(take)\n", - " 1 0.000 0.000 0.033 0.033 base.py:343(tokenize)\n", - " 1 0.000 0.000 19.425 19.425 {built-in method builtins.exec}\n", - " 2 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n", - " 3 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n", - " 2 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n", - " 1 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n", - " 2 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n", - " 4 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n", - " 2 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n", - " 1 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n", - " 1 0.000 0.000 0.000 0.000 core.py:794(__init__)\n", - " 4 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:467()\n", - " 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n", - " 4 0.000 0.000 0.000 0.000 slicing.py:183()\n", - " 2 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:192()\n", - " 4 0.000 0.000 0.000 0.000 slicing.py:207()\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:475()\n", - " 2 0.000 0.000 0.033 0.017 utils.py:502(__call__)\n", - " 2 0.000 0.000 0.000 0.000 slicing.py:125()\n", - " 2 0.000 0.000 0.000 0.000 core.py:1043()\n", - " 4 0.000 0.000 0.000 0.000 slicing.py:197()\n", - " 1 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:613()\n", - " 1 0.000 0.000 0.000 0.000 core.py:1452()\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:149()\n", - " 2 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 slicing.py:150()\n", - " 1 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "# problem is in fact just the dim0 selection\n", - "cProfile.run('d[c]', sort='time')" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/notebooks/dask_copy.ipynb b/notebooks/dask_copy.ipynb deleted file mode 100644 index 5cb712508c..0000000000 --- a/notebooks/dask_copy.ipynb +++ /dev/null @@ -1,1518 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Profile array copy via dask threaded scheduler" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook profiles a very simple array copy operation, using synthetic data." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 1.0.1.dev18+dirty\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " Loading BokehJS ...\n", - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "\n", - "(function(global) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " if (typeof (window._bokeh_onload_callbacks) === \"undefined\") {\n", - " window._bokeh_onload_callbacks = [];\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", - " delete window._bokeh_onload_callbacks\n", - " console.info(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(js_urls, callback) {\n", - " window._bokeh_onload_callbacks.push(callback);\n", - " if (window._bokeh_is_loading > 0) {\n", - " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " }\n", - " if (js_urls == null || js_urls.length === 0) {\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " window._bokeh_is_loading = js_urls.length;\n", - " for (var i = 0; i < js_urls.length; i++) {\n", - " var url = js_urls[i];\n", - " var s = document.createElement('script');\n", - " s.src = url;\n", - " s.async = false;\n", - " s.onreadystatechange = s.onload = function() {\n", - " window._bokeh_is_loading--;\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", - " run_callbacks()\n", - " }\n", - " };\n", - " s.onerror = function() {\n", - " console.warn(\"failed to load library \" + url);\n", - " };\n", - " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", - " }\n", - " };\n", - "\n", - " var js_urls = ['https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-compiler-0.12.0.min.js'];\n", - "\n", - " var inline_js = [\n", - " function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - " \n", - " function(Bokeh) {\n", - " Bokeh.$(\"#d4821cb3-378c-411d-a941-d0708c0c532b\").text(\"BokehJS successfully loaded\");\n", - " },\n", - " function(Bokeh) {\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " }\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " for (var i = 0; i < inline_js.length; i++) {\n", - " inline_js[i](window.Bokeh);\n", - " }\n", - " }\n", - "\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", - " run_inline_js();\n", - " } else {\n", - " load_libs(js_urls, function() {\n", - " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", - " run_inline_js();\n", - " });\n", - " }\n", - "}(this));" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "print('zarr', zarr.__version__)\n", - "from zarr import blosc\n", - "import numpy as np\n", - "import h5py\n", - "import bcolz\n", - "# don't let bcolz use multiple threads internally, we want to \n", - "# see whether dask can make good use of multiple CPUs\n", - "bcolz.set_nthreads(1)\n", - "import multiprocessing\n", - "import dask\n", - "import dask.array as da\n", - "from dask.diagnostics import Profiler, ResourceProfiler, CacheProfiler\n", - "from dask.diagnostics.profile_visualize import visualize\n", - "from cachey import nbytes\n", - "import bokeh\n", - "from bokeh.io import output_notebook\n", - "output_notebook()" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import tempfile\n", - "import operator\n", - "from functools import reduce\n", - "from zarr.util import human_readable_size\n", - "\n", - "\n", - "def h5fmem(**kwargs):\n", - " \"\"\"Convenience function to create an in-memory HDF5 file.\"\"\"\n", - "\n", - " # need a file name even tho nothing is ever written\n", - " fn = tempfile.mktemp()\n", - "\n", - " # file creation args\n", - " kwargs['mode'] = 'w'\n", - " kwargs['driver'] = 'core'\n", - " kwargs['backing_store'] = False\n", - "\n", - " # open HDF5 file\n", - " h5f = h5py.File(fn, **kwargs)\n", - "\n", - " return h5f\n", - "\n", - "\n", - "def h5d_diagnostics(d):\n", - " \"\"\"Print some diagnostics on an HDF5 dataset.\"\"\"\n", - " \n", - " print(d)\n", - " nbytes = reduce(operator.mul, d.shape) * d.dtype.itemsize\n", - " cbytes = d._id.get_storage_size()\n", - " if cbytes > 0:\n", - " ratio = nbytes / cbytes\n", - " else:\n", - " ratio = np.inf\n", - " r = ' compression: %s' % d.compression\n", - " r += '; compression_opts: %s' % d.compression_opts\n", - " r += '; shuffle: %s' % d.shuffle\n", - " r += '\\n nbytes: %s' % human_readable_size(nbytes)\n", - " r += '; nbytes_stored: %s' % human_readable_size(cbytes)\n", - " r += '; ratio: %.1f' % ratio\n", - " r += '; chunks: %s' % str(d.chunks)\n", - " print(r)\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "def profile_dask_copy(src, dst, chunks, num_workers=multiprocessing.cpu_count(), dt=0.1, lock=True):\n", - " dsrc = da.from_array(src, chunks=chunks)\n", - " with Profiler() as prof, ResourceProfiler(dt=dt) as rprof:\n", - " da.store(dsrc, dst, num_workers=num_workers, lock=lock)\n", - " visualize([prof, rprof], min_border_top=60, min_border_bottom=60)\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## NumPy arrays" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1314, 2727, 2905, ..., 1958, 1325, 1971], dtype=uint16)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# a1 = np.arange(400000000, dtype='i4')\n", - "a1 = np.random.normal(2000, 1000, size=200000000).astype('u2')\n", - "a1" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'381.5M'" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "human_readable_size(a1.nbytes)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "a2 = np.empty_like(a1)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "chunks = 2**20, # 4M" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 56 ms, sys: 36 ms, total: 92 ms\n", - "Wall time: 91.7 ms\n" - ] - } - ], - "source": [ - "%time a2[:] = a1" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(a1, a2, chunks, lock=True, dt=.01)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(a1, a2, chunks, lock=False, dt=.01)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Zarr arrays (in-memory)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((200000000,), uint16, chunks=(1048576,), order=C)\n", - " compression: blosc; compression_opts: {'clevel': 1, 'cname': 'lz4', 'shuffle': 2}\n", - " nbytes: 381.5M; nbytes_stored: 318.2M; ratio: 1.2; initialized: 191/191\n", - " store: builtins.dict" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z1 = zarr.array(a1, chunks=chunks, compression='blosc', \n", - " compression_opts=dict(cname='lz4', clevel=1, shuffle=2))\n", - "z1" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((200000000,), uint16, chunks=(1048576,), order=C)\n", - " compression: blosc; compression_opts: {'clevel': 1, 'cname': 'lz4', 'shuffle': 2}\n", - " nbytes: 381.5M; nbytes_stored: 294; ratio: 1360544.2; initialized: 0/191\n", - " store: builtins.dict" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z2 = zarr.empty_like(z1)\n", - "z2" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(z1, z2, chunks, lock=True, dt=.02)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(z1, z2, chunks, lock=False, dt=0.02)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3 loops, best of 5: 251 ms per loop\n" - ] - } - ], - "source": [ - "# for comparison, using blosc internal threads\n", - "%timeit -n3 -r5 z2[:] = z1" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " " - ] - } - ], - "source": [ - "%prun z2[:] = z1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Without the dask lock, we get better CPU utilisation. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## HDF5 datasets (in-memory)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "h5f = h5fmem()\n", - "h5f" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " compression: lzf; compression_opts: None; shuffle: True\n", - " nbytes: 381.5M; nbytes_stored: 357.4M; ratio: 1.1; chunks: (1048576,)\n" - ] - } - ], - "source": [ - "h1 = h5f.create_dataset('h1', data=a1, chunks=chunks, compression='lzf', shuffle=True)\n", - "h5d_diagnostics(h1)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " compression: lzf; compression_opts: None; shuffle: True\n", - " nbytes: 762.9M; nbytes_stored: 0; ratio: inf; chunks: (1048576,)\n" - ] - } - ], - "source": [ - "h2 = h5f.create_dataset('h2', shape=h1.shape, chunks=h1.chunks, \n", - " compression=h1.compression, compression_opts=h1.compression_opts, \n", - " shuffle=h1.shuffle)\n", - "h5d_diagnostics(h2)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(h1, h2, chunks, lock=True, dt=0.1)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(h1, h2, chunks, lock=False, dt=0.1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bcolz carrays (in-memory)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "carray((200000000,), uint16)\n", - " nbytes := 381.47 MB; cbytes := 318.98 MB; ratio: 1.20\n", - " cparams := cparams(clevel=1, shuffle=2, cname='lz4', quantize=0)\n", - " chunklen := 1048576; chunksize: 2097152; blocksize: 16384\n", - "[1314 2727 2905 ..., 1958 1325 1971]" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c1 = bcolz.carray(a1, chunklen=chunks[0],\n", - " cparams=bcolz.cparams(cname='lz4', clevel=1, shuffle=2))\n", - "c1" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "carray((200000000,), uint16)\n", - " nbytes := 381.47 MB; cbytes := 2.00 MB; ratio: 190.73\n", - " cparams := cparams(clevel=1, shuffle=2, cname='lz4', quantize=0)\n", - " chunklen := 1048576; chunksize: 2097152; blocksize: 4096\n", - "[0 0 0 ..., 0 0 0]" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c2 = bcolz.zeros(a1.shape, chunklen=chunks[0], dtype=a1.dtype, \n", - " cparams=bcolz.cparams(cname='lz4', clevel=1, shuffle=2))\n", - "c2" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "profile_dask_copy(c1, c2, chunks, lock=True, dt=0.05)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# not sure it's safe to use bcolz without a lock, but what the heck...\n", - "profile_dask_copy(c1, c2, chunks, lock=False, dt=0.05)" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3 loops, best of 5: 649 ms per loop\n" - ] - } - ], - "source": [ - "# for comparison\n", - "%timeit -n3 -r5 c2[:] = c1" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3 loops, best of 5: 557 ms per loop\n" - ] - } - ], - "source": [ - "# for comparison\n", - "%timeit -n3 -r5 c1.copy()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/dask_count_alleles.ipynb b/notebooks/dask_count_alleles.ipynb deleted file mode 100644 index 8ca462b232..0000000000 --- a/notebooks/dask_count_alleles.ipynb +++ /dev/null @@ -1,648 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Profile allele count from genotype data via dask.array" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 1.0.1.dev18+dirty\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " Loading BokehJS ...\n", - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": [ - "\n", - "(function(global) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " if (typeof (window._bokeh_onload_callbacks) === \"undefined\") {\n", - " window._bokeh_onload_callbacks = [];\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", - " delete window._bokeh_onload_callbacks\n", - " console.info(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(js_urls, callback) {\n", - " window._bokeh_onload_callbacks.push(callback);\n", - " if (window._bokeh_is_loading > 0) {\n", - " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " }\n", - " if (js_urls == null || js_urls.length === 0) {\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " window._bokeh_is_loading = js_urls.length;\n", - " for (var i = 0; i < js_urls.length; i++) {\n", - " var url = js_urls[i];\n", - " var s = document.createElement('script');\n", - " s.src = url;\n", - " s.async = false;\n", - " s.onreadystatechange = s.onload = function() {\n", - " window._bokeh_is_loading--;\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", - " run_callbacks()\n", - " }\n", - " };\n", - " s.onerror = function() {\n", - " console.warn(\"failed to load library \" + url);\n", - " };\n", - " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", - " }\n", - " };\n", - "\n", - " var js_urls = ['https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.js', 'https://cdn.pydata.org/bokeh/release/bokeh-compiler-0.12.0.min.js'];\n", - "\n", - " var inline_js = [\n", - " function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - " \n", - " function(Bokeh) {\n", - " Bokeh.$(\"#b153ad5f-436a-4afb-945c-87790add89c8\").text(\"BokehJS successfully loaded\");\n", - " },\n", - " function(Bokeh) {\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.0.min.css\");\n", - " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.0.min.css\");\n", - " }\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " for (var i = 0; i < inline_js.length; i++) {\n", - " inline_js[i](window.Bokeh);\n", - " }\n", - " }\n", - "\n", - " if (window._bokeh_is_loading === 0) {\n", - " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", - " run_inline_js();\n", - " } else {\n", - " load_libs(js_urls, function() {\n", - " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", - " run_inline_js();\n", - " });\n", - " }\n", - "}(this));" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "print('zarr', zarr.__version__)\n", - "from zarr import blosc\n", - "import numpy as np\n", - "import h5py\n", - "import multiprocessing\n", - "import dask\n", - "import dask.array as da\n", - "from dask.diagnostics import Profiler, ResourceProfiler, CacheProfiler\n", - "from dask.diagnostics.profile_visualize import visualize\n", - "from cachey import nbytes\n", - "import bokeh\n", - "from bokeh.io import output_notebook\n", - "output_notebook()\n", - "from functools import reduce\n", - "import operator\n", - "import allel" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "callset = h5py.File('/data/coluzzi/ag1000g/data/phase1/release/AR3/variation/main/hdf5/ag1000g.phase1.ar3.pass.h5',\n", - " mode='r')\n", - "genotype = callset['3R/calldata/genotype']\n", - "genotype" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((13167162, 765, 2), int8, chunks=(6553, 200, 2), order=C)\n", - " compression: blosc; compression_opts: {'clevel': 1, 'cname': 'lz4', 'shuffle': 2}\n", - " nbytes: 18.8G; nbytes_stored: 683.2M; ratio: 28.1; initialized: 8040/8040\n", - " store: builtins.dict" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# copy into a zarr array\n", - "# N.B., chunks in HDF5 are too small really, use something bigger\n", - "chunks = (genotype.chunks[0], genotype.chunks[1] * 20, genotype.chunks[2])\n", - "genotype_zarr = zarr.array(genotype, chunks=chunks, compression='blosc',\n", - " compression_opts=dict(cname='lz4', clevel=1, shuffle=2))\n", - "genotype_zarr" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We want to perform an allele count. Compare serial and parallel implementations, and compare working direct from HDF5 versus from Zarr." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1min 50s, sys: 512 ms, total: 1min 51s\n", - "Wall time: 1min 50s\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))
nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;
compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);
data: bcolz.carray_ext.carray
0123
01523500
11527100
21527100
31527100
41527100
\n", - "

...

" - ], - "text/plain": [ - "AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))\n", - " nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;\n", - " compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);\n", - " data: bcolz.carray_ext.carray" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "# linear implementation from HDF5 on disk\n", - "allel.GenotypeChunkedArray(genotype).count_alleles()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 2min 27s, sys: 2.14 s, total: 2min 29s\n", - "Wall time: 1min 23s\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))
nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;
compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);
data: bcolz.carray_ext.carray
0123
01523500
11527100
21527100
31527100
41527100
\n", - "

...

" - ], - "text/plain": [ - "AlleleCountsChunkedArray((13167162, 4), int32, chunks=(65536, 4))\n", - " nbytes: 200.9M; cbytes: 38.3M; cratio: 5.2;\n", - " compression: blosc; compression_opts: cparams(clevel=5, shuffle=1, cname='lz4', quantize=0);\n", - " data: bcolz.carray_ext.carray" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%time\n", - "# linear implementation from zarr in memory\n", - "# (although blosc can use multiple threads internally)\n", - "allel.GenotypeChunkedArray(genotype_zarr).count_alleles()" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# multi-threaded implementation from HDF5 on disk\n", - "gd = allel.model.dask.GenotypeDaskArray.from_array(genotype, chunks=chunks)\n", - "ac = gd.count_alleles(max_allele=3)\n", - "with Profiler() as prof, ResourceProfiler(dt=1) as rprof:\n", - " ac.compute(num_workers=8)\n", - "visualize([prof, rprof], min_border_bottom=60, min_border_top=60);" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# multi-threaded implementation from zarr in memory\n", - "gdz = allel.model.dask.GenotypeDaskArray.from_array(genotype_zarr, chunks=chunks)\n", - "acz = gdz.count_alleles(max_allele=3)\n", - "with Profiler() as prof, ResourceProfiler(dt=1) as rprof:\n", - " acz.compute(num_workers=8)\n", - "visualize([prof, rprof], min_border_bottom=60, min_border_top=60);" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/genotype_benchmark_compressors.ipynb b/notebooks/genotype_benchmark_compressors.ipynb deleted file mode 100644 index b262e63fa0..0000000000 --- a/notebooks/genotype_benchmark_compressors.ipynb +++ /dev/null @@ -1,548 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "zarr 1.1.1.dev7+dirty\n", - "blosc ('1.10.0.dev', '$Date:: 2016-07-20 #$')\n" - ] - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import functools\n", - "import timeit\n", - "import zarr\n", - "print('zarr', zarr.__version__)\n", - "from zarr import blosc\n", - "print('blosc', blosc.version())\n", - "import numpy as np\n", - "import h5py\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "callset = h5py.File('/data/coluzzi/ag1000g/data/phase1/release/AR3/variation/main/hdf5/ag1000g.phase1.ar3.pass.h5',\n", - " mode='r')\n", - "genotype = callset['3R/calldata/genotype']\n", - "genotype" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "n_variants = 500000" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(500000, 765, 2)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "genotype_sample = genotype[1000000:1000000+n_variants, ...]\n", - "genotype_sample.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "765000000" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "nbytes = genotype_sample.nbytes\n", - "nbytes" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(685, 765, 2)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# 1M chunks of first dimension\n", - "chunks = (int(2**20 / (genotype_sample.shape[1] * genotype_sample.shape[2])), \n", - " genotype_sample.shape[1], \n", - " genotype_sample.shape[2])\n", - "chunks" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "8" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "blosc.get_nthreads()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((500000, 765, 2), int8, chunks=(685, 765, 2), order=C)\n", - " compression: blosc; compression_opts: {'cname': 'lz4', 'clevel': 1, 'shuffle': 2}\n", - " nbytes: 729.6M; nbytes_stored: 23.0M; ratio: 31.7; initialized: 730/730\n", - " store: builtins.dict" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zarr.array(genotype_sample, chunks=chunks, compression_opts=dict(cname='lz4', clevel=1, shuffle=2))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "zarr.core.Array((500000, 765, 2), int8, chunks=(685, 765, 2), order=C)\n", - " compression: blosc; compression_opts: {'cname': 'zstd', 'clevel': 1, 'shuffle': 2}\n", - " nbytes: 729.6M; nbytes_stored: 12.0M; ratio: 60.7; initialized: 730/730\n", - " store: builtins.dict" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "zarr.array(genotype_sample, chunks=chunks, compression_opts=dict(cname='zstd', clevel=1, shuffle=2))" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "compression_configs = (\n", - " (None, None),\n", - " ('zlib', 1),\n", - " ('bz2', 1),\n", - " ('lzma', dict(preset=1)),\n", - " ('blosc', dict(cname='snappy', clevel=0, shuffle=0)),\n", - " ('blosc', dict(cname='snappy', clevel=0, shuffle=2)),\n", - " ('blosc', dict(cname='snappy', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='snappy', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='blosclz', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='blosclz', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='lz4', clevel=9, shuffle=0)),\n", - " ('blosc', dict(cname='lz4', clevel=9, shuffle=2)),\n", - " ('blosc', dict(cname='lz4hc', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='lz4hc', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='lz4hc', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='lz4hc', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zstd', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='zstd', clevel=5, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=1, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=1, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=3, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=3, shuffle=2)),\n", - " ('blosc', dict(cname='zlib', clevel=5, shuffle=0)),\n", - " ('blosc', dict(cname='zlib', clevel=5, shuffle=2)),\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "def log(*msg):\n", - " print(*msg, file=sys.stdout)\n", - " sys.stdout.flush()" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "@functools.lru_cache(maxsize=None)\n", - "def compression_ratios():\n", - " x = list()\n", - " for compression, compression_opts in compression_configs:\n", - " z = zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", - " ratio = z.nbytes / z.nbytes_stored\n", - " x.append(ratio)\n", - " log(compression, compression_opts, ratio)\n", - " return x\n" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAMWCAYAAADszSe0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcFNW99/HPFwQJKLKYsCkDEjUuERk3FAKjqNEECdcg\ngrhcgxqveSIuSR6vG+CCmohPTKIm5iGokSTiDXpRQxSExmhERDYFxIVNUBTZrugjRub3/FGnsaan\nl+phZnpGfu/Xq19TXXWqzq9+XWKdPudUy8xwzjnnnHPOObfrmpQ6AOecc84555z7svAGlnPOOeec\nc87VEm9gOeecc84551wt8QaWc84555xzztUSb2A555xzzjnnXC3xBpZzzjnnnHPO1RJvYDnnnGv0\nJM2S9KtSx1EsSZWSzix1HM4552qP/HewnHOucZH0NeA64LvAfsAGYDHwGzObVsrYSkVSG+BfZvZx\nqWPJRtJEoL2ZDcpY/zVgs5n9qzSROeecq217lDoA55xzyUkqA/4JbAX+N1HDqglwMnAf0K1kweUg\nqVldNyDMbEtdHj+XXT03M/ugNuNxzjlXej5E0DnnGpf7gErgKDP7q5m9aWbLzewe4Ih0IUn7S3pM\n0v+E118ldYltHy3pVUnnS1opaZukCZKaSfqxpHckfSjpF/HKQ9nRkv4o6SNJ70m6OqNMpaTLQp3b\ngFvD+kMlPRnieV/SnyR1iO13uKQZkraGYy+Q1D9s20PSryStk/SppNWSxsX2rTJEUFIbSQ9K2iTp\nE0nTJR0a235BqOOkkIdtkmaGBmxO2c5NUhNJ/1fSilDXG5J+Gs81cAHw3bD/Dkn9Ysc7M1b28BDr\nJ5I2SpooqXW+mJxzzjUs3sByzrlGQlJb4NtEQwH/X+Z2M/ufUE7AVOCrQH+gAugMPJaxSzdgENFQ\nw38DhgJPAUcS9YiNBEZJ+l7GflcCS4BewI3AOEmDM8rcGI51OHCPpI7AbKIet6OBAUAr4L9j+/wJ\neDds7wmMAT4N20YB3wsxfh04G1hePUs7PQgcA5wR/n4C/F3SnrEyewLXAP8O9AbaAL/Nc8ys50b0\n/9K1wBDgG8C1wH9KujCUvxOYDMwAOgCdiHohq5DUEnga+J+Qg8HACcCEBDE555xrIHyIoHPONR5f\nBwS8XqDcyUQ3/weY2TsAks4B3pJ0kpnNDOWaAP9uZtuApZL+DvQDvmNmnwPLJb1A1BiKN4ReMrPb\nw/Jbko4FrgIej5X5i5n9If1G0lhgoZldG1v378BGSUeb2TygDPiFmb0ZiqyIHa8r8IaZvRDerwXm\nZDt5SQcSNay+lS4v6TxgDTACSMfVFLjMzN4KZe4kWWOmyrkFY2LLayQdBQwHJprZx5L+H9DSzDbk\nOe4IoCVwnpl9EmK6BJgl6QAzW5FnX+eccw2E92A551zjoYTlvgG8m25cAZjZSqLeoUNj5daExlXa\n+0SNmM8z1n0t4/gvZnl/aMa6VzLeHwX0D8PyPpL0EVGDx4AeocxdwARJz0q6VtLBsf0fAHqF4Xe/\nkfSd0FOXzTeAHcQaYKF379WMOLenG1fBu0Dz8MCMfDLPDUmXSnpZ0gfh3K4kahQW4xvA4nTjKvgn\n0ZDQzPw655xroLyB5ZxzjcebRA2SQ3bhGPFHx2Y+nMFyrKvJ/ysyn+bXBHiSaJ5Yz9jrwLAeMxtL\ndG6PEQ2NWxx6uTCzBUQ9XNcQNTQfBJ6pQVzx8/88x7ZC51vl3CSdDfwfop6xU4nO616geQ3iy8Uf\n+eucc42EN7Ccc66RMLPNRHN0/leYr1OFpH3C4jKgs6SusW0HEM3DWlILofTOeH98qDOf+cBhRL1m\nKzJeOxssZva2mf3GzAYSDde7KLbtYzObYmY/Ipo3NkDS17PUtYzo/2/Hp1eEB0V8k9o5/0x9gDlm\ndp+ZLQxD+TLj+oxoSGI+y4BvSmqVcWxROL/OOecaCG9gOedc4/IjohvueZKGSDpI0sGS/gNYBGBm\nM4iGw02SdJSko4GHgXlmlqqFGHpL+t+Svi7pYuBcouF9+dwD7ANMlnSspO6STpb0O0mtJLUIQ//6\nSyqTdBzQl9AgknSlpGGSvhEaVSOIHlW/NrOiMOxvKvA7SX0lfTOc/1bgzwXiTDoMM+4NoFzSaSEn\nNxDNZYtbBRwePq/2krI1tiYRPYzjofA0wX5ED934q8+/cs65xsMbWM4514iEuVTlwHTgdqJG1bNE\nT9i7IlZ0ENEPEM8M298lelJgbbiLaKjfAuAm4AYziz+hsNpwNjN7j6g3ZgcwDXgN+DXRUwK3h/Vt\ngYlED/H4K/ACkH4E/EfAT4GXgHmh/tPNLP2Uwcw6/x2YS/RwjjlETww8zcy2Fzi3QkPxsm3/HdFT\nAieFOrsSPTkw7vdEvVDzgA+IclHleOHJkN8GWhOd52NEORhZICbnnHMNiMx8WLdzzrlkJK0Efm1m\nhXqsnHPOud2S92A555xzzjnnXC3xBpZzzrli+LAH55xzLg8fIuicc84555xztcR7sJxzzjnnnHOu\nlngDyznnnHPOOedqiTewnHPOOeecc66W7FHqAJxzDYskn5jpnHPOOZeAmVX7gXrvwXLOVWNm/irw\nuuCCC0oeQ2N4eZ48T54nz1NDfXmePE+7+srFG1jOOVcD3bp1K3UIjYLnKRnPUzKep2Q8T8l4npLx\nPBXPG1jOOeecc845V0v8d7Ccc1X4HCznnHP5dOhQxvr1q0odxi775S9/yRVXXFHqMBo8z1NukrAs\nc7D8IRfOuSy8jVVYCqgocQyNQQrPUxIpPE9JpPA8JZGiLvP0/vvV7icbpSOPPLLUITQKnqfieQ+W\nc66KqAfL/11wzjmXi/JO8HfF69atG6tXry51GC6HsrIyVq1aVW19rh6sepuDJalM0qs5ts2SVF5f\nsWTU3VXSfEnTYutWliKWXCT1lzQxQbm8cUv6KPztJGlyWL5A0q9rcryEdY6WdFWh4xQjfkxJEyX1\nK1C+v6Qt4XOeL+n6BHXMktS1wPairllJQyQtlfRseP9nSQsljQrncWaB/ZOc6zmSFoXX85KOiG0b\nL2mJpP7FxO2cc865urV69eqSPxHPX7lfxTZ+6/shFw3x647BwDNmdnpsXUOMM0lMhcoYgJm9Z2ZD\nE+xXG3U2FM+ZWXl43VKiGEYCF5nZAEkdgaPN7Egzu7sW61gB9DOznsAtwP3pDWZ2NXAT8INarG83\nlip1AI1EqtQBNBKpUgfQSKRKHUAjkSp1AI1CKpUqdQjuS6q+G1jNJD0cvsWfLKlFZgFJwyUtDq/b\nw7om4dv7xeGb+VFhfQ9J00MvwDxJ3WsQUxvgg4x1G2LxnB/qXCDpwbBuoqS7Jb0g6a10z4OkVpJm\nhFgWSRoU1pdJWhb2Wy5pkqRTwv7LJR0dyrWUNEHSHEmvSDojhPEZsDXBuWwIxxkb4p0vaa2kCenT\nicUT703sGnpklku6MVseCtWZK1dxkg6QNE3Sy5JmSzpIUmtJq2JlWkpaI6lptvJZ6t9ClJ9Cih0w\nvhHYkevaC4ZKeknS65L6hPir9AhKekJSP0k3AH2BCZJ+DjwNdAmfUd8qgUrlklLhvKdJ6pD0XM1s\njpmlr5U5QJeMIuuJrnnnnHPOOVcH6vshFwcDF5rZnHDTfxlwV3qjpE7A7UAvopvJ6aGRshboYmZH\nhHKtwy6TgHFmNlVSc2rWYGwKVMZXmNlxoZ5DgWuB481ss6T4jWlHM+sj6RBgKjAF+BQYbGbbJLUn\nusGdGsr3AL5vZkslzQOGhf0HhTrOBK4DnjWzkZL2AeZKmmFmLwIvhpiOAn5oZpdknkg6bjMbDYwO\nx3gOSN/wx3ub4svHAIeF+F+W9KSZzU8fL5+EuUq7P8T+tqRjgftCb84CSf3NbDYwEPi7me2QVK08\nMCCj/ivTy5LGAi+b2ZNZ6j5e0kJgHfBTM1ta4LyGhGOWk/3aA2hqZsdJOh0YA5yS3j3L8W6WdBJw\nlZktkHQP8ISZlYfjjgx/9yD6vAaZ2UZJQ4FxwMgizjXtImBaxrpKomu+gDGx5Qp8Unk2FaUOoJGo\nKHUAjURFqQNoJCpKHUAjUVHqABqFioqKUofgGplUKpWo57O+G1hrzGxOWH4Y+DGxBhbRjf4sM9sE\nIGkS0I9oqFN3SXcDfwOekbQX0NnMpgKYWZJejCokCegZYsnmJOBRM9sc6tgS2/Z4WLdM0tfShwRu\nUzRPphLoHNu2MnZTvwSYEZZfBbqF5VOBMyT9NLxvDnQFlqcrNbNXgGqNqxweBu4ys4UFyk1Pn5uk\nKUQ9LfMT1pGWL1dIagWcADwa8g7QLPydDJwNzAaGAfcUKJ9VaFhm8wrQ1cw+CY2hx4FsvWHZrCDj\n2ottmxI7flnC4xXqSTsYOJzoywURfWnwbmahPOcaVSKdCFxI9FnGrQMOkrSnmW3PfYQxBcJ0zjnn\nnNu9VFRUVGmYjx07Nmu5Us/ByjZ/p9oNaLhZ70k0qPhS4Pe5ylY5kHRZbKhcx4xtTYCVwCHAU4mi\nryp+c5qOYwSwL9DLzHoRDT1skaV8Zex9JV80dEXUy9UrvLqb2XJqQNIYogZttaF6WST5XHZVE2Bz\nmAOVPr/Dw7apwGmS2gLlwMwC5YtiZtvM7JOwPI1oqGq7hPvmuvbgi89wB198hp9T9b+rasNgCxDw\nWuy8e2bMDyx8gOjBFvcT9YJtjm8zsxXAMmC1pMOKjM1VkSp1AI1EqtQBNBKpUgfQSKRKHUAjkSp1\nAI1CQ5+D1bFjNyTV2atjx26J4ujevTszZ87Muu3555/nkEMOqZXzzVdPIZ9++ilnnHEGbdq04eyz\nzwbg+uuv56tf/SqdO3dm9erVNGnShMrKygJHqh313cAqk5QednYO8I+M7XOBfpLaSWoKDAdmh+F2\nTc3sMeB6oNzMtgHvSPoegKTmkr4SP5iZ3RtuUsvNbH3Gtkoz6wbMI+o9yWYmcFb6Zjw0ALJJN7D2\nAT4ws8rQg1CWpUw+TwOX79xBqtEPDyiau3UyMCpzU45dTpHUJuRvMPBClmMuK1Bt3lyZ2UfASklD\nYsc8Imz7mOhzuBt40iI5yxcrNoeJMNRQsV7SGWFoaq59q117uYqGv6uAIxXZHzg2X2hZ1i0Hviqp\nd6h/jzD8MhFFTz78K3Cemb2dZfsRQHei3t8lSY/rnHPOufr1/vurib7zrptXdPxd07dvX5Yt++IW\ncVcaSbviv/7rv9iwYQObN2/mkUce4Z133uGuu+7i9ddf5913o4FAXwyIqnv13cB6HfiRpKVEE+1/\nG9ann263HriG6KuXBURzTJ4gmqifkrQA+GMoA3A+cLmkRUSNgp030kV4A8jamxGG9N1K1MhbAIyP\nxxsvGv5OAo4J8ZxL1FOQWSbb/mk3E/WuLFb0EIqbMgtIOirMTcrnSqAz0Xyq+aE3K1+9c4mGuy0k\nGuZXZXhgaGTklSdXcecCIxU9lOQ1YFBs2yNEPYB/ia0bkad8NYoe7jEwy6Yhkl4Lcf2SaBhieoho\nD2BTnsPmuvayXgNm9gJRI2tJqOuVzDI53qf3/xcwBLhD0ZyxBcDxRZzrDUTX872h93Zuxva2wCoz\nq5+vcL7UKkodQCNRUeoAGomKUgfQSFSUOoBGoqLUATQKPgfry2P16tUcdNBBOxtRq1evZt9996V9\n+4K3sHUj3zPfd4cX8FPg9lLH0ZBfwHeB/1XqOOrgvA4D7ix1HPV8zkOBPxcoY2D+8pe//OUvf+V4\nYa52Zctp3f//ONnn2K1bN7vtttvs0EMPtXbt2tkPfvAD2759u5mZpVIp22+//czM7LzzzrMmTZpY\ny5Ytbe+997Zf/OIX1Y714Ycf2sCBA61NmzbWrl0769evX5V67rzzTjviiCOsTZs2NmzYsJ31PPDA\nA9a3b98qx5Jkb7/9to0ePdqaN29uzZo1s7333tt+97vf2Ve+8hVr2rSp7b333nbhhRfaqlWrrEmT\nJrZjxw4zM9u6dauNHDnSOnXqZPvtt59df/31VllZWdTnE1tP5qu+H3LREE0BHpA0zYqc67K7MLOa\nzFFr8CwaIveTUsdRXySNB74F/GeC0nUdjnPOuUaqQ4eyUodQK1KplPdiJfSnP/2J6dOn07JlSwYO\nHMgtt9zCTTdFA63SvUYPPfQQ//jHP/jDH/7AiSeemPU448ePZ//992fjxo2YGXPmzKmy/dFHH+WZ\nZ55hzz335IQTTuCBBx7gkksuqVJPWvr9mDFjkMTbb7/NQw89BMDBBx/Meeedx5o1awCq/VDwBRdc\nQKdOnVixYgXbtm1j4MCBdO3alYsvvnhX0rTTbt/AsmieyrdKHYdzdc2iHxpOWrYuQ/lS8P8xJ+N5\nSsbzlIznKRnPk6ttP/7xj+ncuTMA1113HZdffvnOBlamfPcQzZo147333mPlypX06NGDPn36VNk+\natQoOnSIZvycccYZLFyY+0HYNb1Xef/995k2bRpbt25lzz33pEWLFlxxxRXcf//9tdbAqu85WM45\n96XgNy/JeJ6S8Twl43lKxvOUjOcpuf3222/ncllZ2c4HRxTrZz/7GT169ODUU0/l61//OnfccUeV\n7enGFUDLli3Ztm1bzQLOY82aNfzrX/+iU6dOtGvXjrZt23LppZfy4Ycf1lodu30PlnPOOeeccy63\nd955Z+fy6tWrd/ZmZSr0pL5WrVpx5513cuedd7J06VJOPPFEjj322JxDCuP7ffLJJzvfr1+/vsZP\nBdx///1p0aIFGzdurLMnC3oPlnPO1UBD//2UhsLzlIznKRnPUzKep2Q8T8ndc889rFu3jk2bNjFu\n3DiGDRuWtVzHjh1ZsWJFzuM89dRTvP129Csye++9N3vssQdNmzYtWH/Pnj1ZsmQJixcvZvv27Tl/\n4Def9JDCjh07cuqpp3LllVfy0UcfYWasWLGC5557ruhj5uINLOecc8455xqY6GEiqrNX0oeVSOKc\nc87ZOazvwAMP5Lrrrsta9pprruHmm2+mXbt23HXXXdW2v/nmm5x88snsvffe9OnThx/96Ef069dv\nZz25HHjggdx4440MGDCAgw46iG99q/jHJ8SP/9BDD/HZZ59x6KGH0q5dO8466yzWr1+fZ+8i6/LJ\n7M65OEnm/y4455xz9UeSP2CqAcv1+YT11VqG3oPlnHPOOeecc7XEH3LhnKumriZ9Oufc7q5Dlw6s\nX1t7Q5Gy8ce0J+N5cnXFG1jOuerGlDqARmAl0L3UQTQCnqdkPE/JfAny9P6Y90sdgnOujvkcLOdc\nFZLMG1jOOVdHxviPubvqfA5Ww9Zg52BJKpP0ao5tsySV11csGXV3lTRf0rTYupWliCUXSf0lTUxQ\nLm/ckj4KfztJmhyWL5D065ocL2GdoyVdVeg4xYgfU9JESf0KlO8vaUv4nOdLuj5BHbMkdS2wvahr\nVtIQSUslPRve/1nSQkmjwnmcWWD/gucayv1K0pvh2EfG1o+XtERS/2Lids4555xzydX3Qy4aYtN8\nMPCMmZ0eW9cQ40wSU6EyBmBm75nZ0AT71UadDcVzZlYeXreUKIaRwEVmNkBSR+BoMzvSzO6urQok\nnQ70MLMDgR8Cv01vM7OrgZuAH9RWfbu1BvU1TAPmeUrG85SM5ykR/32nZDxPrq7UdwOrmaSHw7f4\nkyW1yCwgabikxeF1e1jXJHx7v1jSIkmjwvoekqaHb+rnSarJyOw2wAcZ6zbE4jk/1LlA0oNh3URJ\nd0t6QdJb6Z4HSa0kzQixLJI0KKwvk7Qs7Ldc0iRJp4T9l0s6OpRrKWmCpDmSXpF0RgjjM2BrgnPZ\nEI4zNsQ7X9JaSRPSpxOLJ96b2DX0yCyXdGO2PBSqM1eu4iQdIGmapJclzZZ0kKTWklbFyrSUtEZS\n02zls9S/hSg/hRT71IaNwI5c114wVNJLkl6X1CfEX6VHUNITkvpJugHoC0yQ9HPgaaBL+Iz6VglU\nKpeUCuc9TVKHIs71e8BDAGb2ErBPbH+A9UTXvHPOOeecqwP1/ZCLg4ELzWxOuOm/DNj5K2SSOgG3\nA72Ibianh0bKWqCLmR0RyrUOu0wCxpnZVEnNqVmDsSlQGV9hZseFeg4FrgWON7PNkuI3ph3NrI+k\nQ4CpwBTgU2CwmW2T1B6YE7YB9AC+b2ZLJc0DhoX9B4U6zgSuA541s5GS9gHmSpphZi8CL4aYjgJ+\naGaXZJ5IOm4zGw2MDsd4Dkjf8Md7m+LLxwCHhfhflvSkmc1PHy+fhLlKuz/E/rakY4H7Qm/OAkn9\nzWw2MBD4u5ntkFStPDAgo/4r08uSxgIvm9mTWeo+XtJCYB3wUzNbWuC8hoRjlpP92gNoambHhV6j\nMcAp6d2zHO9mSScBV5nZAkn3AE+YWXk47sjwdw+iz2uQmW2UNBQYB4xMeK5dgHdi79eFdelZ1ZVE\n13x+s2LL3Wj0k8rrhOckGc9TMp6nZDxPifiT8ZLxPLlipVKpRD2f9d2DtcbM5oTlh4m+0Y87Bphl\nZpvMrJKoAdUPWAF0D71G3wY+krQX0NnMpgKY2Wdm9mkxwUgS0JOoAZfNScCjZrY51LEltu3xsG4Z\n8LX0IYHbJC0CZgCdJaW3rYzd1C8J2wFeJbqFBTgVuEbSAiAFNAeqzAMys1eyNa5yeBi4y8wWFig3\n3cy2hPxNofrnkkS+XCGpFXAC8Gg4v98B6Z6VycDZYXkY8EiB8lmZ2egcjatXgK5mdiTwG8Jnl1C1\nay+2bUrs+Ml+Dr1wT9rBwOFEXy4sIGp0d84slOdcC1kHHCRpz7ylToy9/IbGOeecq3cd9+uIpDp7\nddyvY6lPcacTTzyRP/zhDzXe/8ILL6Rdu3b07t0bgPvuu4+OHTvSunVrNm3aRJMmTVixYsUux1lR\nUcGYMWN2vnKp7x6szG/2s83fqXYDamZbJPUEvg1cCpwFXJGtbJUDSZcBF4d6vmNm62PbmhDdPG8H\nniriHNK2Z4l5BLAv0MvMKhU9AKJFlvKVsfeVfPE5iKiX680axFOFpDFEDdpqQ/WySPK57KomwOZ0\nj02GqcCtktoC5cBMYK885YtiZttiy9Mk3SupnZltSrBvtmvvorA5/Rnu4IvP8HOqfnFRbRhsAQJe\nM7M+Re6Xtg7YP/Z+v7AOADNbIWkZsFrSADNbUsN63JfgcdH1wvOUjOcpGc9TIv77Tsk09Dy9v+79\nOv3ZlC/LTwY8//zzPPvss7z77ru0aNGCzz//nKuvvpq5c+dy+OGHA/X/+5713YNVJik97Owc4B8Z\n2+cC/SS1k9QUGA7MDsPtmprZY8D1QHm4aX5H0vcAJDWX9JX4wczsXjPrFR5ssD5jW6WZdQPm8UXv\nSaaZwFmS2oU62uYol/7U9gE+CI2rE6naq5Hkk30auHznDrEnwBVD0dytk4FRmZty7HKKpDYhf4OB\nF7Icc1mBavPmysw+AlZKGhI75hFh28dEn8PdwJMWyVm+WPE5SGGoodKNK0Vz5jrl2bfatZeraPi7\nCjhSkf2BY/OFlmXdcuCrknqH+vcIwy+TmgqcH/btDWwxs53/goYcdifq/fXGlXPOOecatVWrVtGt\nWzdatIi+016/fj3bt2/nkEMO2Vmmvh+BX98NrNeBH0laSjTRPv2Es/TT7dYD1xANj1tANMfkCaI5\nJKkwZOqPoQxEN5KXhyF5L1BgCFkObwDtsm0IQ/puJWrkLQDGx+ONFw1/JwHHhHjOBZZlKZNt/7Sb\niR4EsljRQyhuyiwg6agwNymfK4mGlb2s6CEKYwrUO5douNtComF+8zPqbF+gvny5ijsXGKnooSSv\nAYNi2x4h6gH8S2zdiDzlq1H0cI+BWTYNkfRaiOuXRMMQ00NEewD5erJyXXtZrwEze4GokbUk1PVK\nZpkc79P7/wsYAtyhaM7YAuD4pOdqZn8japi+RTSs8rKMIm2BVWEIrtsV/i16Mp6nZDxPyXieEmnI\nvTINiecpme7duzN+/Hh69uxJ27ZtGT58OJ999sUzt37/+99z4IEHsu+++zJ48GDee++9rMfZvn07\n5513Hvvuuy9t27bluOOOY8OGL56ntmrVKvr27Uvr1q057bTT2LQpuj2bPXs2+++/f5Vjde/enZkz\nZ/KHP/yBiy++mBdffJHWrVszYsQIvvGNbwDQtm1bTj755GpxfPbZZ/zkJz+hrKyMTp06cdlll7F9\n+/Zq5XbFbv9Dw5J+CrQ3s2sKFt5NSfou0N3MflPqWGqTpMOIHrryk1LHUl8UPTTj38xseJ4y/kPD\nzjlXV8b4Dw276pTlh2wl1ekQwaTXYvfu3enQoQP//d//zZ577skJJ5zAFVdcwSWXXMLMmTM5++yz\nmTFjBoceeihXX301ixYtYvbs2dWOc//99/PUU08xefJkmjdvzsKFCznwwAPZa6+9OPHEE1m7di1/\n//vf2W+//TjttNM4/vjjGTduHLNnz+a8885jzZo1VWKaMGECJ510Eg8++CATJkzgueeeA2D16tUc\ncMABfP755zuHBjZp0oS33nqLAw44gCuvvJKVK1fy4IMPsscee3DOOedw+OGHc+utt+bMQbbPJ7a+\n2oik+p6D1RBNAR6QNC3jt7BcYGY1maPW4IUhcrtT42o88C3gPwsWHlPX0Tjn3O6pQ5eaDLYpTkOf\nW9RQeJ6SGzVqFB06RNfuGWecwcKF0fPT/vSnPzFy5Eh69uwJwG233Ubbtm1Zs2YNXbtWeU4bzZo1\nY+PGjbzxxht885vfpFevXlW2X3jhhfTo0QOAoUOH8sQTT+xSzGaWde7V73//e1599VX22WcfAK65\n5hpGjBiRt4FVrN2+gWVmbxPddDr3pRZ+aDhp2boM5UvB/8ecjOcpGc9TMp4n50oj3bgCaNmy5c5h\ngO+++y5HHXXUzm2tWrWiffv2rFu3rloD6/zzz2ft2rUMGzaMrVu3MmLECMaNG0fTptGvx3Ts2LFK\nHdu2baO2bdiwgU8++aRKzJWVlbV+31Pfc7Ccc+5LwW/ykvE8JeN5SsbzlIznKRnP067r3Lkzq1ev\n3vn+448/ZuPGjXTp0qVa2aZNm3LDDTewZMkS/vnPf/Lkk0/y0EMPFayjVatWfPLJJzvf79ixo8rc\nrWLsu+87K3VnAAAgAElEQVS+tGzZkiVLlrBp0yY2bdrEli1b2Lp1a42Ol4s3sJxzzjnnnHNFGz58\nOBMnTmTx4sVs376da6+9lt69e1frvYKoB/q1116jsrKSvfbai2bNmu3svcrnoIMO4tNPP2XatGl8\n/vnn3HLLLVUespFNrh4pSVx88cVcccUVOxtp69at45lnnklwtsnt9kMEnXOuJnyoUjKep2Q8T8l4\nnpLxPCXT0PPUoUuHOv2tqqTzAfP9htSAAQO4+eabOfPMM9myZQsnnHACf/nLX7KWXb9+PZdeeinr\n1q1jr732YtiwYZx77rkF62jdujX33nsvI0eOpLKykp/97Gfst99+RcUcf3/HHXcwduxYevfuvbO3\n7T/+4z849dRT8x6zGLv9UwSdc1VJMv93obCG/j/mhsLzlIznKRnPUzKep2QaUp5yPaXONQzFPkXQ\nG1jOuSq8geWcc87VL29gNWzFNrB8DpZzzjnnnHPO1RKfg+WcqybfWGjnnGvsOnQoY/36VaUOo840\npKFvDZnnydUVb2A557LwYQqFpYCKEsfQGKTwPCWRwvOURIrayNP77/uXSM65uuNzsJxzVUgyb2A5\n577cfL6La1h8DlbD1mDnYEkqk/Rqjm2zJJXXVywZdXeVNF/StNi6laWIJRdJ/SVNTFAub9ySPgp/\nO0maHJYvkPTrmhwvYZ2jJV1V6DjFiB9T0kRJ/QqUHyRpkaQFkuZJOilBHbMkVf8Rh6rbi7pmJQ2R\ntFTSs+H9nyUtlDQqnMeZBfZPcq7nhHNdJOl5SUfEto2XtERS/2Lids4551zdKisrQ5K/GuirrKys\nqM+zvh9y0RCb5oOBZ8zs9Ni6hhhnkpgKlTEAM3vPzIYm2K826mwIZphZTzPrBVwI3F+iOEYCF5nZ\nAEkdgaPN7Egzu7sW61gB9DOznsAtxM7VzK4GbgJ+UIv17cZSpQ6gkUiVOoBGIlXqABqJVKkDaBRS\nqVSpQ2gUGlKeVq1ahZk1yNesWbNKHkOpX6tWrSrq86zvBlYzSQ+Hb/EnS2qRWUDScEmLw+v2sK5J\n+PZ+cfhmflRY30PSdEW9APMkda9BTG2ADzLWbYjFc76+6P14MKybKOluSS9Iekuh50FSK0kzQiyL\nJA0K68skLQv7LZc0SdIpYf/lko4O5VpKmiBpjqRXJJ0RwvgM2JrgXDaE44wN8c6XtFbShPTpxOKJ\n9yZ2VdQjs1zSjdnyUKjOXLmKk3SApGmSXpY0W9JBklpLWhUr01LSGklNs5XPUv8WovzkZGafxN7u\nBXyY4Lw2AjtyXXvBUEkvSXpdUp8Qf5UeQUlPSOon6QagLzBB0s+Bp4Eu4TPqm5GnckmpcN7TJKV/\nCTDJuc4xs/S1MgfoklFkPdE175xzzjnn6kJ9tfyAMqAS6B3eTwCuCsuzgHKgE7AaaEfU+HsWGBS2\nPRM7Vuvwdw4wKCw3B1rUIK6xwBU5th0KvA60De/bhL8TgUfC8iHAm2G5KbBXWG4fW19GdGN8aHg/\nD5gQlgcBU8LyrcA5YXkfYDnwlYyYjgLuT3hu+wCLgCPD+/+JxbM4LF8ArCO66W4BvAqU1yCPuXI1\nOvY5zwB6hOVjgWfD8mNA/7A8NH1+ecrvPGaWz3JgjvgGA8uAzcCxRZxXrmtvFvCLsHw6MD2Wz1/F\nyj9B1KOU3qdX5mcQu6bOJHrwzAtA+1g+JhRzrrEyP8m8VoBvAU8W2M9gdOw1y8D85S9/+etL9MKc\nc65Ys2bNstGjR+98hX9LyHzV91ME15jZnLD8MPBj4K7Y9mOAWWa2CUDSJKAf0VCn7pLuBv4GPCNp\nL6CzmU0FMLO83+xnI0lAzxBLNicBj5rZ5lDHlti2x8O6ZZK+lj4kcJuieTKVQOfYtpVmtjQsLyFq\nPEDUoOkWlk8FzpD00/C+OdCVqKFFqO8V4JKEp/gwcJeZLSxQbnr63CRNIeppmZ+wjrR8uUJSK+AE\n4NGQd4Bm4e9k4GxgNjAMuKdA+azMbHSebY8Dj4feoj8CByc8rxVkXHuxbVPC31eIGkxJFHp01cHA\n4cD0cN5NgHczC+U7VwBJJxINh+ybsWkdcJCkPc1se+4jjCkQpnPOOefc7qWioqLKo/3Hjh2btVyp\n52BlvocsN6DhZr0n0eDrS4Hf5ypb5UDSZbGhch0ztjUBVhL1QD2VKPqq4jen6ThGAPsS9VL0Ihp6\n2CJL+crY+0q+eFy+gO+bWa/w6m5my6kBSWOIGrTVhuplkeRz2VVNgM1mVh47v8PDtqnAaZLaEvUY\nzSxQvsbM7HlgD0ntE5bPde3BF5/hDr74DD+n6n9X1YbBFiDgtdh597Sq8wMLHyB6sMX9RL27m+Pb\nzGwFUU/eakmHFRmbqyJV6gAaiVSpA2gkUqUOoJFIlTqARqEhzS1qyDxPyXieilffDawySceF5XOA\nf2Rsnwv0k9ROUlNgODA73Aw3NbPHgOuJhrBtA96R9D0ASc0lfSV+MDO7N9yklpvZ+oxtlWbWjWi4\n3tk54p0JnCWpXaijbY5y6QbWPsAHZlYZehDKspTJ52ng8p07SEcm2Kd6MNHcrZOBUZmbcuxyiqQ2\nIX+DiYaoZR5zWYFq8+bKzD4CVkoaEjvmEWHbx0Sfw91Ew9csX/liSeoRWy4PdW4M72dI6pRn32rX\nXq6i4e8q4EhF9ica2pjz8FnWLQe+Kql3qH8PSYfmOUZmvF2BvwLnmdnbWbYfAXQn6v1dkvS4zjnn\nnHMumfpuYL0O/EjSUqI5P78N6w0gNIKuIfqKagHwspk9QTRRPyVpAdHwrmvCfucDl0taRNQoSD8M\noBhvEM35qiYM6buVqJG3ABgfjzdeNPydBBwT4jmXqKcgs0y2/dNuJnoQyOLwEIqbMgtIOkpSoafg\nXQl0Bl4OvXdjCtQ7l2i420KiYX5Vhgcm6e3Jk6u4c4GRih5K8hrR/LO0R4h6AP8SWzciT/lqFD3c\nY2CWTd+X9Jqk+USNuGGhvIAewKY8h8117WW9BszsBaJG1hLgl0TDB8m3T8b+/wKGAHdIWkj038Hx\nRZzrDUTX872h93Zuxva2wCozq8yyrytKRakDaCQqSh1AI1FR6gAaiYpSB9AoxIcwudw8T8l4noq3\n2//QcJjv1N7MrilYeDcl6btAdzP7TaljqU1hiNyFZvaTUsdSXyQNBf7NzIbnKWN1M0rUOecaCv9R\nV+fcrlOpf2i4AZsC9FHsh4ZdVWb21JetcQVgZkt2s8bVeKInC/7fUsfy5ZAqdQCNRKrUATQSqVIH\n0EikSh1Ao+BzZpLxPCXjeSpefT9FsMEJ81S+Veo4nKtrFv3QcEJJpgw651zj1KFD0ge/Oudc8Xb7\nIYLOuaokmf+74JxzzjmXnw8RdM4555xzzrk65g0s55yrAR+TnoznKRnPUzKep2Q8T8l4npLxPBXP\nG1jOOeecc845V0t8DpZzrgqfg+Wcc845V5jPwXLOOeecc865OuYNLOecqwEfk56M5ykZz1Mynqdk\nPE/JeJ6S8TwVb7f/HSznXHWS/w6Wc7u7Dl06sH7t+lKH4ZxzjY7PwXLOVSHJGFPqKJxzJTcG/B7B\nOedy8zlYzjnnnHPOOVfH6q2BJalM0qs5ts2SVF5fsWTU3VXSfEnTYutWliKWXCT1lzQxQbm8cUv6\nKPztJGlyWL5A0q9rcryEdY6WdFWh4xQjfkxJEyX1K1B+kKRFkhZImifppAR1zJLUtcD2oq5ZSUMk\nLZX0bHj/Z0kLJY0K53Fmgf0Lnmso9ytJb4ZjHxlbP17SEkn9i4nb5dCg/pVowDxPyXieEvG5IMl4\nnpLxPCXjeSpefc/BaohjDQYDz5jZNbF1DTHOJDEVKmMAZvYeMDTBfrVRZ0Mww8ymAkj6JvAY8PUS\nxDESuMjM/impI3C0mR0Y4irYgE5C0ulADzM7UNJxwG+B3gBmdrWkucAPgNm1UZ9zzjnnnKuqvocI\nNpP0cPgWf7KkFpkFJA2XtDi8bg/rmoRv7xeHnohRYX0PSdPDN/XzJHWvQUxtgA8y1m2IxXN+rPfj\nwbBuoqS7Jb0g6a10z4OkVpJmhFgWSRoU1pdJWhb2Wy5pkqRTwv7LJR0dyrWUNEHSHEmvSDojhPEZ\nsDXBuWwIxxkb4p0vaa2kCenTicUT703sGnpklku6MVseCtWZK1dxkg6QNE3Sy5JmSzpIUmtJq2Jl\nWkpaI6lptvJZ6t9ClJ+czOyT2Nu9gA8TnNdGYEeuay8YKuklSa9L6hPir9IjKOkJSf0k3QD0BSZI\n+jnwNNAlfEZ9M/JULikVznuapA5JzxX4HvBQOO+XgH1i+wOsJ7rm3a6qyb82uyPPUzKep0QqKipK\nHUKj4HlKxvOUjOepePXdg3UwcKGZzQk3/ZcBd6U3SuoE3A70IrqZnB4aKWuBLmZ2RCjXOuwyCRhn\nZlMlNadmDcamQGV8hZkdF+o5FLgWON7MNkuK35h2NLM+kg4BpgJTgE+BwWa2TVJ7YE7YBtAD+L6Z\nLZU0DxgW9h8U6jgTuA541sxGStoHmCtphpm9CLwYYjoK+KGZXZJ5Ium4zWw0MDoc4zkgfcMf722K\nLx8DHBbif1nSk2Y2P328fBLmKu3+EPvbko4F7jOzAaFB1t/MZgMDgb+b2Q5J1coDAzLqvzK9LGks\n8LKZPZlZsaTBwG1AR+DbCc5rSNivnOzXHkBTMztOUa/RGOCU9O5ZjnezoqGJV5nZAkn3AE+YWXk4\n7sjwdw+iz2uQmW2UNBQYB4xMeK5dgHdi79eFde+H95VE13x+s2LL3fCbP+ecc87t9lKpVKIhk/Xd\ng7XGzOaE5YeJvtGPOwaYZWabzKySqAHVD1gBdA+9Rt8GPpK0F9A5PfTLzD4zs0+LCUaSgJ5EDbhs\nTgIeNbPNoY4tsW2Ph3XLgK+lDwncJmkRMAPoLCm9baWZLQ3LS8J2gFeJbmEBTgWukbQASAHNgSrz\ngMzslWyNqxweBu4ys4UFyk03sy0hf1Oo/rkkkS9XSGoFnAA8Gs7vd0C6Z2UycHZYHgY8UqB8VmY2\nOlvjKmx73MwOAc4A/ljEeVW79mLbpoS/rwBlCY9X6PnnBwOHE325sICo0d05s1C+cy1gHXCQpD3z\nljox9vLGVXY+ZyYZz1MynqdEfC5IMp6nZDxPyXievlBRUcGYMWN2vnIp9RysbPN3qt2AmtkWST2J\neh4uBc4CrshWtsqBpMuAi0M93zGz9bFtTYhunrcDTxVxDmnbs8Q8AtgX6GVmlYoeANEiS/nK2PtK\nvvgcRNTL9WYN4qlC0hiiBm21oXpZJPlcdlUTYHO6xybDVOBWSW2BcmAm0VC+XOVrzMyel7SHpPZm\ntjFB+WzX3kVhc/oz3MEXn+HnVP3iotow2AIEvGZmfYrcL20dsH/s/X5hHQBmtkLSMmC1pAFmtqSG\n9TjnnHPOuSzquwerTNHEe4BzgH9kbJ8L9JPUTlJTYDgwOwy3a2pmjwHXA+Vmtg14R9L3ACQ1l/SV\n+MHM7F4z62Vm5fHGVdhWaWbdgHl80XuSaSZwlqR2oY62OcqlG1j7AB+ExtWJVO3VSPLLrU8Dl+/c\nIfYEuGIomrt1MjAqc1OOXU6R1CbkbzDwQpZjLitQbd5cmdlHwEpJQ2LHPCJs+5joc7gbeNIiOcsX\nS1KP2HJ5qHNjeD8jDE3NtW+1ay9X0fB3FXCkIvsDx+YLLcu65cBXJfUO9e8Rhl8mNRU4P+zbG9hi\nZunhgekcdifq/fXG1a7wnr1kPE/JeJ4S8bkgyXiekvE8JeN5Kl59N7BeB34kaSnRRPvfhvXpp9ut\nB64hGh63gGiOyRNEc0hSYcjUH0MZiG4kLw9D8l6gwBCyHN4A2mXbEIb03UrUyFsAjI/HGy8a/k4C\njgnxnAssy1Im2/5pNxM9CGSxoodQ3JRZQNJRYW5SPlcSDSt7OTxEYUyBeucSDXdbSDTMb35Gne0L\n1JcvV3HnAiMVPZTkNWBQbNsjRD2Af4mtG5GnfDWKHu4xMMum70t6TdJ8okbcsFBeRHPjNuU5bK5r\nL+s1YGYvEDWylgC/JBo+SL59Mvb/FzAEuEPSQqL/Do5Peq5m9jeihulbRMMqL8so0hZYFYbgOuec\nc865Wqbd/VfaJf0UaJ/xmHYXI+m7QHcz+02pY6lNkg4jeujKT0odS30JD834NzMbnqeMMab+Ymq0\nVuK9Dkl4npJpiHkaAw3tHiGVSvm36Ql4npLxPCXjecpNEmZWbURSfc/BaoimAA9ImmZmp5c6mIbI\nzGoyR63BC0PkdqfG1XjgW8B/Fiw8pq6jcc41dB261GRQiHPOud2+B8s5V5Uk838XnHPOOefyy9WD\nVd9zsJxzzjnnnHPuS8sbWM45VwP+uyDJeJ6S8Twl43lKxvOUjOcpGc9T8byB5ZxzzjnnnHO1xOdg\nOeeq8DlYzjnnnHOF+Rws55xzzjnnnKtj3sByzrka8DHpyXiekvE8JeN5SsbzlIznKRnPU/H8d7Cc\nc9VI1Xq7nXOuQejQoYz161eVOgznnMvJ52A556qQZOD/LjjnGirh9y7OuYbA52A555xzzjnnXB2r\ntwaWpDJJr+bYNktSeX3FklF3V0nzJU2LrVtZilhykdRf0sQE5fLGLemj8LeTpMlh+QJJv67J8RLW\nOVrSVYWOU4z4MSVNlNSvQPmDJf1T0qdJYwnXZNcC24u6ZiUNkbRU0rPh/Z8lLZQ0KpzHmQX2T3Ku\n50haFF7PSzoitm28pCWS+hcTt8slVeoAGolUqQNoJFKlDqCRSJU6gEbB58wk43lKxvNUvPqeg9UQ\n+/QHA8+Y2TWxdQ0xziQxFSpjAGb2HjA0wX61UWdDsBH4MdFnXUojgYvM7J+SOgJHm9mBEDWeaqmO\nFUA/M9sq6TTgfqA3gJldLWku8ANgdi3V55xzzjnnYup7iGAzSQ+Hb/EnS2qRWUDScEmLw+v2sK5J\n+PZ+cfhmflRY30PS9NALME9S9xrE1Ab4IGPdhlg854c6F0h6MKybKOluSS9Ieivd8yCplaQZIZZF\nkgaF9WWSloX9lkuaJOmUsP9ySUeHci0lTZA0R9Irks4IYXwGbE1wLhvCccaGeOdLWitpQvp0YvHE\nexO7hh6Z5ZJuzJaHQnXmylWcpAMkTZP0sqTZkg6S1FrSqliZlpLWSGqarXyW+rcQ5ScnM/vQzF4B\nPk9wPmkbgR25rr1gqKSXJL0uqU+Iv0qPoKQnJPWTdAPQF5gg6efA00CX8Bn1zchTuaRUOO9pkjoU\nca5zzCx9rcwBumQUWU90zbtdVlHqABqJilIH0EhUlDqARqKi1AE0ChUVFaUOoVHwPCXjeSpeffdg\nHQxcaGZzwk3/ZcBd6Y2SOgG3A72Ibianh0bKWqCLmR0RyrUOu0wCxpnZVEnNqVmDsSlQGV9hZseF\neg4FrgWON7PNkuI3ph3NrI+kQ4CpwBTgU2CwmW2T1J7oBndqKN8D+L6ZLZU0DxgW9h8U6jgTuA54\n1sxGStoHmCtphpm9CLwYYjoK+KGZXZJ5Ium4zWw0MDoc4zkgfcMf722KLx8DHBbif1nSk2Y2P328\nfBLmKu3+EPvbko4F7jOzAaFB1t/MZgMDgb+b2Q5J1coDAzLqvzK9LGks8LKZPVko7gTnNSQcs5zs\n1x5AUzM7TtLpwBjglPTuWY53s6STgKvMbIGke4AnzKw8HHdk+LsH0ec1yMw2ShoKjANG1uBcLwKm\nZayrJLrmCxgTW67Ab2qcc845t7tLpVKJhkzWdwNrjZnNCcsPEw3buiu2/RhglpltApA0CegH3AJ0\nl3Q38DfgGUl7AZ3NbCqAmeX9Zj8bSQJ6hliyOQl41Mw2hzq2xLY9HtYtk/S19CGB2xTNk6kEOse2\nrTSzpWF5CTAjLL8KdAvLpwJnSPppeN8c6AosT1caemKqNa5yeBi4y8wWFig3PX1ukqYQ9bTMT1hH\nWr5cIakVcALwaMg7QLPwdzJwNtGwtWHAPQXKZxUalrVtBRnXXmzblPD3FaAs4fEKPf/8YOBwoi8X\nRPSlwbuZhQqdq6QTgQuJPsu4dcBBkvY0s+25jzCmQJgumgtSUeIYGoMUnqckUniekkiVOoBGIZVK\nea9DAp6nZDxPX6ioqKiSi7Fjx2YtV+o5WNnm71S7ATWzLZJ6At8GLgXOAq7IVrbKgaTLgItDPd8x\ns/WxbU2Ibp63A08VcQ5p8ZvTdBwjgH2BXmZWqegBEC2ylK+Mva/ki89BRL1cb9YgniokjSFq0FYb\nqpdFks9lVzUBNqd7bDJMBW6V1BYoB2YCe+UpX29yXHsXhc3pz3AHX3yGn1O1J7XaMNgCBLxmZn1q\nFjEoerDF/cBp6QZvmpmtkLQMWC1pgJktqWk9zjnnnHOuuvqeg1UmKT3s7BzgHxnb5wL9JLWT1BQY\nDswOw+2amtljwPVAuZltA96R9D0ASc0lfSV+MDO718x6mVl5vHEVtlWaWTdgHlHvSTYzgbMktQt1\ntM1RLt3A2gf4IDSuTqRqr0aSX259Grh85w7SkQn2qR5MNHfrZGBU5qYcu5wiqU3I32DghSzHXFag\n2ry5MrOPgJWShsSOeUTY9jHR53A38KRFcpbfRVVyoGjOXKechbNcewWOuwo4UpH9gWOTxhIsB74q\nqXeof48w/DIRRU8+/Ctwnpm9nWX7EUB3ot5fb1ztkopSB9BIVJQ6gEaiotQBNBIVpQ6gUfDehmQ8\nT8l4nopX3w2s14EfSVpKNNH+t2F9+ul264FriMYALCCaY/IE0UT9lKQFwB9DGYDzgcslLSJqFKQf\nBlCMN4B22TaEIX23EjXyFgDj4/HGi4a/k4BjQjznAsuylMm2f9rNRA8CWazoIRQ3ZRaQdFSYm5TP\nlUBnovlU80NvVr565xINd1tINMyvyvDA0MjIK0+u4s4FRip6KMlrwKDYtkeIegD/Els3Ik/5ahQ9\n3GNglvUdJL1DlJfrFD1EY68wBK8HsCnPYXNde1mvATN7gaiRtQT4JdHwQfLtk7H/v4AhwB2SFhL9\nd3B80nMFbiC6nu8Nc9vmZmxvC6wys8rquzrnnHPOuV2l3f3X0MN8p/YZj2l3MZK+C3Q3s9+UOpba\nJOkwooeu/KTUsdSX8NCMfzOz4XnKWON4+n6ppfBv05NI4XlKIoXnKYkUcCK7+71LIT5nJhnPUzKe\np9wkYWbVRiTV9xyshmgK8ICkaWZ2eqmDaYjMrCZz1Bq8MERud2pcjQe+BfxngtJ1HY5zztVIhw5J\nnynknHOlsdv3YDnnqpJk/u+Cc84551x+uXqw6nsOlnPOOeecc859aXkDyznnaiDJDw06z1NSnqdk\nPE/JeJ6S8Twl43kqnjewnHPOOeecc66W+Bws51wVPgfLOeecc64wn4PlnHPOOeecc3XMG1jOOVcD\nPiY9Gc9TMp6nZDxPyXiekvE8JeN5Kp7/DpZzrhrJfwfLuYakQ5cOrF+7vtRhOOecS8DnYDnnqpBk\njCl1FM65KsaA///aOecaFp+D5ZxzzjnnnHN1rN4aWJLKJL2aY9ssSeX1FUtG3V0lzZc0LbZuZSli\nyUVSf0kTE5TLG7ekj8LfTpImh+ULJP26JsdLWOdoSVcVOk4x4seUNFFSvwLlD5b0T0mfJo0lXJNd\nC2wv6pqVNETSUknPhvd/lrRQ0qhwHmcW2L/guYZyv5L0Zjj2kbH14yUtkdS/mLhdDg3qX4kGzPOU\njOcpEZ8LkoznKRnPUzKep+LV9xyshji+YTDwjJldE1vXEONMElOhMgZgZu8BQxPsVxt1NgQbgR8T\nfdalNBK4yMz+KakjcLSZHQhR46k2KpB0OtDDzA6UdBzwW6A3gJldLWku8ANgdm3U55xzzjnnqqrv\nIYLNJD0cvsWfLKlFZgFJwyUtDq/bw7om4dv7xZIWSRoV1veQND18Uz9PUvcaxNQG+CBj3YZYPOeH\nOhdIejCsmyjpbkkvSHor3fMgqZWkGSGWRZIGhfVlkpaF/ZZLmiTplLD/cklHh3ItJU2QNEfSK5LO\nCGF8BmxNcC4bwnHGhnjnS1oraUL6dGLxxHsTu4YemeWSbsyWh0J15spVnKQDJE2T9LKk2ZIOktRa\n0qpYmZaS1khqmq18lvq3EOUnJzP70MxeAT5PcD5pG4Edua69YKiklyS9LqlPiL9Kj6CkJyT1k3QD\n0BeYIOnnwNNAl/AZ9c3IU7mkVDjvaZI6JD1X4HvAQ+G8XwL2ie0PsJ7omne7qib/2uyOPE/JeJ4S\nqaioKHUIjYLnKRnPUzKep+LVdw/WwcCFZjYn3PRfBtyV3iipE3A70IvoZnJ6aKSsBbqY2RGhXOuw\nyyRgnJlNldScmjUYmwKV8RVmdlyo51DgWuB4M9ssKX5j2tHM+kg6BJgKTAE+BQab2TZJ7YE5YRtA\nD+D7ZrZU0jxgWNh/UKjjTOA64FkzGylpH2CupBlm9iLwYojpKOCHZnZJ5omk4zaz0cDocIzngPQN\nf7y3Kb58DHBYiP9lSU+a2fz08fJJmKu0+0Psb0s6FrjPzAaEBll/M5sNDAT+bmY7JFUrDwzIqP/K\n9LKkscDLZvZkobgTnNeQcMxysl97AE3N7DhFvUZjgFPSu2c53s2STgKuMrMFku4BnjCz8nDckeHv\nHkSf1yAz2yhpKDAOGJnwXLsA78Terwvr3g/vK4mu+fxmxZa74Td/zjnnnNvtpVKpREMm67uBtcbM\n5oTlh4mGbd0V234MMMvMNgFImgT0A24Buku6G/gb8IykvYDOZjYVwMwKfbNfjSQBPUMs2ZwEPGpm\nm0MdW2LbHg/rlkn6WvqQwG2K5slUAp1j21aa2dKwvASYEZZfJbqFBTiV/8/e/UdJVd353n9/QBlF\nUcEf/DBja1wa442jYvwVvdr6RExMNF6UJGoGV+JosnCpmJi1fDJ50uAvjI5JdObJGI1DouIa5F6S\nQY0iYpejKILQgAqiTtCIT1Bv/MksRq/yff7Y34LT1XW6ThXQRdHf11q1+tQ5+5z9Pd9zGmr33vsU\nnDQBvhkAACAASURBVC7ph/5+ELAPsLJcqffE9Ghc5bgb+JmZLalRbk753CTNJPW0LC5YR1lvuULS\nTsAXgBmed4Dt/ee9wDdIw9a+Cfy/NcpX5Q3Lze2PVNx7mW0z/ecioK3g8Wo9//wzwOdIf1wQ6Y8G\n/19loU0419eBAyX9lZl9mFvqpAaP3p+sIhqeRUSeiok8FVIqleKv6QVEnoqJPBUTedqovb29Wy4m\nT55ctVyz52BVm7/T4wOomb0r6VDgVOB7wDhgYrWy3Q4kTQAu9HpOM7M1mW0DSB+ePwQeqOMcyrIf\nTstxnAfsARxuZuuVHgCxQ5Xy6zPv17PxOojUy/VSA/F0I2kSqUHbY6heFUWuy6YaALxT7rGpMAu4\nVtJQYDTwKLBzL+X7TM6993e+uXwNP2HjNfyY7j2pPYbB1iDgOTM7rrGIeR3468z7T/k6AMzsj5JW\nAK9K+r/M7PkG6wkhhBBCCFX09RysNqWJ9wDnAo9XbF8AnCBpmKSBwDnAYz7cbqCZ/Q74MTDazNYC\nr0n6GoCkQZJ2zB7MzH5pZoeb2ehs48q3rTezfYFnSL0n1TwKjJM0zOsYmlOu3MDaFXjTG1cn0b1X\no8g3t84GLt2wQ+YJcPVQmrv1ReCyyk05u5wiaTfP35nAvCrHXFGj2l5zZWYfAKsknZ055t/4tv8k\nXYebgfstyS2/ibrlQGnO3MjcwlXuvRrHfQU4TMlfA0cVjcWtBPaUdIzXv50PvyxqFjDe9z0GeNfM\nysMDyzncj9T7G42rTRG9DcVEnoqJPBUSf0UvJvJUTOSpmMhT/fq6gfUCcLGk5aSJ9rf6+vLT7dYA\nVwIloIs0x+Q+0hySkqQu4C4vA+mD5KWSlpIaBdnJ/EW9CAyrtsGH9F1LauR1ATdl480W9Z/TgCM9\nnm8BK6qUqbZ/2dWkB4EsU3oIxVWVBSQd4XOTenM5MIo0n2qx92b1Vu8C0nC3JaRhft2GB3ojo1e9\n5CrrW8AFSg8leQ44I7NtOqkH8F8z687rpXwPSg/3+GqV9cMlvUbKy98rPURjZx+Ctz/wdi+Hzbv3\nqt4DZjaP1Mh6HvgFafggve1Tsf//Ac4GfippCen34Nii52pmfyA1TF8GfkWa55g1FHjFzNZX7htC\nCCGEEDad+vs3w/t8p90rHtMeMiR9BdjPzP6p2bFsTpL+G+mhK1c0O5a+4g/N+B9mdk4vZYxJfRdT\ny4o5M8VEnoqpladJ0N//v4aYC1JU5KmYyFMxkad8kjCzHiOS+noO1tZoJvAbSQ+a2ZebHczWyMwa\nmaO21fMhcv2pcXUT8N+B/7tm4UlbOpoQQj2G793IAI0QQgjN0O97sEII3Umy+HchhBBCCKF3eT1Y\nfT0HK4QQQgghhBC2WdHACiGEBhT5osEQeSoq8lRM5KmYyFMxkadiIk/1iwZWCCGEEEIIIWwmMQcr\nhNBNzMEKIYQQQqgt5mCFEEIIIYQQwhYWDawQQmhAjEkvJvJUTOSpmMhTMZGnYiJPxUSe6hffgxVC\n6EHq0dsdQgghhD40fHgba9a80uwwQgNiDlYIoRtJBvHvQgghhNBcIj6nb91iDlYIIYQQQgghbGFb\ntIElqU3SsznbOiWN3pL155G0j6TFkh7MrFvVjFjySDpR0tQC5eqOW9JlknbI2Xa+pFt8uUPS+BrH\nOl9SR40yH9QbYy3lY/o91lmgfKekFyR1+bXfo0b5XvPv2++rM+ZBkuZ4/eMkHS/pOX9/UN7vSmb/\nmucqaUdJ90taIelZSddlth3o9U2vJ+6Qp9TsAFpEqdkBtIhSswNoEaVmB9AiSs0OoEWUmh1AS4g5\nWPXrix6srbFv80zgYTP7cmbd1hhnkZgaiXsiMLiB/RqNYUvk1nKWe3OOmR1uZqPN7H/XWUcj2yuN\nBszrnwGcB1xnZqOBdQWPV6TMjWb2WeBw4HhJp5IqftHMPgccImm/OmMPIYQQQggF9EUDa3tJd0ta\nLuneaj0nks6RtMxf1/u6AZKm+rqlki7z9ft7L8ASSc80+EFxN+DNinVvZeIZ73V2Sfqtr5sq6WZJ\n8yS9LGmsr99J0iMey1JJZ/j6Nu9FmCpppaRpkk7x/VdK+ryXGyzpDknzJS2SdLqH8RHwXoFzecuP\nMznTO7PajznYezO6PI/jJF0CjAI6Jc31fb/tMc0Hjsscey3pg39v1nk5JO0laaZfmy5Jx5RTmsnt\nFZIWeJkOXzdF0oRMmQ5J388rX+ET4O0CeYL67vcN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0d4\n79mewF3AkX6ci4CvA1dn9/V9Bki6QdLTft4XFj1XM1tnZo/58sfAYuBTFcXeIP0OhE3S3uwAWkR7\nswNoEe3NDqBFtDc7gBbR3uwAWkR7swNoCe3t7c0OofWY2RZ7AW3AeuAYf38H8H1f7iT9RX8k8Cow\njPQBeC5whm97OHOsXfznfOAMXx4E7NBAXJOBiTnbDgZeAIb6+93851Rgui9/FnjJlwcCO/vy7pn1\nbaQP6Qf7+2eAO3z5DGCmL18LnOvLuwIrgR0rYjoCuK3gue0KLCX1XowFfpXZNsR//jFzfiMy+d8O\neAK4pcHr/a/Apb6sTH3v+89TyvH49vuA44HDgFLmOM8De+eV9/cfVKl/JHB/TmydwLOkBseP6zyv\nWcCxvjzY79MTgXe8TgFPAl/I5HdY5to96ssnArMyx50KjM3cL8t8+ULgR5l7fCHQVvRcM2V2A/4D\n2Ldi/Vzg873sZ9CReXUaWLziFa94xSte8erTFxa2Lp2dndbR0bHh5deIyldf9GD9yczm+/LdpA/U\nWUcCnWb2tpmtB6YBJ5A+pO7nvUanAh9I2hkYZWazSGf0kZn9Vz3BSBJwKLA6p8jJwAwze8freDez\n7fe+bgWwV/mQwBRJS4FHgFGSyttWmdlyX37et0P6oL+vL48BrpTURRoMPAjYJxuQmS0ys4sKnuLd\nwE1m1uX1nOI9RMebWXkulNjYq3Q0G/P/MbAp83NOBv7ZY7ZMfWVjPJ7FpIbOZ4ADzGwJsKekEZL+\nBnjbzF7PK59XuZn92cy+mrP5XDM7BPjvwH+X9K06zmse8HPv/Rvq9ynAAq/TgCVsvKab+ozzMcB4\nvyeeJjV+u513jXNF0kDgHuAXZvZKxebVpN+BXkzKvNqLR96vlJodQIsoNTuAFlFqdgAtotTsAFpE\nqdkBtIhSswNoCTEHa6P29nYmTZq04ZWnL74Hy2q8hyofSM3sXUmHAqcC3wPGkeYO9frh1YeaXej1\nnGZmazLbBpAabh8CD9RxDmUfVon5PGAP4HAzW6/00IkdqpRfn3m/no25F3CWmb3UQDzdSJpEatDe\nCWBmLyk9SOQ04BpJj5jZNdV23dS6XbVrW1nPFDO7vcq2GaRrPIKNjbzeyteqq3thsz/7z/+UdA9w\nFKkxWmTfn0q6H/gKME/SGN+Uvb6fsPGafszG4YhVHyZSg4BLzGxOA/uW3QasNLN/rLLtV8BsSUeZ\n2Xc3oY4QQgghhFChL3qw2iQd7cvnAo9XbF8AnCBpmP/V/RzgMUm7AwPN7HfAj4HRZrYWeE3S12DD\nU9l2zB7MzH5pGx9ksKZi23oz25c0XO8bOfE+CozLzKEZmlOu3CjZFXjTG1cnkYZ6VZbpzWzg0g07\nSIcV2KdnMGnu1heByzLrRgLrzOwe4EbSsEuA94FdfPlpUv6HStqe1MipdvyLs/OkcswFJnj5AZKG\nlHf3n7OB75TnMEka5XOTAO4FvgmcRWps5ZXfo+KYNUka6PcTfo5fBZ7z92cq86S9nP0/bWbPm9kN\npOF6B9WochVpaCB+PvWaDUyQtJ3Xf0DlfV4j3mtIQ2ovzylyBXBBNK42VXuzA2gR7c0OoEW0NzuA\nFtHe7ABaRHuzA2gR7c0OoCXEHKz69UUD6wXgYknLSXNCbvX1BuCNoCtJ/bRdwEIzu480B6fkw6Tu\n8jIA44FLfUjePGB4AzG9SBp21YMP6buW1MjrAm7Kxpst6j+nkR5csBT4FrCiSplq+5ddTXoQyDKl\nx3RfVVnAH5RwWy/nA3A56eEVC/0hCpOAQ4AFfh4/Acq9V7cDD0ma6/mfTJrb9jiwvMeRk4OAv9SI\nYSJwkqRlpEbswb6+fK3nkIatPeVlZgA7+7blwBBgtZm90Uv5IdljZkka6T1Nlf6K1GOzhDTUcLXn\nAGB/aj9MZKLSI8+XkubVPVilTDaeq4BbJC0g9Wblybsnfk26Dov9nriVit7mvHOVtDfwI+DgzIM5\nvlNRbCjwci9xhRBCCCGEBilNH+lfJP0Q2N3MrqxZOAAgaRbpgQy9NRhajqQ7gcvNrFbjcZvgcxCX\nAWeb2cqcMlbnCMx+qkT89bOIEpGnIkpEnoooEXkqokTkqYgSW3eexNbwOb1UKkUvVg5JmFmPUVV9\n0YO1NZoJHKfMFw2H3pnZGdta4wrAzMb3o8bVgaRe4i5SL24IIYQQQtjM+mUPVgghX+rBCiGEEEIz\nDR/expo1rzQ7jNCLvB6svniKYAihxcQfXkIIIYQQGtNfhwiGEMImie8FKSbyVEzkqZjIUzGRp2Ii\nT8VEnuoXDawQQgghhBBC2ExiDlYIoRtJFv8uhBBCCCH0Lp4iGEIIIYQQQghbWDSwQgihATEmvZjI\nUzGRp2IiT8VEnoqJPBUTeapfNLBCCCGEEEIIYTOJOVghhG7ie7BCCCFUM3zv4axZvabZYYSw1cib\ngxUNrBBCN5KMSc2OIoQQwlZnUnxPYghZ8ZCLEELYnFY1O4AWEXkqJvJUTOSpmMhTITG3qJjIU/22\naANLUpukZ3O2dUoavSXrzyNpH0mLJT2YWbdV/XMk6URJUwuUqztuSZdJ2iFn2/mSbvHlDknjaxzr\nfEkdNcp8UG+MtZSP6fdYZ4HynZJekNTl136PGuV7zb9vv6/OmAdJmuP1j5N0vKTn/P1Beb8rmf2L\nnutoScskvSjpF5n1B3p90+uJO4QQQgghFNcXPVhbY1/ymcDDZvblzLqtMc4iMTUS90RgcAP7NRrD\nlsit5Sz35hwzO9zMRpvZ/66zjka2VxoNmNc/AzgPuM7MRgPrCh6vSJl/Bi4wswOBAyWdSqr4RTP7\nHHCIpP3qjD1UigwWE3kqJvJUTOSpmMhTIe3t7c0OoSVEnurXFw2s7SXdLWm5pHur9ZxIOsf/4r5M\n0vW+boCkqb5uqaTLfP3+3guwRNIzDX5Q3A14s2LdW5l4xnudXZJ+6+umSrpZ0jxJL0sa6+t3kvSI\nx7JU0hm+vk3SCt9vpaRpkk7x/VdK+ryXGyzpDknzJS2SdLqH8RHwXoFzecuPMznTO7PajzlY0v2+\nfpn3mlwCjAI6Jc31fb/tMc0Hjsscey3pg39v1nk5JO0laaZfmy5Jx5RTmsntFZIWeJkOXzdF0oRM\nmQ5J388rX+ET4O0CeYL67vcN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0d479mewF3AkX6ci4Cv\nA1dn9/V9Bki6QdLTft4XFj1XSSOAIWa20FfdSfqDQtYbpN+BEEIIIYSwmW3XB3V8Bvi2mc2XdAcw\nAfhZeaOkkcD1wOHAu8Acb6SsBvY2s7/xcrv4LtNIf/WfJWkQjTUSBwLrsyvM7Giv52DgR8CxZvaO\npOwH0RFmdpykzwKzgJnAfwFnmtlaSbsD830bwP7AWWa2XNIzwDd9/zO8jrHA3wNzzewCSbsCCyQ9\nYmZPAU95TEcA3zWziypPpBy3mXUAHX6Mfwf+CfgS8LqZfdWPM8TMPpB0OdDu5zcCmETK//tACVjs\nx7ypViLN7N7M21uAkpmNlSRg53Ixr/8U4AAzO8q3z5J0PDAd+AXwSy//dWBMXnkzewJvtJnZauBs\nP/5I4Pby+VbxG0n/B5hpZtfUOK8N+Qd+AEwws6ckDSZdc4DDgIOBNcA8SV8wsyfp2ctkZvaWpL8D\nfmBm5Ub4scB9ZjZTUlum/AXAu2Z2tN/j8yQ9bGavFjjXvUm/O2WrfV3WetLvQL7sQMR9ib+GVrOK\nyEsRkadiIk/FRJ6KiTwVUiqVonemgMjTRqVSqdCctL5oYP3JzOb78t3AJWQaWMCRQKeZvQ0gaRpw\nAnANsJ+km4E/AA9L2hkYZWazAMzso3qD8Q/qh3os1ZwMzDCzd7yOdzPbfu/rVkjaq3xIYIqkE0gf\nXEdltq0ys+W+/DzwiC8/S/rYCjAGOF3SD/39IGAfYGW5UjNbBPRoXOW4G7jJzLokrQX+QdIU4AFv\nmJRjLvcqHU33/E8HDihYV6WTgb/1mA2onHs1BjhF0mKvfydSA2qqpD29sbcX8LaZvS5pYrXywBNU\nYWZ/BvIaV+ea2Z+992mmpG+ZWd49UGke8HO/N2d6bAALvE4kLSFd0yfJ9Ng1aAxpGN84f78L6bxf\nLReoca61rCb9DjyTW+KkBo8cQgghhLCNam9v79bYnDx5ctVyfdHA6vHX/CplenwgNbN3JR0KnAp8\nDxhHmjvU64dXH2p2oddzmpmtyWwbAPwR+BB4oI5zKPuwSsznAXsAh5vZeqWHTuxQpfz6zPv1bMy9\nSL1cLzUQTzeSJpEatHcCmNlLSg8SOQ24xnvGqvXcbGqDoKzW/CABU8zs9irbZpCu8QhSj1at8nXN\nfyo3hMzsPyXdAxxFfiO7ct+fSrof+AqpN2mMb8pe30/YeE0/ZmPPatWHidQg4BIzm9PAvq8Df515\n/ylfl/UrYLako8zsuw3UESD+OlxU5KmYyFMxkadiIk+FRK9MMZGn+vXFHKw2SUf78rnA4xXbFwAn\nSBomaSBwDvCYD7cbaGa/A34MjDaztcBrkr4GG57KtmP2YGb2y8yDDNZUbFtvZvuS/nL/jZx4HwXG\nZebQDM0pV26U7Aq86Y2rk4C2KmV6Mxu4dMMO0mEF9ukZTJq79UXgssy6kcA6M7sHuJH0kAVIQwHL\nQy6fJuV/qKTtSY2case/ODtPKsdc0hDQ8jyiIeXd/eds4DvlOUySRvncJIB7gW8CZ5EaW3nl96g4\nZk2SBvr9hJ/jV4Hn/P2Zkq6rsf+nzex5M7sBWAgcVKPKVcARvnxW0TgzZgMTJG3n9R9QeZ/n8Xv+\nPUnlYZXjgX+rKHYF6SEY0bgKIYQQQtjM+qKB9QJwsaTlpIn1t/p6gw0fCK8kzf3pAhaa2X2keSMl\nSV2khwNc6fuNBy6VtJQ0dGt4AzG9CAyrtsGH9F1LauR1AeV5SHk9cdNIDy5YCnwLWFGlTLX9y64m\nPQhkmdJjuq+qLKD0oITbejkfgMtJD69YqPQQhUnAIaQ5XV3AT0jDLgFuBx6SNNfzP5k0d+xxYHmP\nIycHAX+pEcNE4CRJy0iN2IN9fflazwHuAZ7yMjPweVqe9yHAajN7o5fyQ7LHzJI00nuaKv0Vqcdm\nCWl+2WrPAaR5crUeJjJR0rN+jT8CHqxSJhvPVcAtkhaQerPy5N0TvyZdh8V+T9xKRW9zL+cKcDFw\nB+k+f8nMHqrYPhR4uZe4QhFb1Rc7bMUiT8VEnoqJPBUTeSokvt+pmMhT/dQfv5Hb5zvtbmZX1iwc\nAJA0CxhrZr01GFqOpDuBy82sVuNxm+C9WsuAs81sZU4ZY1KfhtWaYhJ5MZGnYiJPxUSeitlSeZoE\n29Lnxnh4QzGRp3ySMLMeo6r6awNrf+A3wNqK78IKYZsl6UDSUMxlwPmW88svqf/9oxBCCKGm4XsP\nZ83qNbULhtBPRAMrhFCIpLy2VwghhBBCcHkNrL6YgxVCCNucGJNeTOSpmMhTMZGnYiJPxUSeiok8\n1S8aWCGEEEIIIYSwmcQQwRBCNzFEMIQQQgihthgiGEIIIYQQQghbWDSwQgihATEmvZjIUzGRp2Ii\nT8VEnoqJPBUTeapfNLBCCCGEEEIIYTOJOVghhG7ie7BCCCH0d8OHt7FmzSvNDiNs5eJ7sEIIhaQG\nVvy7EEIIoT8T8Rk51BIPuQghhM2q1OwAWkSp2QG0iFKzA2gRpWYH0CJKzQ6gRZSaHUBLiDlY9dui\nDSxJbZKezdnWKWn0lqw/j6R9JC2W9GBm3apmxJJH0omSphYoV3fcki6TtEPOtvMl3eLLHZLG1zjW\n+ZI6apT5oN4Yaykf0++xzgLlH5TUJek5Sb+WtF2N8r3m37ffV2fMgyTN8XtvnKTjPZ7Fkg7K+13J\n7F/zXCXtKOl+SSskPSvpusy2A72+6fXEHUIIIYQQiuuLHqytsX/1TOBhM/tyZt3WGGeRmBqJeyIw\nuIH9Go1hS+TWcpbzjDOzw83sc8BuwDfqrKOR7ZVGA2Zmo81sBnAecJ2ZjQbWFTxekTI3mtlngcOB\n4yWdSqr4RT//QyTtV2fsoYf2ZgfQItqbHUCLaG92AC2ivdkBtIj2ZgfQItqbHUBLaG9vb3YILacv\nGljbS7pb0nJJ91brOZF0jqRl/rre1w2QNNXXLZV0ma/f33sBlkh6psEPirsBb1aseysTz3ivs0vS\nb33dVEk3S5on6WVJY339TpIe8ViWSjrD17d5L8JUSSslTZN0iu+/UtLnvdxgSXdImi9pkaTTPYyP\ngPcKnMtbfpzJHu9iSav9mIO9N6PL8zhO0iXAKKBT0lzf99se03zguMyx15I++PdmnZdD0l6SZvq1\n6ZJ0TDmlmdxeIWmBl+nwdVMkTciU6ZD0/bzyFT4B3q6VJDMrx7g9MAj4S41dNuTfe6vKuV0kaScv\nM0TSDL/Od2XiXyVpmC8fodRbuydwF3CkH+ci4OvA1dl9fZ8Bkm6Q9LSf94VFz9XM1pnZY778MbAY\n+FRFsTdIvwMhhBBCCGFzM7Mt9gLagPXAMf7+DuD7vtxJ+ov+SOBVYBipwTcXOMO3PZw51i7+cz5w\nhi8PAnZoIK7JwMScbQcDLwBD/f1u/nMqMN2XPwu85MsDgZ19effM+jbSh/SD/f0zwB2+fAYw05ev\nBc715V2BlcCOFTEdAdxW8Nx2BZaSei/GAr/KbBviP/+YOb8RmfxvBzwB3NLg9f5X4FJfVqa+9/3n\nKeV4fPt9wPHAYUApc5zngb3zyvv7D6rUPxK4v5f4HiI1rKbXeV6zgGN9ebDfpycC73idAp4EvpDJ\n77DMtXvUl08EZmWOOxUYm7lflvnyhcCPMvf4QqCtnnMt37vAfwD7VqyfC3y+l/0MOjKvTgOLV49X\n5CXyFHmKPG2tr8jTpucJC0lnZ2ezQ9hqdHZ2WkdHx4aX3ydUvnqdh7KZ/MnM5vvy3cAlwM8y248E\nOs3sbQBJ04ATgGuA/STdDPwBeFjSzsAoM5tFOqOP6g1GkoBDPZZqTgZmmNk7Xse7mW2/93UrJO1V\nPiQwRdIJpMbkqMy2VWa23JefBx7x5WeBfX15DHC6pB/6+0HAPqSGFl7fIuCigqd4N3CTmXVJWgv8\ng6QpwANm9kQm5nKv0tF0z/904ICCdVU6Gfhbj9mAyrlXY4BTJC32+ncCDjCzqZL2lDQC2At428xe\nlzSxWnlSI7AHM/sz8NW84MzsS5IGAfdKGm9mdxY8r3nAz/3enOmxASzwOpG0hHRNnyTTY9egMaRh\nfOP8/S6k8341cy69nqukgcA9wC/M7JWKzatJvwPP5Icwqf6oQwghhBC2Ye3t7d2GTE6ePLlqub5o\nYFmN91DlA6mZvSvpUOBU4HvAONLcoV4/vPpQswu9ntPMbE1m2wBS78KHwAN1nEPZh1ViPg/YAzjc\nzNYrPXRihyrl12fer2dj7gWcZWYvNRBPN5ImkRq0dwKY2UtKDxI5DbhG0iNmdk21XTe1blft2lbW\nM8XMbq+ybQbpGo8AphcoX6uu6gGafSTpfwFHAYUaWGb2U0n3A18B5kka45uy1/cTNl7Tj9k4/Lbq\nw0RqEHCJmc1pYN+y24CVZvaPVbb9Cpgt6Sgz++4m1NHPtTc7gBbR3uwAWkR7swNoEe3NDqBFtDc7\ngBbR3uwAWkLMwapfX8zBapN0tC+fCzxesX0BcIKkYf5X93OAxyTtDgw0s98BPwZGW5pH85qkr8GG\np7LtmD2Ymf3S0sMMRmcbV75tvZntS/rLfd5DDh4FxmXm0AzNKVdulOwKvOmNq5NIQ70qy/RmNnDp\nhh2kwwrs0zOYNHfri8BlmXUjgXVmdg9wI2nYJcD7pF4RgKdJ+R/q85PGUYWki7PzpHLMBSZ4+QGS\nhpR395+zge+U5zBJGuVzkwDuBb4JnEVqbOWV36PimDUpzZMb4cvbkRpKS/z9mco8aS9n/0+b2fNm\ndgNpuN5BNapcRRoaiJ9PvWYDEzxWJB1QeZ/XiPca0pDay3OKXAFcEI2rEEIIIYTNry8aWC8AF0ta\nTpoTcquvNwBvBF1J+jKCLmChmd1HmoNTktRFejjAlb7feOBSSUtJQ7eGNxDTi6Q5Rz34kL5rSY28\nLuCmbLzZov5zGunBBUuBbwErqpSptn/Z1aQHgSxTekz3VZUF/EEJt/VyPgCXkx5esdAfojAJOARY\n4OfxE9KwS4DbgYckzfX8TybNbXscWN7jyMlB1H4wxETgJEnLSI3Yg319+VrPIQ1be8rLzAB29m3L\ngSHAajN7o5fyQ7LHzJI00nuaKu0EzPJhfIuA14B/8W37U/thIhOVHnm+lDSv7sEqZbLxXAXcImkB\nqTcrT9498WvSdVjs98StVPQ2552rpL2BHwEHZx7M8Z2KYkOBl3uJKxRSanYALaLU7ABaRKnZAbSI\nUrMDaBGlZgfQIkrNDqAlxPdg1U9pqkz/4vOddjezK2sWDgBImkV6IENvDYaWI+lO4HIzq9V43Cb4\nHMRlwNlmtjKnjDU4ArOfKRHDS4ooEXkqokTkqYgSkaciSkSeiiiRnyfRHz8jV1MqlWKYYA5JmFmP\nUVX9tYG1P/AbYK11/y6sELZZkg4kDcVcBpxvOb/8qYEVQggh9F/Dh7exZs0rzQ4jbOWigRVCKERS\nXtsrhBBCCCG4vAZWX8zBCiGEbU6MSS8m8lRM5KmYyFMxkadiIk/FRJ7qFw2sEEIIIYQQQthMbNT0\nHgAAIABJREFUYohgCKGbGCIYQgghhFBbDBEMIYQQQgghhC0sGlghhNCAGJNeTOSpmMhTMZGnYiJP\nxUSeiok81S8aWCGEEEIIIYSwmcQcrBBCN/E9WCGE0DeG7z2cNavXNDuMEEKD4nuwQgiFSDImNTuK\nEELoByZBfA4LoXXFQy5CCGFzWtXsAFpE5KmYyFMxkadCYs5MMZGnYiJP9duiDSxJbZKezdnWKWn0\nlqw/j6R9JC2W9GBm3Vb1z7akEyVNLVCu7rglXSZph5xt50u6xZc7JI2vcazzJXXUKPNBvTHWUj6m\n32OdBco/KKlL0nOSfi1puxrle82/b7+vzpgHSZrj9944Scd7PIslHZT3u5LZv+i5jpa0TNKLkn6R\nWX+g1ze9nrhDCCGEEEJxfdGDtTX2fZ8JPGxmX86s2xrjLBJTI3FPBAY3sF+jMWyJ3FrOcp5xZna4\nmX0O2A34Rp11NLK90mjAzGy0mc0AzgOuM7PRwLqCxytS5p+BC8zsQOBASaeSKn7Rz/8QSfvVGXuo\nFBksJvJUTOSpmMhTIe3t7c0OoSVEnoqJPNWvLxpY20u6W9JySfdW6zmRdI7/xX2ZpOt93QBJU33d\nUkmX+fr9vRdgiaRnGvyguBvwZsW6tzLxjPc6uyT91tdNlXSzpHmSXpY01tfvJOkRj2WppDN8fZuk\nFb7fSknTJJ3i+6+U9HkvN1jSHZLmS1ok6XQP4yPgvQLn8pYfZ7LHu1jSaj/mYEn3+/pl3mtyCTAK\n6JQ01/f9tsc0Hzguc+y1pA/+vVnn5ZC0l6SZfm26JB1TTmkmt1dIWuBlOnzdFEkTMmU6JH0/r3yF\nT4C3ayXJzMoxbg8MAv5SY5cN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0co9dbuCdwFHOnHuQj4\nOnB1dl/fZ4CkGyQ97ed9YdFzlTQCGGJmC33VnaQ/KGS9QfodCCGEEEIIm1mvw6Q2k88A3zaz+ZLu\nACYAPytvlDQSuB44HHgXmOONlNXA3mb2N15uF99lGumv/rMkDaKxRuJAYH12hZkd7fUcDPwIONbM\n3pGU/SA6wsyOk/RZYBYwE/gv4EwzWytpd2C+bwPYHzjLzJZLegb4pu9/htcxFvh7YK6ZXSBpV2CB\npEfM7CngKY/pCOC7ZnZR5YmU4zazDqDDj/HvwD8BXwJeN7Ov+nGGmNkHki4H2v38RgCTSPl/HygB\ni/2YN9VKpJndm3l7C1Ays7GSBOxcLub1nwIcYGZH+fZZko4HpgO/AH7p5b8OjMkrb2ZP4I02M1sN\nnO3HHwncXj7fSpIeAo4EHjGzh2qc14b8Az8AJpjZU5IGk645wGHAwcAaYJ6kL5jZk/TsZTIze0vS\n3wE/MLNyI/xY4D4zmympLVP+AuBdMzva7/F5kh42s1cLnOvepN+dstW+Lms96XcgX3Yg4r7EX42r\nWUXkpYjIUzGRp2IiT4WUSqXodSgg8lRM5GmjUqlUaE5aXzSw/mRm8335buASMg0s0gfeTjN7G0DS\nNOAE4BpgP0k3A38AHpa0MzDKzGYBmNlH9QbjH9QP9ViqORmYYWbveB3vZrb93tetkLRX+ZDAFEkn\nkD64jspsW2Vmy335eeARX36W9LEVYAxwuqQf+vtBwD7AynKlZrYI6NG4ynE3cJOZdUlaC/yDpCnA\nA94wKcdc7lU6mu75nw4cULCuSicDf+sxG1A592oMcIqkxV7/TqQG1FRJe3pjby/gbTN7XdLEauWB\nJ6jCzP4MVG1c+fYveYPlXknjzezOguc1D/i535szPTaABV4nkpaQrumTZHrsGjSGNIxvnL/fhXTe\nr2bOpddzrWE16XfgmdwSJzV45BBCCCGEbVR7e3u3xubkyZOrluuLBlaPv+ZXKdPjA6mZvSvpUOBU\n4HvAONLcoV4/vPpQswu9ntPMbE1m2wDgj8CHwAN1nEPZh1ViPg/YAzjczNYrPXRihyrl12fer2dj\n7kXq5XqpgXi6kTSJ1KC9E8DMXlJ6kMhpwDXeM3ZNtV03tW5Xa36QgClmdnuVbTNI13gEqUerVvmG\n5nWZ2UeS/hdwFGn4XJF9firpfuArpN6kMb4pe30/YeM1/ZiNPatVHyZSg4BLzGxOA/u+Dvx15v2n\nfF3Wr4DZko4ys+82UEeA+Ct6UZGnYiJPxUSeConehmIiT8VEnurXF3Ow2iQd7cvnAo9XbF8AnCBp\nmKSBwDnAYz7cbqCZ/Q74MTDa59G8JulrsOGpbDtmD2Zmv/SHGYzONq5823oz25f0l/u8hxw8CozL\nzKEZmlOu3CjZFXjTG1cnAW1VyvRmNnDphh2kwwrs0zOYNHfri8BlmXUjgXVmdg9wI+khC5CGApaH\nXD5Nyv9Qn580jiokXZydJ5VjLmkIaHke0ZDy7v5zNvCd8hwmSaN8bhLAvcA3gbNIja288ntUHLMm\npXlyI3x5O1JDaYm/P1PSdTX2/7SZPW9mNwALgYNqVLkKOMKXzyoaZ8ZsYILHiqQDKu/zPH7Pvyep\nPKxyPPBvFcWuID0EIxpXIYQQQgibWV80sF4ALpa0nDSx/lZfb7DhA+GVpLk/XcBCM7uPNG+kJKmL\n9HCAK32/8cClkpaShm4NbyCmF4Fh1Tb4kL5rSY28LqA8DymvJ24a6cEFS4FvASuqlKm2f9nVpAeB\nLFN6TPdVlQWUHpRwWy/nA3A56eEVC5UeojAJOIQ0p6sL+Alp2CXA7cBDkuZ6/ieT5o49DizvceTk\nIGo/GGIicJKkZaRG7MG+vnyt5wD3AE95mRn4PC3P+xBgtZm90Uv5IdljZkka6T1NlXYizd9aAiwC\nXgP+xbftT+2HiUyU9Kxf44+AB6uUycZzFXCLpAWk3qw8effEr0nXYbHfE7dS0dvcy7kCXAzcQbrP\nX6oy32wo8HIvcYUitqovdtiKRZ6KiTwVE3kqJL63qJjIUzGRp/qpP36DuM932t3MrqxZOAAgaRYw\n1sx6azC0HEl3ApebWa3G4zbBe7WWAWeb2cqcMsakPg2rNcVk+2IiT8VEnorZ1vI0CbbE57B4KEEx\nkadiIk/5JGFmPUZV9dcG1v7Ab4C1Fd+FFcI2S9KBpKGYy4DzLeeXX1L/+0chhBCaYPjew1mzek3t\ngiGErVI0sEIIhUjKa3uFEEIIIQSX18DqizlYIYSwzYkx6cVEnoqJPBUTeSom8lRM5KmYyFP9ooEV\nQgghhBBCCJtJDBEMIXQTQwRDCCGEEGqLIYIhhBBCCCGEsIVFAyuEEBoQY9KLiTwVE3kqJvJUTOSp\nmMhTMZGn+kUDK4QQQgghhBA2k5iDFULoJr4HK4QQQn8zfHgba9a80uwwQouJ78EKIRSSGljx70II\nIYT+RMRn4lCveMhFCCFsVqVmB9AiSs0OoEWUmh1Aiyg1O4AWUWp2AC2i1OwAWkLMwarfFm1gSWqT\n9GzOtk5Jo7dk/Xkk7SNpsaQHM+tWNSOWPJJOlDS1QLm645Z0maQdcradL+kWX+6QNL7Gsc6X1FGj\nzAf1xlhL+Zh+j3UWKH+NpD9Jer/g8XvNv2+/r3jEIGmQpDl+742TdLyk5/z9QXm/K5n9a56rpB0l\n3S9phaRnJV2X2Xag1ze9nrhDCCGEEEJxfdGDtTX2t54JPGxmX86s2xrjLBJTI3FPBAY3sF+jMWyJ\n3FrOcp5ZwJGbUEcj2yuNBszMRpvZDOA84DozGw2sK3i8ImVuNLPPAocDx0s6lVTxi2b2OeAQSfvV\nGXvoob3ZAbSI9mYH0CLamx1Ai2hvdgAtor3ZAbSI9mYH0BLa29ubHULL6YsG1vaS7pa0XNK91XpO\nJJ0jaZm/rvd1AyRN9XVLJV3m6/f3XoAlkp5p8IPibsCbFeveysQz3uvskvRbXzdV0s2S5kl6WdJY\nX7+TpEc8lqWSzvD1bd6LMFXSSknTJJ3i+6+U9HkvN1jSHZLmS1ok6XQP4yPgvQLn8pYfZ7LHu1jS\naj/mYO/N6PI8jpN0CTAK6JQ01/f9tsc0Hzguc+y1pA/+vVnn5ZC0l6SZfm26JB1TTmkmt1dIWuBl\nOnzdFEkTMmU6JH0/r3yFT4C3ayXJzBaY2Ru1ymVsyL/3VpVzu0jSTl5miKQZfp3vysS/StIwXz5C\nqbd2T+Au4Eg/zkXA14Grs/v6PgMk3SDpaT/vC4ueq5mtM7PHfPljYDHwqYpib5B+B0IIIYQQwuZm\nZlvsBbQB64Fj/P0dwPd9uZP0F/2RwKvAMFKDby5whm97OHOsXfznfOAMXx4E7NBAXJOBiTnbDgZe\nAIb6+93851Rgui9/FnjJlwcCO/vy7pn1baQP6Qf7+2eAO3z5DGCmL18LnOvLuwIrgR0rYjoCuK3g\nue0KLCX1XowFfpXZNsR//jFzfiMy+d8OeAK4pcHr/a/Apb6sTH3v+89TyvH49vuA44HDgFLmOM8D\ne+eV9/cfVKl/JHB/jRjfb+C8ZgHH+vJgv09PBN7xOgU8CXwhk99hmWv3qC+fCMzKHHcqMDZzvyzz\n5QuBH2Xu8YVAWwPnuhvwH8C+FevnAp/vZT+Djsyr08Di1eMVeYk8RZ4iT1vrK/JUf56wUF1nZ2ez\nQ9hqdHZ2WkdHx4aX3zdUvrZjy/uTmc335buBS4CfZbYfCXSa2dsAkqYBJwDXAPtJuhn4A/CwpJ2B\nUWY2i3RGH9UbjCQBh3os1ZwMzDCzd7yOdzPbfu/rVkjaq3xIYIqkE0iNyVGZbavMbLkvPw884svP\nAvv68hjgdEk/9PeDgH1IDS28vkXARQVP8W7gJjPrkrQW+AdJU4AHzOyJTMzlXqWj6Z7/6cABBeuq\ndDLwtx6zAZVzr8YAp0ha7PXvBBxgZlMl7SlpBLAX8LaZvS5pYrXypEZgD2b2Z+CrDcbem3nAz/3e\nnOmxASzwOpG0hHRNnyTTY9egMaRhfOP8/S6k8361XKDWuUoaCNwD/MLMXqnYvJr0O/BMfgiT6o86\nhBBCCGEb1t7e3m3I5OTJk6uW64sGltV4D1U+kJrZu5IOBU4FvgeMI80d6vXDqw81u9DrOc3M1mS2\nDSD1LnwIPFDHOZR9WCXm84A9gMPNbL3SQyd2qFJ+feb9ejbmXsBZZvZSA/F0I2kSqUF7J4CZvaT0\nIJHTgGskPWJm11TbdVPrdtWubWU9U8zs9irbZpCu8QhgeoHyterabMzsp5LuB74CzJM0xjdlr+8n\nbLymH7Nx+G3Vh4nUIOASM5vTSLzuNmClmf1jlW2/AmZLOsrMvrsJdfRz7c0OoEW0NzuAFtHe7ABa\nRHuzA2gR7c0OoEW0NzuAlhBzsOrXF3Ow2iQd7cvnAo9XbF8AnCBpmP/V/RzgMUm7AwPN7HfAj4HR\nZrYWeE3S12DDU9l2zB7MzH5pZodbepDAmopt681sX9Jf7r+RE++jwLjMHJqhOeXKjZJdgTe9cXUS\naahXZZnezAYu3bCDdFiBfXoGk+ZufRG4LLNuJLDOzO4BbiQNuwR4n9QrAvA0Kf9DJW1PauRUO/7F\n2XlSOeYCE7z8AElDyrv7z9nAd8pzmCSN8rlJAPcC3wTOIjW28srvUXHMenXbT9KZyjxpr+oO0qfN\n7Hkzu4E0XO+gGnWsIg0NhHQ+9ZoNTJC0ndd/QOV9XiPea0hDai/PKXIFcEE0rkIIIYQQNr++aGC9\nAFwsaTlpTsitvt4AvBF0JenLCLqAhWZ2H2kOTklSF+nhAFf6fuOBSyUtJQ3dGt5ATC+S5hz14EP6\nriU18rqAm7LxZov6z2mkBxcsBb4FrKhSptr+ZVeTHgSyTOkx3VdVFvAHJdzWy/kAXE56eMVCf4jC\nJOAQYIGfx09Iwy4BbgcekjTX8z+ZNLftcWB5jyMnBwF/qRHDROAkSctIjdiDfX35Ws8hDVt7ysvM\nAHb2bcuBIcBq84dR5JQfkj1mlqSR3tPUg6SfSnoN2FHpce0/8U37U/thIhOVHnm+lDSv7sEqZbLx\nXAXcImkBqTcrT9498WvSdVjs98StVPQ2552rpL2BHwEHZx7M8Z2KYkOBl3uJKxRSanYALaLU7ABa\nRKnZAbSIUrMDaBGlZgfQIkrNDqAlxPdg1U9pqkz/4vOddjezK2sWDgBImkV6IENvDYaWI+lO4HIz\nq9V43Cb4HMRlwNlmtjKnjPXhCMwWViKGlxRRIvJURInIUxElIk9FlIg8FVFiY55Ef/xMXESpVIph\ngjkkYWY9RlX11wbW/sBvgLXW/buwQthmSTqQNBRzGXC+5fzyRwMrhBBC/xMNrFC/aGCFEApJDawQ\nQgih/xg+vI01a15pdhihxeQ1sPriKYIhhBYTf3ipLYZMFBN5KibyVEzkqZjIUzGRp2IiT/Xri4dc\nhBBCCCGEEEK/EEMEQwjdSMqbnhVCCCGEEFzeEMHowQohhBBCCCGEzSQaWCGE0ID4XpBiIk/FRJ6K\niTwVE3kqJvJUTOSpftHACiGEEEIIIYTNJOZghRC6iTlYIYQQQgi1xWPaQwiFST3+rQghhH5l+N7D\nWbN6TbPDCCG0oOjBCiF0I8mY1OwoWsAqYL9mB9ECIk/FRJ6K6cs8TWrd7wSM7y0qJvJUTOQpXzxF\nMIQQQgghhBC2sC3awJLUJunZnG2dkkZvyfrzSNpH0mJJD2bWrWpGLHkknShpaoFydcct6TJJO+Rs\nO1/SLb7cIWl8jWOdL6mjRpkP6o2xlvIx/R7rLFD+Gkl/kvR+weP3mn/ffl/xiEHSIElz/N4bJ+l4\nSc/5+4Pyflcy+xc919GSlkl6UdIvMusP9Pqm1xN3yBG9DcVEnoqJPBUTeSokehuKiTwVE3mqX1/0\nYG2N/etnAg+b2Zcz67bGOIvE1EjcE4HBDezXaAxbIreWs5xnFnDkJtTRyPZKowEzs9FmNgM4D7jO\nzEYD6woer0iZfwYuMLMDgQMlnUqq+EUz+xxwiKT4mBJCCCGEsAX0RQNre0l3S1ou6d5qPSeSzvG/\nuC+TdL2vGyBpqq9bKukyX7+/9wIskfRMgx8UdwPerFj3Viae8V5nl6Tf+rqpkm6WNE/Sy5LG+vqd\nJD3isSyVdIavb5O0wvdbKWmapFN8/5WSPu/lBku6Q9J8SYskne5hfAS8V+Bc3vLjTPZ4F0ta7ccc\nLOl+X7/Me00uAUYBnZLm+r7f9pjmA8dljr2W9MG/N+u8HJL2kjTTr02XpGPKKc3k9gpJC7xMh6+b\nImlCpkyHpO/nla/wCfB2rSSZ2QIze6NWuYwN+ffeqnJuF0naycsMkTTDr/NdmfhXSRrmy0co9dbu\nCdwFHOnHuQj4OnB1dl/fZ4CkGyQ97ed9YdFzlTQCGGJmC33VnaQ/KGS9QfodCJtiq+rz3opFnoqJ\nPBUTeSokvreomMhTMZGn+vXFUwQ/A3zbzOZLugOYAPysvFHSSOB64HDgXWCON1JWA3ub2d94uV18\nl2mkv/rPkjSIxhqJA4H12RVmdrTXczDwI+BYM3tHUvaD6AgzO07SZ0k9IjOB/wLONLO1knYH5vs2\ngP2Bs8xsuaRngG/6/md4HWOBvwfmmtkFknYFFkh6xMyeAp7ymI4AvmtmF1WeSDluM+sAOvwY/w78\nE/Al4HUz+6ofZ4iZfSDpcqDdz28EMImU//eBErDYj3lTrUSa2b2Zt7cAJTMbK0nAzuViXv8pwAFm\ndpRvnyXpeGA68Avgl17+68CYvPJm9gTeaDOz1cDZfvyRwO3l890U2fwDPwAmmNlTkgaTrjnAYcDB\nwBpgnqQvmNmT9OxlMjN7S9LfAT8ws3Ij/FjgPjObKaktU/4C4F0zO9rv8XmSHjazVwuc696k352y\n1b4uaz3pdyBfdiDivsSwnBBCCCH0e6VSqVCDsy8aWH8ys/m+fDdwCZkGFmnYVqeZvQ0gaRpwAnAN\nsJ+km4E/AA9L2hkYZWazAMzso3qD8Q/qh3os1ZwMzDCzd7yOdzPbfu/rVkjaq3xIYIqkE0gfXEdl\ntq0ys+W+/DzwiC8/S/rYCjAGOF3SD/39IGAfYGW5UjNbBPRoXOW4G7jJzLokrQX+QdIU4AFvmJRj\nLvcqHU33/E8HDihYV6WTgb/1mA2onHs1BjhF0mKvfydSA2qqpD29sbcX8LaZvS5pYrXywBNUYWZ/\nBja5cVXFPODnfm/O9NgAFnidSFpCuqZPkumxa9AY0jC+cf5+F9J5v1ousInnupr0O/BMbomTGjxy\nfxKNzmIiT8VEnoqJPBUSc2aKiTwVE3naqL29vVs+Jk+eXLVcXzSwevw1v0qZHh9IzexdSYcCpwLf\nA8aR5g71+uHVh5pd6PWcZmZrMtsGAH8EPgQeqOMcyj6sEvN5wB7A4Wa2XumhEztUKb8+8349G3Mv\nUi/XSw3E042kSaQG7Z0AZvaS0oNETgOu8Z6xa6rtuql1u1rzgwRMMbPbq2ybQbrGI0g9WrXK99mc\nOTP7qaT7ga+QepPG+Kbs9f2Ejdf0Yzb2rFZ9mEgNAi4xszkN7Ps68NeZ95/ydVm/AmZLOsrMvttA\nHSGEEEIIIUdfzMFqk3S0L58LPF6xfQFwgqRhkgYC5wCP+XC7gWb2O+DHwGgzWwu8JulrsOGpbDtm\nD2ZmvzSzw/1BAmsqtq03s31Jf7n/Rk68jwLjMnNohuaUKzdKdgXe9MbVSUBblTK9mQ1cumEH6bAC\n+/QMJs3d+iJwWWbdSGCdmd0D3Eh6yAKkoYDlIZdPk/I/VNL2pEZOteNfnJ0nlWMuaQhoeR7RkPLu\n/nM28J3yHCZJo3xuEsC9wDeBs0iNrbzye1Qcs17d9pN0pqTret1B+rSZPW9mNwALgYNq1LEKOMKX\nz2ogxtnABEnbef0HVN7nefyef09SeVjleODfKopdQXoIRjSuNkXMBSkm8lRM5KmYyFMhMWemmMhT\nMZGn+vVFA+sF4GJJy0kT62/19QYbPhBeSZr70wUsNLP7SPNGSpK6SA8HuNL3Gw9cKmkpaejW8AZi\nehEYVm2DD+m7ltTI6wLK85DyeuKmkR5csBT4FrCiSplq+5ddTXoQyDKlx3RfVVnAH5RwWy/nA3A5\n6eEVC/0hCpOAQ0hzurqAn5CGXQLcDjwkaa7nfzJp7tjjwPIeR04OAv5SI4aJwEmSlpEasQf7+vK1\nngPcAzzlZWbg87Q870OA1eWHUeSUH5I9Zpakkd7T1IOkn0p6DdhR6XHtP/FN+1P7YSITJT3r1/gj\n4MEqZbLxXAXcImkBqTcrT9498WvSdVjs98StVPQ293auwMXAHaT7/CUze6hi+1Dg5V7iCiGEEEII\nDVKrfkv5pvD5Trub2ZU1CwcAJM0CxppZbw2GliPpTuByM6vVeNwmeK/WMuBsM1uZU8aY1KdhhRDC\n1mcS9MfPSCGE4iRhZj1GVfXXBtb+wG+AtRXfhRXCNkvSgaShmMuA8y3nl19S//tHIYQQKgzfezhr\nVq+pXTCE0G9FAyuEUIikvLZXyCiVSvFkpQIiT8VEnoqJPBUTeSom8lRM5ClfXgOrL+ZghRBCCCGE\nEEK/ED1YIYRuogcrhBBCCKG26MEKIYQQQgghhC0sGlghhNCA+F6QYiJPxUSeiok8FRN5KibyVEzk\nqX7RwAohhBBCCCGEzSTmYIUQuok5WCGEEEIIteXNwdquGcGEELZu6fuIQwihfxk+vI01a15pdhgh\nhBYXQwRDCFVYvGq+OreCGFrhFXmKPLVOnt5441W2BTFnppjIUzGRp/pFAyuEEEIIIYQQNpPN0sCS\n1Cbp2ZxtnZJGb4566iVpH0mLJT2YWbeqGbHkkXSipKkFyq3yn7m5rig/RNJrkm7JHkPSsDpiq5kr\nv7779LL9fEn/WLTOgnGdXz4vSR2Sxtcof6SkLn8tlfSNAnVMlXRCje1j64z7eEnP+T35V5JulPSs\npJ/6eXy/xv5FzvWLkp7x81wo6aTMth9IeqHI+Yci2psdQItob3YALaK92QG0iPZmB9AS2tvbmx1C\nS4g8FRN5qt/mnINlm/FYm8uZwMNmdmVm3dYYZ5GYLGc5z9XAYw3Usynlt/RxGvUscISZrZc0AnhO\n0v80s0/6OI7zgOvM7B4ASRcCQ83MJHVspjreAr5qZmsk/TdgNvApADO7SdITwI3A9M1UXwghhBBC\nyNicQwS3l3S3pOWS7pW0Q2UBSedIWuav633dAO8NWOZ/db/M1+8vaY6kJf4X+f0aiGk34M2KdW9l\n4hnvdXZJ+q2vmyrpZknzJL1c7qWQtJOkRzK9A2f4+jZJK3y/lZKmSTrF918p6fNebrCkOyTNl7RI\n0ukexkfAewXO5a3KFZJuz/TMvCnp//H1RwB7AQ9X7gJc6vUvlXRg5tz+xa/BEkn/I6/OKv4CfOLH\n+ZIfe4mkOVXi3UPS/5T0tL+OVbJK0i6Zci9K2rNa+Sr1rwXW9Ragmf2Xma33tzsC7xVoXL1LujZI\nut57npZIuiFT5sQq98mJku7LnMs/+n12AfB14GpJd0n6N2BnYJGkcRV5+rSkB70H6rHydQI+KHCu\nS81sjS8/D+wgaftMkTXArjXOPRRSanYALaLU7ABaRKnZAbSIUrMDaAkxZ6aYyFMxkaf6bc4erM/w\n/7N372F2VHW+/98fMokRAgkhkEA0F+IZJA6JSUBEEVoBHWYGRBQU5aIPjyAXQS4eGUZJIojMIJwT\n9ScOwsQIkesAAyJDQLujB4hAroSEIBAFZBJACJIZBKG/vz9q7VC9u/betTsddnfyeT3Pfrp21aqq\n7/pWdbJXr7VqwxciYoGkK4GTgUsrGyXtDFwETCH78HpXaqQ8DYyOiEmpXOWD9lyyv/bfKmkQPWsM\nDgA68ysiYu90nonAucA+EfGipGG5YqMi4oOSdgduBW4C/gwcFhHrJe0ALEjbACYAn4yIFZIeBD6T\n9j80neNw4J+AX0TE8ZKGAvdLujsi7gPuSzFNA06MiBOqK1KJu2rdF9N+Y4A7gNmSBHyHrLfkoIKc\nPBsR0ySdBJwNnAB8A1iXuwZDa52zIIZPpX1GAJcD+0bEk1X5rJgFXBoR90p6J3BnREwmIQquAAAg\nAElEQVSUdAvwCWCOpPcBv4uI5yTNrS4PTKw6/yWVZUknZqvi8uoTp+P+GzAe+GyJep2R9htOdt3f\nnd5vlytWdJ9AQY9dRFwpaV/gtoi4KR3rTxExNS3ne7AuJ7sPHk9xXwYcEBH536eadc2V+RSwKCL+\nklvdSanf+xm55TY8LMfMzMy2dB0dHaUanL3ZwHoyIhak5auBL5NrYAF7Ae0R8QJA+vC8H3ABMF7S\nLODnwDxJQ4BdIuJWgIh4rdlgUkNjcoqlyEeAGyLixXSOdbltt6R1KyXtVDkk8G1l83I6gV1y21ZH\nxIq0/DBwd1p+CBiXlj8KHCLpq+n9IGAMsKpy0ohYSNbgaaaeg4EbgFMj4mlJpwC3R8QzWQqoft72\nzennQrJGDcCBwIZ5ORFRpket2vuB+RHxZDrGuoIyBwK7p2sDMETS1sD1wHnAHOAzvDl8rVb5QhHx\nr3W23Q/8jaTdgDsltUfEn0rU6yXgFUlXALcDP8ttK7pPNoqkbYAPADfk6j2wuly9uqbjvAf4Nt0b\n2c8DO0oaVuMaJTPKB73Famt1AP1EW6sD6CfaWh1AP9HW6gD6Bc+ZKcd5Ksd5elNbW1uXfMycObOw\n3Kacg1U076bbl+tExDpJk4GPAV8CjgC+UlS2y4Gkk4EvpvP8XWVYVNq2FfAE8CrZh+JmvVoQ8+eA\nEcCUNJdnNTC4oHxn7n2+t0BkvVy/7UE89VwG3BgR7en9PsC+KT/bkg3dfDkizq2K9Q16/3vQGn15\nkoC9q3pUAO5TNiR0BNm8uW/WK6+N+I6miFgl6XHgf5E1MhuVfyP1Ih1Adm+empah+D55na69rd2G\nyjawFfBipWerJyS9g6w37ZiI+F1+W0S8Iula4AlJn46IbkM5zczMzKznenMO1lhJlSFlnwV+XbX9\nfmA/ScMlDQCOAuan4XYDIuJm4OvA1IhYDzwl6eMAkgZJenv+YBHxg4iYEhFT842rtK0zIsYBD5Lr\nmanyS+CINAQMSdvXKFf54DyUbHhdp7Ins40tKFPPncBpG3aQ3ltin7pSb9WQiLi4si4ijo6IcRGx\nK9kQwJ/kGle13AWckjtut+F9yuaf7VznGAuAD0kam8oX5XMecHrumJNz224m6/FcketZqVe+NEnj\n0j1Hiu9dwG/T+zlK8+Rq7LsNMCwi/hM4E5hUq2j6+XtgoqSBKY8H1Cif32eDiHgZWJ2G91ViqHXO\noniHkvWyfS3Xo5zfPozsd2K0G1cbq6PVAfQTHa0OoJ/oaHUA/URHqwPoFzxnphznqRznqXm92cB6\nBDhF0gqyh0v8MK0PgNQIOofsX8fFwAMRcRswGuiQtBi4KpUBOJbsgQxLgXuAkT2I6VGg8LHkaUjf\nt8gaeYuBylyeWj1xc4G9UjxHAysLyhTtX3E+WW/SMmWPWf9mdQFJ0yTVnFNT4CxgD2UPuVgkqdHw\nwlqxXQAMV/bI8MVUjcFIQ9UmAC/UPHDE82TDG29Ox7i2oNjpwJ7KHrCxHDgxt+16sl7Ca0uW70bS\niTVysC+wVNKidJ4TcsMDJwHP1DnstsDP0nX/FXBGWl94n0TE0+kcy1NdFlWXqfO+4mjgeGUP1VgO\nHFpdoE5dTyW7Vufl7osRue1DgbURUfdhGWZmZmbWM4po9RO0N50032mHqse0W5PSfJ4vRMTZrY6l\nN0naFrgiIraY74VKwx1nRUTRExkrZaL1T9Y3M2sFsTl/LjKz3iWJiOg2Imlzb2BNAH4MrI+Ig1sc\njllLSTqLrJfw4oi4pk65zfcfBTOzOkaOHMuaNb9rdRhm1k9skQ0sM2uepPC/C411dHT4yUolOE/l\nOE/lOE/lOE/lOE/lOE+11Wpg9eYcLDMzMzMzsy2ae7DMrAv3YJmZmZk15h4sMzMzMzOzTcwNLDOz\nHvD3gpTjPJXjPJXjPJXjPJXjPJXjPDXPDSwzMzMzM7Ne4jlYZtaF52CZmZmZNVZrDtZftSIYM+vb\npG7/VphZHzNy9EjWPL2m1WGYmVkV92CZWReSghmtjqIfWA2Mb3UQ/YDzVE5P8jQDtrT/w/19POU4\nT+U4T+U4T7X5KYJmZmZmZmabWK80sCSNlfRQjW3tkqb2xnmaJWmMpEWS7sitW92KWGqRtL+k2SXK\nrU4/a+a6qvy2kp6S9N38MSQNbyK2hrlK13dMne3HSfpe2XOWjOu4Sr0kTZd0bIPye0lanF5LJX26\nxDlmS9qvwfbDm4x7X0nL0z35NkkXS3pI0j+nepzZYP+GdU3l/lHSbyWtlPTR3PqzJD1Spv5Wgntl\nynGeynGeSvFf0ctxnspxnspxnprXmz1YfXGcwmHAvIg4OLeuL8ZZJqaosVzL+cD8HpxnY8pv6uP0\n1EPAtIiYAnwM+P8kDWhBHJ8DLoyIqRHxKvBFYFJEfK23TiBpd+BIYHfgYOAHShOqIuIS4DjglN46\nn5mZmZl11ZsNrIGSrpa0QtL1kgZXF5B0lKRl6XVRWrdV6g1YlnoXTk/rJ0i6S9ISSQ9K6snf94YB\nz1atey4Xz7HpnIslzUnrZkuaJekeSY9VeikkbSPp7hTLUkmHpvVjU0/BbEmrJM2VdFDaf5WkPVO5\nrSVdKWmBpIWSDklhvAa8VKIuz1WvkPSjXM/Ms5K+kdZPA3YC5lXvApyWzr9U0l/n6vZv6RoskfSJ\nWucs8EfgjXScv03HXiLproJ4R0i6UdJv0msfZVZL2i5X7lFJOxaVLzj/euCVegFGxJ8jojO9fTvw\nUkS80aBe68iuDZIuSj1PSyT9S67M/gX3yf6SbsvV5XvpPjuerOFzvqSrJP0HMARYKOmIqjztKukO\nSQ9Iml+5TsDLjeoKfBy4NiJej4jfAb8F3pfbvgYY2uAYVkaf6gvvw5yncpynUvx9POU4T+U4T+U4\nT83rzacI7gZ8ISIWSLoSOBm4tLJR0s7ARcAUsg+vd6VGytPA6IiYlMpVPmjPJftr/62SBtGzxuAA\noDO/IiL2TueZCJwL7BMRL0oalis2KiI+mHoDbgVuAv4MHBYR6yXtACxI2wAmAJ+MiBWSHgQ+k/Y/\nNJ3jcOCfgF9ExPGShgL3S7o7Iu4D7ksxTQNOjIgTqitSibtq3RfTfmOAO4DZqbfiO2S9JQcV5OTZ\niJgm6STgbOAE4BvAutw1GFrrnAUxfCrtMwK4HNg3Ip6symfFLODSiLhX0juBOyNioqRbgE8AcyS9\nD/hdRDwnaW51eWBi1fkvqSxLOjFbFZdXnzgd99/IBuJ8tkS9zkj7DSe77u9O77fLFSu6T6Cgxy4i\nrpS0L3BbRNyUjvWniJialqfnil9Odh88nuK+DDggIvK/T7XqOpp0PyV/SOsqOinze9+eWx6Hhy+Z\nmZnZFq+jo6NUg7M3G1hPRsSCtHw18GVyDSxgL6A9Il4ASB+e9wMuAMZLmgX8HJgnaQiwS0TcChAR\nrzUbTGpoTE6xFPkIcENEvJjOsS637Za0bqWknSqHBL6tbF5OJ7BLbtvqiFiRlh8G7k7LD5F9PAX4\nKHCIpK+m94OAMcCqykkjYiFZg6eZeg4GbgBOjYinJZ0C3B4Rz2QpoPrJJjennwvJGjUABwIb5uVE\nRJketWrvB+ZHxJPpGOsKyhwI7J6uDcAQSVsD1wPnAXOAzwDXNShfKCL+tc62+4G/kbQbcKek9oj4\nU4l6vQS8IukK4HbgZ7ltRffJRpG0DfAB4IZcvQdWl6tX1waeB3aUNKzGNcp8uIdH35K40VmO81SO\n81SK54KU4zyV4zyV4zy9qa2trUs+Zs6cWViuNxtY1X+1L5p30+0xhhGxTtJksrkxXwKOAL5SVLbL\ngaSTyeawBPB3EbEmt20r4AngVbIPxc16tSDmzwEjgCkR0ansARCDC8p35t7newtE1sv12x7EU89l\nwI0RUelz2AfYN+VnW7Khmy9HxLlVsb5B738PWqMvTxKwd0T8pWr9fcqGhI4gmzf3zXrltRHf0RQR\nqyQ9DvwvskZmo/JvpF6kA8juzVPTMhTfJ6/Ttbe121DZBrYCXqz0bPXAH4B35t6/I60DICJekXQt\n8ISkT0dEt6GcZmZmZtZzvTkHa6ykypCyzwK/rtp+P7CfpOHKHjBwFDA/DbcbEBE3A18HpkbEeuAp\nSR8HkDRI0tvzB4uIH0TElPTAgDVV2zojYhzwILmemSq/BI5IQ8CQtH2NcpUPzkPJhtd1SvowMLag\nTD13Aqdt2EF6b4l96kq9VUMi4uLKuog4OiLGRcSuZEMAf5JrXNVyF7kHHxQN71M2/2znOsdYAHxI\n0thUviif84DTc8ecnNt2M1mP54pcz0q98qVJGpfuOVJ87yKbm4SkOUrz5Grsuw0wLCL+EzgTmFSr\naPr5e2CipIEpjwfUKJ/fZ4OIeBlYLelTuRhqnbPIrcBn0u/MeLK63p871jCy34nRblxtJM+ZKcd5\nKsd5KsVzQcpxnspxnspxnprXmw2sR4BTJK0ge7jED9P6AEiNoHOADmAx8EBE3EY2P6RD0mLgqlQG\n4FiyBzIsBe4BRvYgpkeBwseSpyF93yJr5C0GKnN5avXEzQX2SvEcDawsKFO0f8X5ZL1Jy5Q9Zv2b\n1QUkTZPUbf5QHWcBeyh7yMUiSY2GF9aK7QJguLJHhi8G2qriEtk8sxdqHjjiebLhjTenY1xbUOx0\nYE9lD9hYDpyY23Y9WS/htSXLdyPpxBo52BdYKmlROs8JueGBk4Bn6hx2W+Bn6br/CjgjrS+8TyLi\n6XSO5akui6rL1HlfcTRwvLKHaiwHDq0uUKuu6b6+HlhBNuT25Oj6TaRDgbUR0ehhGWZmZmbWA9qc\nvwU+zXfaISLOaVjYapL0HrIHmJzd6lh6k6RtgSsiYov5Xqg03HFWRBQ9kbFSJpjx1sVkZj00Azbn\n/8PNzPo6SUREtxFJm3sDawLwY2B91XdhmW1xJJ1F1kt4cURcU6fc5vuPgtlmZOTokax5ek3jgmZm\ntknUamD15hDBPiciHo+ID7lxZZY90j7NWazZuMqV9avBq729veUx9IeX87Tp8rQlNq48F6Qc56kc\n56kc56l5m3UDy8zMzMzM7K20WQ8RNLPmSQr/u2BmZmZW3xY5RNDMzMzMzOyt5AaWmVkPeEx6Oc5T\nOc5TOc5TOc5TOc5TOc5T89zAMjMzMzMz6yWeg2VmXXgOlpmZmVljteZg/VUrgjGzvk3q9m+FmVmh\nkSPHsmbN71odhplZn+EhgmZWIPxq+GrvAzH0h5fztLnnae3a3/NW8VyQcpyncpyncpyn5rmBZWZm\nZmZm1kt6pYElaaykh2psa5c0tTfO0yxJYyQtknRHbt3qVsRSi6T9Jc0uUW51+lkz11Xlt5X0lKTv\n5o8haXgTsTXMVbq+Y+psP07S98qes2Rcx1XqJWm6pGMblB8u6ZeSXs7no8E+syXt12D74U3Gva+k\n5emefJukiyU9JOmfUz3ObLB/mboeKOlBSUslPSDpw7ltZ0l6RNKnm4nbamlrdQD9RFurA+gn2lod\nQL/Q1tbW6hD6BeepHOepHOepeb3ZgxW9eKzechgwLyIOzq3ri3GWiSlqLNdyPjC/B+fZmPKb+jg9\n9Wfg68BZLY7jc8CFETE1Il4FvghMioiv9eI5ngP+ISImA58HrqpsiIhLgOOAU3rxfGZmZmaW05sN\nrIGSrpa0QtL1kgZXF5B0lKRl6XVRWrdV6g1Ylv7qfnpaP0HSXZKWpL/Ij+9BTMOAZ6vWPZeL59h0\nzsWS5qR1syXNknSPpMcqvRSStpF0d6534NC0fqyklWm/VZLmSjoo7b9K0p6p3NaSrpS0QNJCSYek\nMF4DXipRl+eqV0j6UYp9saRnJX0jrZ8G7ATMq94FOC2df6mkv87V7d/SNVgi6RO1zlngj8Ab6Th/\nm469RNJdBfGOkHSjpN+k1z7KrJa0Xa7co5J2LCpfcP71wCv1AoyI/4mIe4FXS9SnYh3ZtUHSRann\naYmkf8mV2b/gPtlf0m25unwv3WfHA0cC50u6StJ/AEOAhZKOqMrTrpLuSD1Q8yvXCXi5RF2XRsSa\ntPwwMFjSwFyRNcDQJvJgNXW0OoB+oqPVAfQTHa0OoF/wXJBynKdynKdynKfm9eZTBHcDvhARCyRd\nCZwMXFrZKGln4CJgCtmH17tSI+VpYHRETErlKh+055L9tf9WSYPoWWNwANCZXxERe6fzTATOBfaJ\niBclDcsVGxURH5S0O3ArcBNZL8hhEbFe0g7AgrQNYALwyYhYIelB4DNp/0PTOQ4H/gn4RUQcL2ko\ncL+kuyPiPuC+FNM04MSIOKG6IpW4q9Z9Me03BrgDmC1JwHfIeksOKsjJsxExTdJJwNnACcA3gHW5\nazC01jkLYvhU2mcEcDmwb0Q8WZXPilnApRFxr6R3AndGxERJtwCfAOZIeh/wu4h4TtLc6vLAxKrz\nX1JZlnRitioubxR3iXqdkY45nOy6vzu93y5XrOg+gYIeu4i4UtK+wG0RcVM61p8iYmpanp4rfjnZ\nffB4ysdlwAERkf99alhXSZ8CFkXEX3KrOyn1ez8jt9yGhy+ZmZnZlq6jo6NUg7M3G1hPRsSCtHw1\n8GVyDSxgL6A9Il4ASB+e9wMuAMZLmgX8HJgnaQiwS0TcChARrzUbTGpoTE6xFPkIcENEvJjOsS63\n7Za0bqWknSqHBL6tbF5OJ7BLbtvqiFiRlh8G7k7LDwHj0vJHgUMkfTW9HwSMAVZVThoRC8kaPM3U\nczBwA3BqRDwt6RTg9oh4JksB1c/bvjn9XEjWqAE4ENgwLyciyvSoVXs/MD8inkzHWFdQ5kBg93Rt\nAIZI2hq4HjgPmAN8BriuQflCEfGvPYi7kZeAVyRdAdwO/Cy3reg+2SiStgE+ANyQq/fA6nKN6irp\nPcC36d7Ifh7YUdKwGtcomVE+6C1WW6sD6CfaWh1AP9HW6gD6Bc8FKcd5Ksd5Ksd5elNbW1uXfMyc\nObOwXG82sKr/al8076bbl+tExDpJk4GPAV8CjgC+UlS2y4Gkk8nmsATwd5VhUWnbVsATZEPCbm+i\nDhX5oWSVOD4HjACmRESnsgdADC4o35l7n+8tEFkv1297EE89lwE3RkR7er8PsG/Kz7ZkQzdfjohz\nq2J9g97/HrRGX54kYO+qHhWA+5QNCR1BNm/um/XK6y38jqaIeCP1Ih1Adm+empah+D55na69rd2G\nyjawFfBipWerJyS9g6w37ZiI+F1+W0S8Iula4AlJn46IbkM5zczMzKznenMO1lhJlSFlnwV+XbX9\nfmA/ZU90GwAcBcxPw+0GRMTNZA8imBoR64GnJH0cQNIgSW/PHywifhARU9IDA9ZUbeuMiHHAg+R6\nZqr8EjgiDQFD0vY1ylU+OA8lG17XqezJbGMLytRzJ3Dahh2k95bYp67UWzUkIi6urIuIoyNiXETs\nSjYE8Ce5xlUtd5F78EHR8D5l8892rnOMBcCHJI1N5YvyOQ84PXfMybltN5P1eK7I9azUK99TXa6V\npDlK8+QKC2c9SsMi4j+BM4FJDY77e2CipIEpjwfUKN8tFoCIeBlYnYb3VWKodc6ieIeS9bJ9Ldej\nnN8+jOx3YrQbVxuro9UB9BMdrQ6gn+hodQD9gueClOM8leM8leM8Na83G1iPAKdIWkH2cIkfpvUB\nkBpB55D9L7IYeCAibgNGAx2SFpM98eyctN+xZA9kWArcA4zsQUyPAoWPJU9D+r5F1shbDFTm8tTq\niZsL7JXiORpYWVCmaP+K88l6k5Ype8z6N6sLSJomqZn5Q2cBeyh7yMUiSY2GF9aK7QJguLJHhi+m\naqxKGqo2AXih5oEjnicb3nhzOsa1BcVOB/ZU9oCN5cCJuW3Xk/USXluyfDeSTqyVg9TjeAlwnKQn\nJb07bZoEPFPnsNsCP0vX/VfAGWl94X0SEU+nuixPdVlUXabO+4qjgeOVPVRjOXBoQX1q1fVUsmt1\nXu6+GJHbPhRYGxF1H5ZhZmZmZj2jiFY/QXvTSfOddoiIcxoWtprSfJ4vRMTZrY6lN0naFrgiIraY\n74VKwx1nRUTRExkrZaL1T9Y3s/5DbM6fJczMapFERHQbkbS5N7AmAD8G1ld9F5bZFkfSWWS9hBdH\nxDV1yrmBZWZNcAPLzLZMtRpYvTlEsM+JiMcj4kNuXJllj7RPcxZrNq7eJL/88suvUq+RI/NTkjct\nzwUpx3kqx3kqx3lqXm8/Rc7MNgP+a3RjHR0dfnRtCc5TOc6TmdnmY7MeImhmzZMU/nfBzMzMrL4t\ncoigmZmZmZnZW8kNLDOzHvCY9HKcp3Kcp3Kcp3Kcp3Kcp3Kcp+a5gWVmZmZmZtZLPAfLzLrwHCwz\nMzOzxjwHy8zMzMzMbBNzA8vMupHkl1996jXqHaNa/WuxSXmOQznOUznOUznOUznOU/P8PVhm1t2M\nVgfQD6wGxrc6iH6gl/K0dsbajT+ImZnZW8BzsMysC0nhBpb1OTP8BdhmZta3SJtwDpaksZIeqrGt\nXdLU3jhPsySNkbRI0h25datbEUstkvaXNLtEudXpZ81cV5XfVtJTkr6bP4ak4U3E1jBX6fqOqbP9\nOEnfK3vOknEdV6mXpOmSjm1QfrikX0p6OZ+PBvvMlrRfg+2HNxn3vpKWp3vybZIulvSQpH9O9Tiz\nwf4N65rK/aOk30paKemjufVnSXpE0qebidvMzMzMyuvNOVh98U+LhwHzIuLg3Lq+GGeZmKLGci3n\nA/N7cJ6NKb+pj9NTfwa+DpzV4jg+B1wYEVMj4lXgi8CkiPhab51A0u7AkcDuwMHADyQJICIuAY4D\nTumt823R+tSfavow56kUz3Eox3kqx3kqx3kqx3lqXm82sAZKulrSCknXSxpcXUDSUZKWpddFad1W\nqTdgmaSlkk5P6ydIukvSEkkPSurJKP5hwLNV657LxXNsOudiSXPSutmSZkm6R9JjlV4KSdtIujvF\nslTSoWn92NRTMFvSKklzJR2U9l8lac9UbmtJV0paIGmhpENSGK8BL5Woy3PVKyT9KMW+WNKzkr6R\n1k8DdgLmVe8CnJbOv1TSX+fq9m/pGiyR9Ila5yzwR+CNdJy/TcdeIumugnhHSLpR0m/Sax9lVkva\nLlfuUUk7FpUvOP964JV6AUbE/0TEvcCrJepTsY7s2iDpotTztETSv+TK7F9wn+wv6bZcXb6X7rPj\nyRo+50u6StJ/AEOAhZKOqMrTrpLukPSApPmV6wS83KiuwMeBayPi9Yj4HfBb4H257WuAoU3kwczM\nzMya0JsPudgN+EJELJB0JXAycGllo6SdgYuAKWQfXu9KjZSngdERMSmVq3zQnkv21/5bJQ2iZ43B\nAUBnfkVE7J3OMxE4F9gnIl6UNCxXbFREfDD1BtwK3ETWC3JYRKyXtAOwIG0DmAB8MiJWSHoQ+Eza\n/9B0jsOBfwJ+ERHHSxoK3C/p7oi4D7gvxTQNODEiTqiuSCXuqnVfTPuNAe4AZqfeiu+Q9ZYcVJCT\nZyNimqSTgLOBE4BvAOty12BorXMWxPCptM8I4HJg34h4siqfFbOASyPiXknvBO6MiImSbgE+AcyR\n9D7gdxHxnKS51eWBiVXnv6SyLOnEbFVc3ijuEvU6Ix1zONl1f3d6v12uWNF9AgU9dhFxpaR9gdsi\n4qZ0rD9FxNS0PD1X/HKy++DxlI/LgAMiIv/7VKuuo0n3U/KHtK6ikzK/9+255XH4YQ5FnJNynKdS\n2traWh1Cv+A8leM8leM8leM8vamjo6NUj15vNrCejIgFaflq4MvkGljAXkB7RLwAkD487wdcAIyX\nNAv4OTBP0hBgl4i4FSAiXms2mNTQmJxiKfIR4IaIeDGdY11u2y1p3UpJO1UOCXxb2bycTmCX3LbV\nEbEiLT8M3J2WHyL7eArwUeAQSV9N7wcBY4BVlZNGxEKyBk8z9RwM3ACcGhFPSzoFuD0inslSQPXE\nu5vTz4VkjRqAA4EN83IiokyPWrX3A/Mj4sl0jHUFZQ4Edk/XBmCIpK2B64HzgDnAZ4DrGpQvFBH/\n2oO4G3kJeEXSFcDtwM9y24ruk40iaRvgA8ANuXoPrC63EXV9HthR0rAa1yjz4R4e3czMzGwz1dbW\n1qXBOXPmzMJym3IOVtG8m25P2Ugf8iYDHcCXgB/VKtvlQNLJaWjcIkmjqrZtRTbyf3eyD8XNyg8l\nq8TxOWAEMCUippANPRxcUL4z9z7fWyCyXq4p6TU+Ilax8S4DboyISp/DPsCpkp4g68k6RtKFBXV7\ng95/TH/da5a2753LwZg0fO8+YELqBTsM+Pd65Xs55roi4g2yIXY3Av8A/Gduc9F98jpdf6+6DZVt\nYCvgxTRPq1Lvv2li/z8A78y9f0daB0BEvAJcCzwhqaiH08ry3KJynKdSPMehHOepHOepHOepHOep\neb3ZwBorqTKk7LPAr6u23w/sp+yJbgOAo4D5abjdgIi4mexBBFMjYj3wlKSPA0gaJOnt+YNFxA/S\nh8+pEbGmaltnRIwDHiTXM1Pll8ARaQgYkravUa7ywXko2fC6TkkfBsYWlKnnTuC0DTtI7y2xT12p\nt2pIRFxcWRcRR0fEuIjYlWwI4E8i4twGh7qL3IMPiob3KZt/tnOdYywAPiRpbCpflM95wOm5Y07O\nbbuZrMdzRa5npV75nupyrSTNUZonV1g461EaFhH/CZwJTGpw3N8DEyUNTHk8oGwsABHxMrBa0qdy\nMdQ6Z5Fbgc+k35nxwLvIfvcqxxpG9jsxOiK6zZMzMzMzs43Tmw2sR4BTJK0ge7jED9P6AEiNoHPI\neqoWAw9ExG1k80M6JC0GrkplAI4leyDDUuAeYGQPYnoUKHwseRrS9y2yRt5ioDKXp1ZP3FxgrxTP\n0cDKgjJF+1ecT/YgkGXKHrP+zeoCkqZJamb+0FnAHrmevEbDC2vFdgEwXNkjwxcDbVVxiWye2Qs1\nDxzxPNnwxpvTMa4tKHY6sKeyB2wsB07MbbuerJfw2pLlu5F0Yq0cKHvk/CXAcSw5BAgAACAASURB\nVJKelPTutGkS8Eydw24L/Cxd918BZ6T1hfdJRDyd6rI81WVRdZk67yuOBo5X9lCN5cChBfUprGu6\nr68HVpANuT05un550FBgberJso3huUXlOE+leI5DOc5TOc5TOc5TOc5T8zbrLxpO8512iIhzGha2\nmiS9h+wBJme3OpbeJGlb4IqI2GK+Fyo9NGNWRBQ9kbFSxl80bH3PDH/RsJmZ9S2q8UXDm3sDawLw\nY2B91XdhmW1xJJ1F1kt4cURcU6fc5vuPgvVbI0ePZM3TaxoX7Kc6Ojr8V+ISnKdynKdynKdynKfa\najWwevshB31KRDwOfKjVcZj1BemR9pc0LIh7CsrwfzjlOE9mZral2ax7sMyseZLC/y6YmZmZ1Ver\nB6s3H3JhZmZmZma2RXMDy8ysB/y9IOU4T+U4T+U4T+U4T+U4T+U4T81zA8vMzMzMzKyXeA6WmXXh\nOVhmZmZmjXkOlpmZmZmZ2SbmBpaZdSPJL7+2+NeoUePest85z3Eox3kqx3kqx3kqx3lq3mb9PVhm\n1lMeIthYB9DW4hj6gw76a57Wru026sPMzKwhz8Eysy4khRtYZgDyl26bmVlNUh+agyVprKSHamxr\nlzT1rY4pnXuMpEWS7sitW92KWGqRtL+k2SXKrU4/a+a6qvy2kp6S9N3cunZJYxrsN1vSfg3iva3R\n+ZuRP6ak4yRNL7HPP0t6SNIySUeWKD9d0rENtp/ZZNy7SVosaaGk8ZJOk7RC0lWpHt9rsH/Dukqa\nLOneVNcl+bpKOkrSI5LOaCZuMzMzMyuvlXOw+uKfBQ8D5kXEwbl1fTHOMjFFjeVazgfm9yycpmLZ\nFMese3xJfwe8F5gEvB84W9KQTRBTI4cBN0TEtIhYDZwEHBgRx6TtzV7XIv8NHBMRewAHA/9X0nYA\nEXENsD/gBlav6Gh1AP1ER6sD6Bc8x6Ec56kc56kc56kc56l5rWxgDZR0dfoL/vWSBlcXSH9xX5Ze\nF6V1W6Vek2WSlko6Pa2fIOmu9Ff7ByWN70FMw4Bnq9Y9l4vn2HTOxZLmpHWzJc2SdI+kxyQdntZv\nI+nuFMtSSYem9WMlrUz7rZI0V9JBaf9VkvZM5baWdKWkBanH45AUxmvASyXq8lz1Ckk/SrEvlvSs\npG+k9dOAnYB5Vbv8EXijwXnWpZiQtFeqx5IU9zZV5y+sk6T7JO2eK9cuaWqdHOS9AqxvEONE4FeR\n+R9gGfC3DfZ5OR2b1NP0cKrXT3Nl3pNifUzSl1PZLj2Gks5KvV0HA18BTpL0C0mXAbsCd1Tu4dw+\nIyTdKOk36bVP2bpGxGMR8Xha/i+y+3nH3Pa1wNAGdTczMzOznoqIt/wFjAU6gfen91cCZ6bldmAq\nsDPwe2A4WUPwF8Chadu83LG2Sz8XAIem5UHA4B7ENRP4So1tE4FHgO3T+2Hp52zgurS8O/DbtDwA\nGJKWd8itH0vWIJmY3j8IXJmWDwVuSsvfAj6blocCq4C3V8U0Dbi8RK6XVa0bAzwMvANQyvkuwHHA\nd3t4TQcCjwNT0/sh6brtD9xar07A6cCMtH4UsLJB+Q3HrIrhkMpxqtYfBPw67TsixXlGE3X7AzCw\n6n6bDvw/sgfF7AA8n655l3wDZwHn5fY5M7ftidz9tCH3wFzgA2n5ncCKsnWtKvM+4OGC9S832C9g\neu7VHhB++bUFvggzM7OK9vb2mD59+oZX+n+C6lcrnyL4ZEQsSMtXA18GLs1t3wtoj4gXACTNBfYD\nLgDGS5oF/ByYl4Z77RIRtwJExGvNBiNJwOQUS5GPkA3vejGdY11u2y1p3UpJO1UOCXxb2fykTmCX\n3LbVEbEiLT8M3J2WHwLGpeWPAodI+mp6P4isYbSqctKIWAic0GQ9BwM3AKdGxNOSTgFuj4hnshTQ\n08dm7QY8ExGLUmzr0/nyZWrV6Qay3rMZwJHAjQ3KF4qI24Bu870i4i5JewH3kvXo3Evjnrm8pcBP\nJd1CutbJ7RHxOvBHSWuBkU0cE7JcF+X7QGB3vZm8IZK2jqz3Dahd1w0HlnYGfgIcU7D5BUkTIvV0\nFZvRMHgzMzOzLUlbWxttbW0b3s+cObOwXF+ag1X9Hgo+fKaGzWSygf1fAn5Uq2yXA0knp6FxiySN\nqtq2FbCarAfq9lLRd/VqQcyfI+stmRIRU8g+2A8uKN+Ze9/Jm4/OF/DJiJiSXuMjYhUb7zLgxoho\nT+/3AU6V9ATwHeAYSRf28NiNGmeFdYqIZ4DnJe0BfBq4LrdPr+QgIi5Mx/gY2X3/aBO7/z3wfbLe\n0wfS/QLdr+NfAa+T9WRVdBv6WoKAvXP1HpNvXDXcWdoW+BnwjxHxQEGRWcASSZ/vQWy2QUerA+gn\nOlodQL/gOQ7lOE/lOE/lOE/lOE/Na2UDa6ykvdPyZ8mGcOXdD+wnabikAcBRwHxJOwADIuJm4Otk\nQ9LWA09J+jiApEGS3p4/WET8IH1YnRoRa6q2dUbEOLLhep+uEe8vgSMkDU/n2L5GuUojYyjwbER0\nSvow2dCx6jL13AmctmEH6b0l9qkr9VYNiYiLK+si4uiIGBcRuwJnAz+JiHML9p1TmR9WwypgVJrP\nhaQh6brl1avTdcD/JhuCt7xE+dKUzdurXLdJwB6k+WaSLqzcNzX2FTAmIuYD5wDbkQ1/rGUtsKOk\n7SW9DfiHHoQ8j2zYZCWGyWV3lDSQrJdtTvodKXIu8K6I+HEPYjMzMzOzOlrZwHoEOEXSCrKHS/ww\nrQ+A1Ag6h+zPn4uBB9KwqNFAh6TFwFWpDMCxwGmSlgL30PxwLch6NYYXbUhD+r5F1shbDFySjzdf\nNP2cC+yV4jkaWFlQpmj/ivPJHgSyLD004ZvVBSRNk3R5nfpUOwvYI9eT18zwwknAM7U2RsRfyBqn\n35e0hKyR8LaqYvXq9O907726oE75biQdImlGwaaBwK8lLSe7z46OiM60bQ9gTcE+FQOAq9N1XAjM\niog/FZSr3LevpzgfIGsgriwo22WfAqcDeyp7OMpy4MTqAnXqeiSwL/D53HWeVFVmUGQPu7CN0tbq\nAPqJtlYH0C/kh5xYbc5TOc5TOc5TOc5T8/xFwzlprs8OEXFOw8JbkDTk7IqIqNW7129JuiO6PpZ/\ns5bmAS6NiJ3rlInabT+zLYm/aNjMzGpTX/qi4T7sJuCDyn3RsEFEvLw5Nq4AtrDG1VFkPYv/UqK0\nX35t8a+RI/Mjuzctz3Eox3kqx3kqx3kqx3lqXiufItjnpKeqfajVcZhtCpF90fA1Jctu4mj6v46O\nDg+bKMF5MjOzLY2HCJpZF5LC/y6YmZmZ1echgmZmZmZmZpuYG1hmZj3gMenlOE/lOE/lOE/lOE/l\nOE/lOE/NcwPLzMzMzMysl3gOlpl14TlYZmZmZo15DpaZmZmZmdkm5gaWmXUjyS+/Nslr1DtGtfr2\n7pM8x6Ec56kc56kc56kc56l5/h4sM+tuRqsD6AdWA+NbHUQ/UJWntTPWtiwUMzOzt4LnYJlZF5LC\nDSzbZGb4i6zNzGzzIPWhOViSxkp6qMa2dklT3+qY0rnHSFok6Y7cutWtiKUWSftLml2i3Or0s2au\nq8pvK+kpSd/NrWuXNKbBfrMl7dcg3tsanb8Z+WNKOk7S9BL7/LOkhyQtk3RkifLTJR3bYPuZTca9\nm6TFkhZKGi/pNEkrJF2V6vG9BvuXretxkh6VtCpfB0lHSXpE0hnNxG1mZmZm5bVyDlZf/BPmYcC8\niDg4t64vxlkmpqixXMv5wPyehdNULJvimHWPL+nvgPcCk4D3A2dLGrIJYmrkMOCGiJgWEauBk4AD\nI+KYtL3Z69qNpO2B84C9gL2B6ZKGAkTENcD+gBtYvaFP/emlD3OeSvEch3Kcp3Kcp3Kcp3Kcp+a1\nsoE1UNLV6S/410saXF0g/cV9WXpdlNZtlXpNlklaKun0tH6CpLskLZH0oKSezI4YBjxbte65XDzH\npnMuljQnrZstaZakeyQ9JunwtH4bSXenWJZKOjStHytpZdpvlaS5kg5K+6+StGcqt7WkKyUtSD0e\nh6QwXgNeKlGX56pXSPpRin2xpGclfSOtnwbsBMyr2uWPwBsNzrMuxYSkvVI9lqS4t6k6f2GdJN0n\nafdcuXZJU+vkIO8VYH2DGCcCv4rM/wDLgL9tsM/L6diknqaHU71+mivznhTrY5K+nMp26TGUdFbq\n7ToY+ApwkqRfSLoM2BW4o3IP5/YZIelGSb9Jr32aqOvHyP5I8FJErCO7phvqGhFrgaENjmFmZmZm\nPdTKh1zsBnwhIhZIuhI4Gbi0slHSzsBFwBSyD/F3pUbK08DoiJiUym2XdpkLXBgRt0oaRM8ajwOA\nzvyKiNg7nWcicC6wT0S8KGlYrtioiPhgaiTcCtwE/Bk4LCLWS9oBWJC2AUwAPhkRKyQ9CHwm7X9o\nOsfhwD8Bv4iI41MPxP2S7o6I+4D7UkzTgBMj4oTqilTirlr3xbTfGOAOYLYkAd8BPgccVFX+U40S\nFhFnpGMOBK4FjoiIRamH6JWq4oV1Svt9GpghaVTK5yJJ36pRPn/+6yvLqQE2LSJmVJ13KXCepEuB\nbYAPAw83qNelubdfA8ZFxF9y9xtk93AbWYNllaQfVHbvfri4Q9IPgZcrx5b0MaAt3U/H5crPAi6N\niHslvRO4E5hYsq6jgady7/+Q1uU1/t1ozy2Pww9zKOKclOM8ldLW1tbqEPoF56kc56kc56kc5+lN\nHR0dpXr0WtnAejIiFqTlq4Evk2tgkQ1xao+IFwAkzQX2Ay4AxkuaBfwcmJc+zO8SEbcCRMRrzQaT\nGhqTUyxFPkI2vOvFdI51uW23pHUrJe1UOSTwbWXzkzqBXXLbVkfEirT8MFBpNDxE9nEW4KPAIZK+\nmt4PAsYAqyonjYiFQLfGVYN6DgZuAE6NiKclnQLcHhHPZCmg20S9knYDnomIRSm29el8+TK16nQD\nWU/LDOBI4MYG5QtFxG1At/leEXGXpL2Ae8l6KO+lcc9c3lLgp5JuIV3r5PaIeB34o6S1wMgmjglZ\nrovyfSCwu95M3hBJW6feN6B2XUt6QdKEiHi8ZokP9/DIZmZmZpuptra2Lg3OmTNnFpbrS3OwiuaW\ndPvwmRo2k4EO4EvAj2qV7XIg6eQ0NG5R6iXJb9uKbKbA7sDtpaLv6tWCmD8HjACmRMQUsg/2gwvK\nd+bed/Jmo1dkvVxT0mt8RKxi410G3BgRlT6KfYBTJT1B1pN1jKQLe3jsRo2zwjpFxDPA85L2IOvJ\nui63T6/kICIuTMf4GNl9/2gTu/898H1gKvBAul+g+3X8K+B1sp7Qim5DX0sQsHeu3mPyjasG/kDX\nRug70rq8WcASSZ/vQWxW4blF5ThPpXiOQznOUznOUznOUznOU/Na2cAaK6kyjO2zwK+rtt8P7Cdp\nuKQBwFHA/DTcbkBE3Ax8HZiaekuekvRxAEmDJL09f7CI+EH6sDo1ItZUbeuMiHHAg2Qf8Iv8EjhC\n0vB0ju1rlKs0MoYCz0ZEp6QPA2MLytRzJ3Dahh2k95bYp67UWzUkIi6urIuIoyNiXETsCpwN/CQi\nzi3Yd47S/LAaVgGj0rBFJA1J1y2vXp2uA/43sF1ELC9RvjRl8/Yq120SsAdpvpmkCyv3TY19BYyJ\niPnAOcB2QL0HZKwFdpS0vaS3Af/Qg5DnARvmZUma3MS+dwIHSRqa7tGD0rq8c4F3RcSPexCbmZmZ\nmdXRygbWI8ApklaQPVzih2l9AKRG0DlkPVWLgQfSsKjRQIekxcBVqQzAscBpkpYC99D8cC3IejWG\nF21IQ/q+RdbIWwxcko83XzT9nAvsleI5GlhZUKZo/4rzyR4Esiw9NOGb1QUkTZN0eZ36VDsL2CPX\nk9fM8MJJwDO1NkbEX8gap9+XtISskfC2qmL16vTvdO+9uqBO+W4kHSJpRsGmgcCvJS0nu8+OjojK\nXLs9gDUF+1QMAK5O13EhMCsi/lRQrnLfvp7ifICsYbOyoGyXfQqcDuyp7OEoy4ETqwvUqmsawno+\n2R8LfgPMrBrOCjAoPezCNobnFpXjPJXiOQ7lOE/lOE/lOE/lOE/N8xcN56S5PjtExDkNC29BJG0L\nXBERtXr3+i1Jd1Q9ln+zluYBLo2IneuU8RcN26Yzw180bGZmmwfV+KJhN7ByJE0Afgys35I+dNuW\nQdJRZE9EnBMR/6dOOf+jYJvMyNEjWfN0vU7jLVNHR4f/SlyC81SO81SO81SO81RbrQZWK58i2Oek\np6p9qNVxmG0K6YuGrylZdhNH0//5P5xynCczM9vSuAfLzLqQFP53wczMzKy+Wj1YrXzIhZmZmZmZ\n2WbFDSwzsx7w94KU4zyV4zyV4zyV4zyV4zyV4zw1zw0sMzMzMzOzXuI5WGbWhedgmZmZmTXmOVhm\nZmZmZmabmBtYZtaNJL/88mszeo0aNa7V/6z0Cs8FKcd5Ksd5Ksd5ap6/B8vMCniIYGMdQFuLY+gP\nOnCeyuhgU+Zp7dpuI1jMzGwT8RwsM+tCUriBZba5kb9A3Mysl0l9aA6WpLGSHqqxrV3S1Lc6pnTu\nMZIWSbojt251K2KpRdL+kmaXKLc6/ayZ66ry20p6StJ3c+vaJY1psN9sSfs1iPe2RudvRv6Yko6T\nNL3EPm+ka7tY0i0lyk+XdGyD7Wc2Gfdu6fwLJY2XdJqkFZKuSvX4XoP9G9ZV0mRJ90p6SNISSUfm\nth0l6RFJZzQTt5mZmZmV18o5WH3xT2mHAfMi4uDcur4YZ5mYosZyLecD83sWTlOxbIpjljn+f0fE\n1IiYEhGHbYJ4yjgMuCEipkXEauAk4MCIOCZtb/a6Fvlv4JiI2AM4GPi/krYDiIhrgP0BN7B6RUer\nA+gnOlodQD/R0eoA+gXPBSnHeSrHeSrHeWpeKxtYAyVdnf6Cf72kwdUF0l/cl6XXRWndVqnXZJmk\npZJOT+snSLor/dX+QUnjexDTMODZqnXP5eI5Np1zsaQ5ad1sSbMk3SPpMUmHp/XbSLo7xbJU0qFp\n/VhJK9N+qyTNlXRQ2n+VpD1Tua0lXSlpQerxOCSF8RrwUom6PFe9QtKPUuyLJT0r6Rtp/TRgJ2Be\n1S5/BN5ocJ51KSYk7ZXqsSTFvU3V+QvrJOk+SbvnyrVLmlonB3mvAOsbxAjQ7ASEl9OxST1ND6d6\n/TRX5j0p1sckfTmV7dJjKOms1Nt1MPAV4CRJv5B0GbArcEflHs7tM0LSjZJ+k177lK1rRDwWEY+n\n5f8iu593zG1fCwxtMhdmZmZmVlZEvOUvYCzQCbw/vb8SODMttwNTgZ2B3wPDyRqCvwAOTdvm5Y61\nXfq5ADg0LQ8CBvcgrpnAV2psmwg8Amyf3g9LP2cD16Xl3YHfpuUBwJC0vENu/ViyBsnE9P5B4Mq0\nfChwU1r+FvDZtDwUWAW8vSqmacDlJXK9rGrdGOBh4B1kDY92YBfgOOC7PbymA4HHganp/ZB03fYH\nbq1XJ+B0YEZaPwpY2aD8hmNWxXBI5TgF215Lub4X+HiTdfsDMLDqfpsO/D+yB8XsADyfrnmXfANn\nAefl9jkzt+2J3P20IffAXOADafmdwIpm6por8z7g4YL1LzfYL2B67tUeEH755Ve/fhFmZrZx2tvb\nY/r06Rte6d9Wql+tfIrgkxGxIC1fDXwZuDS3fS+gPSJeAJA0F9gPuAAYL2kW8HNgnqQhwC4RcStA\nRLzWbDCSBExOsRT5CNnwrhfTOdbltt2S1q2UtFPlkMC3lc1P6gR2yW1bHREr0vLDwN1p+SFgXFr+\nKHCIpK+m94PIGkarKieNiIXACU3WczBwA3BqRDwt6RTg9oh4JktB0z09FbsBz0TEohTb+nS+fJla\ndbqBrPdsBnAkcGOD8oUi4jag1nyvsRHxX6ln85eSlkU2TK+MpcBPlc3dys/fuj0iXgf+KGktMLLk\n8SpEcb4PBHbXm8kbImnriPifSoEGdUXSzsBPgGMKNr8gaUKknq5iMxoGb2ZmZrYlaWtro62tbcP7\nmTNnFpbrS3Owqt9DwYfP1LCZTDZg/UvAj2qV7XIg6eQ0NG6RpFFV27YCVpP1QN1eKvquXi2I+XPA\nCGBKREwhG6o1uKB8Z+59J28+Ol/AJyObMzQlIsZHxCo23mXAjRHRnt7vA5wq6QngO8Axki7s4bEb\nNc4K6xQRzwDPS9oD+DRwXW6fXslBZMPlSI2qDmBKE7v/PfB9st7TB9L9At2v418Br5P1ZFV0G/pa\ngoC9c/Uek29cNdxZ2hb4GfCPEfFAQZFZwBJJn+9BbLZBR6sD6Cc6Wh1AP9HR6gD6Bc8FKcd5Ksd5\nKsd5al4rG1hjJe2dlj8L/Lpq+/3AfpKGSxoAHAXMl7QDMCAibga+TjYkbT3wlKSPA0gaJOnt+YNF\nxA/Sh9WpEbGmaltnRIwjG0L26Rrx/hI4QtLwdI7ta5SrNDKGAs9GRKekD5MNHasuU8+dwGkbdpDe\nW2KfulJv1ZCIuLiyLiKOjohxEbErcDbwk4g4t2DfOZX5YTWsAkal+VxIGpKuW169Ol0H/G+yIXjL\nS5QvTdIwSYPS8gjgg8CK9P7Cyn1TY18BYyJiPnAOsB3Z8Mda1gI7Stpe0tuAf+hByPPIhk1WYphc\ndkdJA8l62eak35Ei5wLviogf9yA2MzMzM6ujlQ2sR4BTJK0ge7jED9P6AEiNoHPI/qy3GHggDYsa\nDXRIWgxclcoAHAucJmkpcA/ND9cCeJRszlc3aUjft8gaeYuBS/Lx5oumn3OBvVI8RwMrC8oU7V9x\nPtmDQJalhyZ8s7qApGmSLq9Tn2pnAXvkevKaGV44CXim1saI+AtZ4/T7kpaQNRLeVlWsXp3+ne69\nVxfUKd+NpEMkzSjYtDvwYLpuvwAujIhH0rY9gDUF+1QMAK5O13EhMCsi/lRQrnLfvp7ifICsgbiy\noGyXfQqcDuyp7OEoy4ETqwvUqeuRwL7A53PXeVJVmUGRPezCNkpbqwPoJ9paHUA/0dbqAPqF/NAc\nq815Ksd5Ksd5ap6/aDgnzfXZISLOaVh4C5KGnF0REbV69/otSXdE18fyb9bSPMClEbFznTJRu+1n\nZv2Tv2jYzKy3qS990XAfdhPwQeW+aNggIl7eHBtXAFtY4+oosp7Ff2l1LJuHjlYH0E90tDqAfqKj\n1QH0C54LUo7zVI7zVI7z1LxWPkWwz0lPVftQq+Mw2xQi+6Lha8qV7unDJM2sLxo5cmzjQmZm1is8\nRNDMupAU/nfBzMzMrD4PETQzMzMzM9vE3MAyM+sBj0kvx3kqx3kqx3kqx3kqx3kqx3lqnhtYZmZm\nZmZmvcRzsMysC8/BMjMzM2vMc7DMzMzMzMw2MTewzMx6wGPSy3GeynGeynGeynGeynGeynGemufv\nwTKzbiR/D5b1npGjR7Lm6TWtDsPMzOwt4TlYZtaFpGBGq6OwzcoM8P81Zma2ufEcLDMzMzMzs02s\nJQ0sSWMlPVRjW7ukqW91TOncYyQtknRHbt3qVsRSi6T9Jc0uUW51+lkz11Xlt5X0lKTv5ta1SxrT\nYL/ZkvZrEO9tjc7fjPwxJR0naXqJfd5I13axpFtKlJ8u6dgG289sMu7d0vkXShov6TRJKyRdlerx\nvQb7l63rcZIelbQqXwdJR0l6RNIZzcRtNfSpfxn6MOepFM9xKMd5Ksd5Ksd5Ksd5al4r52D1xfEi\nhwHzIuKc3Lq+GGeZmKLGci3nA/N7Fk5TsWyKY5Y5/n9HREsa7jmHATdExIUAkk4CDoiIZyQdR/PX\ntRtJ2wPnAVMBAQsl/UdEvBQR10j6JfAA8H82piJmZmZmVqyVQwQHSro6/QX/ekmDqwukv7gvS6+L\n0rqtUq/JMklLJZ2e1k+QdJekJZIelDS+BzENA56tWvdcLp5j0zkXS5qT1s2WNEvSPZIek3R4Wr+N\npLtTLEslHZrWj5W08v9n7+7Dra7q/P8/X5CIiiJqgTpf0JguJyfvQEYdHTyaN6l5MyYVYdpMl9lo\natlopI0eU/Om9MrJ1Ex/yAiaWt6ghiLGIRIJhcNBBckKy5tBLaVB8/68f3+s94bP2WffrH06uDny\nflzXuc7ea63P5/Ne77057HXWWp/jxy2TNFXSgX78Mkm7e7uNJV0vaZ7PeBzuYbwF/CWjLy+VF0j6\nscfeLulFSf/l5aOBDwEzyg75M/Buneus9JiQNMb7scjj3qTs+hX7JOlhSR8ttJslaVSNHBS9Drxa\nJ0ZIg41GrPJz4zNNT3i/biq0+UeP9beSTvG2XWYMJX3dZ7sOAb4K/IekByVdDXwYmF56DxeO2UrS\nTyX92r/2aqCvB5N+SfAXM1tJek0/Uao0sxeAwQ3mIlTSk58w66PIU5aWlpZmh9AnRJ7yRJ7yRJ7y\nRJ4a18wZrB2AfzOzeZKuB04CLi9VStoauBjYjfQh/gEfpDwLbGtmO3u7zfyQqcB3zGyapAH0bPDY\nH+gsFpjZHn6dHYGzgL3M7BVJmxeaDTOzvX2QMA24HXgDOMrMXpW0JTDP6wBGAp8ysyWSHgU+68cf\n4dc4GjgbeNDMvihpMDBf0kwzexh42GMaDZxoZl8q70gp7rKyE/y44cB0YJIkAd8DJgAHlrU/pl7C\nzOxrfs4NgJ8A48xsoaRB+ACloGKf/LjPAK2Shnk+F0q6sEr74vVvLT32AdhoM2utEOqGnuu3gEvM\n7K46/bq88PQbwHZm9nbh/QbpPdxCGrAsk3RV6fDup7Ppkq4BVpXOLelgoMXfT8cX2l8BXG5mcyX9\nP+B+YMfMvm4LPFN4/pyXFdX/tzGr8Hg74kNyCCGEENZ7bW1tWUsmmzmD9Uczm+ePpwD7lNWPAWaZ\n2ctm1kkaQI0Ffg9s77NGBwOr/MP8NmY2DcDM3jKzNxoJxgcau5AGcJXsT1re9YpfY2Wh7k4vW0qa\nCYI0Y3KRpA5gJrCNpFLdcjNb4o+f8HqAx0gfZwEOAiZKagfagAFAl/1QzdMG8wAAIABJREFUZrag\n0uCqTj8HArcBXzGzZ0kD23vN7PlC3D2xA/C8mS302F71162oWp9uA0qDuU8DP63TviIzu7vK4Apg\nhJntThpIfr/BGc4O4CZJE+g6o3evmb1jZn8GXgCGNnBOSLmulO8DgCu939OAQZI2Ljao09d6XpY0\nsmaL/QpfMbiqLPYW5Yk8ZYk9DnkiT3kiT3kiT3kiT2u0tLTQ2tq6+quadWkPVqW9Jd0+fJrZSkm7\nkJZCfRkYR1p6VXNgIOkk4AS/zqFmtqJQ1480cHsTuLeBPpS8WSHmCcBWwG5m1ql004mBFdp3Fp53\nsuY1EWmW66kexFPL1cBPzaw0R7EXsI/nZ1PS0s1VZnZWD85db3BWtU+S/iRpJ9JM1omFqm7tfZar\nIWb2v/59uaQ20sxo7ke/w0iD+yOAsyV9zMvLX8cPAO+QZkJLui19zSBgDzN7uwfHPkeaVSv5O7rO\nR0GaIVsk6RQzu6EH1wghhBBCCFU0cwZrhKTSMrbPAXPK6ucDYyVtIak/MB6Y7cvt+pvZHcC3gFFm\n9irwjKQjASQNkLRR8WRmdpWZ7WZmo4qDK6/rNLPtgEdJH/Ar+QUwTtIWfo0hVdqVBhmDgRd9cLUf\nMKJCm1ruB05dfYC0a8YxNUk6GRhkZt8tlZnZsWa2nZl9GPhP4H8qDa4kTZbvD6tiGTDMly0iaZC/\nbkW1+nQLcCawmZk9ntE+m6TNfdkokrYC9gaW+PPvlN43VY4VMNzMZgMTgc2AQTUu9wLwQUlDJG0I\nfLIHIc8AVu/L8l8o5LofOFDSYH+PHuhlRWcBfx+Dq79RzOzliTxliT0OeSJPeSJPeSJPeSJPjWvm\nAOtJ4GRJS0g3l7jGyw3AB0ETSUvD2oFHzOxu0n6SNl8+daO3ATgOONWX5D1E48u1AH4DbFGpwpf0\nXUga5LUDlxXjLTb171OBMR7PscDSCm0qHV9yPmk2abHfNOHb5Q0kjZZ0bY3+lPs6sJPSTS4WSmpk\neeHOwPPVKn225TOkpW2LSIOEDcua1erTz/z4WwplF9Ro342kwyW1Vqj6KPCov24PkvbqPel1OwEr\nKhxT0h+Y4q/jAuAKM/u/Cu1K79t3PM5HSAObpRXadjmmgtOA3ZVujvI4XWf0gOp99SWs55N+WfBr\n4Lyy5awAA/xmFyGEEEIIoZfJbF28C3lzSDoD2LLsNu3rPUmbAteZWbXZvT5L0nQzO6TZcbxXfB9g\nh5ltXaON0frexdRnLSdmZ3IsByZD/F9TW1tbW/yWOEPkKU/kKU/kKU/kqTpJmFm3lWnN3IO1Lrod\nuGF9+9Bdj5mtovrSyT5tfXqdJY0n3RHx0rqNW9d2NGF9MnTbniwoCCGEEPqmmMEKIXQhyeLnQggh\nhBBCbdVmsJq5ByuEEEIIIYQQ3ldigBVCCD0QfxckT+QpT+QpT+QpT+QpT+QpT+SpcTHACiGEEEII\nIYReEnuwQghdxB6sEEIIIYT6Yg9WCCGEEEIIIaxlMcAKIYQeiDXpeSJPeSJPeSJPeSJPeSJPeSJP\njYu/gxVC6EbqNtsdQliHDB06ghUrnm52GCGEECqIPVghhC4kGcTPhRDWbSL+/w4hhOaKPVghhBBC\nCCGEsJY1ZYAlaYSkx6rUzZI06r2Oya89XNJCSdMLZcubEUs1kvaVNCmj3XL/XjXXZe03lfSMpP8u\nlM2SNLzOcZMkja0T7931rt+I4jklHS/p3Ixjpkt6RdK0zGucK+m4OvWn50cNknaQ1C5pgaTtJZ0q\naYmkG70fP6hzfN2+StpF0lxJj0laJOnThbrxkp6U9LVG4g7VtDU7gD6irdkB9BFtzQ6gT4i9IHki\nT3kiT3kiT41r5gzWuri24ShghpkdUihbF+PMicmqPK7mfGB2z8JpKJa1cc6c818KHLsW4mjEUcBt\nZjbazJYD/wEcYGaf9/pGX9dKXgM+b2Y7AYcA35e0GYCZ3QzsC8QAK4QQQghhLWnmAGsDSVP8N/i3\nShpY3sB/477Yvy72sn4+a7JYUoek07x8pKQH/Lf2j0ravgcxbQ68WFb2UiGe4/ya7ZIme9kkSVdI\nekjSbyUd7eWbSJrpsXRIOsLLR0ha6sctkzRV0oF+/DJJu3u7jSVdL2mez3gc7mG8Bfwloy8vlRdI\n+rHH3i7pRUn/5eWjgQ8BM8oO+TPwbp3rrPSYkDTG+7HI496k7PoV+yTpYUkfLbSbJWlUjRwUvQ68\nWidGzGxWTruCVX5ufKbpCe/XTYU2/+ix/lbSKd62y4yhpK/7bNchwFeB/5D0oKSrgQ8D00vv4cIx\nW0n6qaRf+9deuX01s9+a2e/88f+S3s8fLNS/AAxuIA+hqpZmB9BHtDQ7gD6ipdkB9AktLS3NDqFP\niDzliTzliTw1rpl3EdwB+DczmyfpeuAk4PJSpaStgYuB3Ugf4h/wQcqzwLZmtrO328wPmQp8x8ym\nSRpAzwaP/YHOYoGZ7eHX2RE4C9jLzF6RtHmh2TAz29sHCdOA24E3gKPM7FVJWwLzvA5gJPApM1si\n6VHgs378EX6No4GzgQfN7IuSBgPzJc00s4eBhz2m0cCJZval8o6U4i4rO8GPGw5MByZJEvA9YAJw\nYFn7Y+olzMy+5ufcAPgJMM7MFkoahA9QCir2yY/7DNAqaZjnc6GkC6u0L17/1tJjH4CNNrPWenFn\n9OvywtNvANuZ2duF9xuk93ALacCyTNJVpcO7n86mS7oGWFU6t6SDgRZ/Px1faH8FcLmZzZX0/4D7\ngR0b7aukfwI2KA24CjL+bRRP20J8+AshhBDC+q6trS1ryWQzZ7D+aGbz/PEUYJ+y+jHALDN72cw6\nSQOoscDvge191uhgYJV/mN/GzKYBmNlbZvZGI8H4QGMX0gCukv1Jy7te8WusLNTd6WVLSTNBAAIu\nktQBzAS2kVSqW25mS/zxE14P8BiwnT8+CJgoqZ20OH8A0GU/lJktqDS4qtPPgcBtwFfM7FnSwPZe\nM3u+EHdP7AA8b2YLPbZX/XUrqtan24DSYO7TwE/rtK/IzO7ujcFVBR3ATZIm0HVG714ze8fM/gy8\nAAxt8Lyicr4PAK70fk8DBknauNigXl/9FxT/A3yhQvXLkkbWDq218NVSu+l6q63ZAfQRbc0OoI9o\na3YAfULsBckTecoTecoTeVqjpaWF1tbW1V/VNHMGq9tv+Su06fbh08xWStoFOBj4MjCOtPSq5sBA\n0knACX6dQ81sRaGuH2ng9iZwbwN9KHmzQswTgK2A3cysU+mmEwMrtO8sPO9kzWsi0izXUz2Ip5ar\ngZ/6kjmAvYB9PD+bkpZurjKzs3pw7nqDs6p9kvQnSTuRZrJOLFR1a++zXO+lw0iD+yOAsyV9zMvL\nX8cPAO+QZkJLui19zSBgDzN7uwfHImlT4B7gm2b2SIUmVwCLJJ1iZjf05BohhBBCCKGyZs5gjZBU\nWsb2OWBOWf18YKykLST1B8YDs325XX8zuwP4FjDKzF4FnpF0JICkAZI2Kp7MzK4ys93MbFRxcOV1\nnWa2HfAo6QN+Jb8Axknawq8xpEq70iBjMPCiD672A0ZUaFPL/cCpqw+Qds04piZJJwODzOy7pTIz\nO9bMtjOzDwP/CfxPpcGVpMny/WFVLAOG+bJFJA3y162oVp9uAc4ENjOzxzPa90S3GSNJ3ym9byoe\nkGY2h5vZbGAisBkwqMY1XgA+KGmIpA2BT/YgzhnA6n1Z/guFLL5U805gsv8bqeQs4O9jcPW3aml2\nAH1ES7MD6CNamh1AnxB7QfJEnvJEnvJEnhrXzAHWk8DJkpaQbi5xjZcbgA+CJpLWTbQDj5jZ3cC2\nQJsvn7rR2wAcB5zqS/IeovHlWgC/AbaoVOFL+i4kDfLagcuK8Rab+vepwBiP51hgaYU2lY4vOZ80\nm7TYb5rw7fIGkkZLurZGf8p9HdhJ6SYXCyU1srxwZ+D5apU+2/IZ0tK2RaRBwoZlzWr16Wd+/C2F\nsgtqtO9G0uGSWqvU/dLPvb+kP0oq7TfbCVhR6RjXH5jir+MC4Aoz+78K7Urv23c8zkdIA8SlFdp2\nOaaC04DdlW6O8jhdZ/RK/anW10+Tltt+ofA671zWZoDf7CKEEEIIIfQyxV+CX0PSGcCWZjaxbuP1\niC85u87Mqs3u9VmSppfdlv99zfcBdpjZ1jXa2Lr51wnWNW3ErEOONiJPOdpoLE9iffz/u62tLX6b\nniHylCfylCfyVJ0kzKzbyrRm7sFaF90O3LC+feiux8xWUX3pZJ+2Pr3OksaT7oh4aUbrtR1OCOFv\nMHToiPqNQgghNEXMYIUQupBk8XMhhBBCCKG2ajNYzdyDFUIIIYQQQgjvKzHACiGEHoi/C5In8pQn\n8pQn8pQn8pQn8pQn8tS4GGCFEEIIIYQQQi+JPVghhC5iD1YIIYQQQn2xByuEEEIIIYQQ1rIYYIUQ\nQg/EmvQ8kac8kac8kac8kac8kac8kafGxd/BCiF0I8XfwQp/m6HbDmXFsyuaHUYIIYTwnos9WCGE\nLiQZrc2OIvR5rRD/v4QQQng/iz1YIYQQQgghhLCWNWWAJWmEpMeq1M2SNOq9jsmvPVzSQknTC2XL\nmxFLNZL2lTQpo91y/14112XtN5X0jKT/LpTNkjS8znGTJI2tE+/d9a7fiOI5JR0v6dyMY6ZLekXS\ntMxrnCvpuDr1p+dHDZJ2kNQuaYGk7SWdKmmJpBu9Hz+oc3xuX4+X9BtJy4p9kDRe0pOSvtZI3KGK\ndeonw7or1u7niTzliTzliTzliTzliTw1rpkzWOvi2pGjgBlmdkihbF2MMycmq/K4mvOB2T0Lp6FY\n1sY5c85/KXDsWoijEUcBt5nZaDNbDvwHcICZfd7rG31du5E0BDgHGAPsAZwraTCAmd0M7AvEACuE\nEEIIYS1p5gBrA0lT/Df4t0oaWN7Af+O+2L8u9rJ+PmuyWFKHpNO8fKSkByQtkvSopO17ENPmwItl\nZS8V4jnOr9kuabKXTZJ0haSHJP1W0tFevomkmR5Lh6QjvHyEpKV+3DJJUyUd6Mcvk7S7t9tY0vWS\n5vmMx+EexlvAXzL68lJ5gaQfe+ztkl6U9F9ePhr4EDCj7JA/A+/Wuc5KjwlJY7wfizzuTcquX7FP\nkh6W9NFCu1mSRtXIQdHrwKt1YsTMZuW0K1jl58Znmp7wft1UaPOPHutvJZ3ibbvMGEr6us92HQJ8\nFfgPSQ9Kuhr4MDC99B4uHLOVpJ9K+rV/7dVAXw8m/ZLgL2a2kvSafqKQhxeAwQ3kIVTTk58w66GW\nlpZmh9AnRJ7yRJ7yRJ7yRJ7yRJ4a18y7CO4A/JuZzZN0PXAScHmpUtLWwMXAbqQP8Q/4IOVZYFsz\n29nbbeaHTAW+Y2bTJA2gZ4PH/kBnscDM9vDr7AicBexlZq9I2rzQbJiZ7e2DhGnA7cAbwFFm9qqk\nLYF5XgcwEviUmS2R9CjwWT/+CL/G0cDZwINm9kWfgZgvaaaZPQw87DGNBk40sy+Vd6QUd1nZCX7c\ncGA6MEmSgO8BE4ADy9ofUy9hZvY1P+cGwE+AcWa2UNIgfIBSULFPftxngFZJwzyfCyVdWKV98fq3\nlh77AGy0mbXWizujX5cXnn4D2M7M3i683yC9h1tIA5Zlkq4qHd79dDZd0jXAqtK5JR0MtPj76fhC\n+yuAy81srqT/B9wP7JjZ122BZwrPn/Oyovr/NmYVHm9HDCZCCCGEsN5ra2vLWjLZzBmsP5rZPH88\nBdinrH4MMMvMXjazTtIAaizwe2B7nzU6GFjlH+a3MbNpAGb2lpm90UgwPtDYhTSAq2R/0vKuV/wa\nKwt1d3rZUtJMEICAiyR1ADOBbSSV6pab2RJ//ITXAzxG+jgLcBAwUVI70AYMALrshzKzBZUGV3X6\nORC4DfiKmT1LGtjea2bPF+LuiR2A581socf2qr9uRdX6dBtQGsx9GvhpnfYVmdndvTG4qqADuEnS\nBLrO6N1rZu+Y2Z+BF4ChDZ5XVM73AcCV3u9pwCBJGxcb/I19fVnSyJot9it8xeCqstiDlSXW7ueJ\nPOWJPOWJPOWJPOWJPK3R0tJCa2vr6q9qmjmD1e23/BXadPvwaWYrJe1CWgr1ZWAcaelVzYGBpJOA\nE/w6h5rZikJdP9LA7U3g3gb6UPJmhZgnAFsBu5lZp9JNJwZWaN9ZeN7JmtdEpFmup3oQTy1XAz/1\nJXMAewH7eH42JS3dXGVmZ/Xg3PUGZ1X7JOlPknYizWSdWKjq1t5nud5Lh5EG90cAZ0v6mJeXv44f\nAN4hzYSWdFv6mkHAHmb2dg+OfY40q1byd3Sdj4I0Q7ZI0ilmdkMPrhFCCCGEEKpo5gzWCEmlZWyf\nA+aU1c8HxkraQlJ/YDww25fb9TezO4BvAaPM7FXgGUlHAkgaIGmj4snM7Coz283MRhUHV17XaWbb\nAY+SPuBX8gtgnKQt/BpDqrQrDTIGAy/64Go/YESFNrXcD5y6+gBp14xjapJ0MjDIzL5bKjOzY81s\nOzP7MPCfwP9UGlxJmlzaH1bFMmCYL1tE0iB/3Ypq9ekW4ExgMzN7PKN9T3SbMZL0ndL7puIBaWZz\nuJnNBiYCmwGDalzjBeCDkoZI2hD4ZA/inAGs3pflv1DIdT9woKTB/h490MuKzgL+PgZXf6OY2csS\na/fzRJ7yRJ7yRJ7yRJ7yRJ4a18wB1pPAyZKWkG4ucY2XG4APgiaSloa1A4+Y2d2k/SRtvnzqRm8D\ncBxwqi/Je4jGl2sB/AbYolKFL+m7kDTIawcuK8ZbbOrfpwJjPJ5jgaUV2lQ6vuR80mzSYr9pwrfL\nG0gaLenaGv0p93VgJ6WbXCyU1Mjywp2B56tV+mzLZ0hL2xaRBgkbljWr1aef+fG3FMouqNG+G0mH\nS2qtUvdLP/f+kv4oqbTfbCdgRaVjXH9gir+OC4ArzOz/KrQrvW/f8TgfIQ1sllZo2+WYCk4Ddle6\nOcrjdJ3RK/WnYl99Cev5pF8W/Bo4r2w5K8AAv9lFCCGEEELoZTJbF+9C3hySzgC2NLOJdRuvRyRt\nClxnZtVm9/osSdPLbsv/vub7ADvMbOsabYzW9y6mPms5MYtVSyuYGW1tbfHbzwyRpzyRpzyRpzyR\npzyRp+okYWbdVqY1cw/Wuuh24Ib17UN3PWa2iupLJ/u09el1ljSedEfES+s2bl3b0YT3u6Hb9mQR\nQQghhND3xQxWCKELSRY/F0IIIYQQaqs2g9XMPVghhBBCCCGE8L4SA6wQQuiB+LsgeSJPeSJPeSJP\neSJPeSJPeSJPjYsBVgghhBBCCCH0ktiDFULoIvZghRBCCCHUF3uwQgghhBBCCGEtiwFWCCH0QKxJ\nzxN5yhN5yhN5yhN5yhN5yhN5alz8HawQQjdSt9nuEHrN0KEjWLHi6WaHEUIIIawVsQcrhNCFJIP4\nuRDWJhH/94QQQujrYg9WCCGEEEIIIaxlWQMsSSMkPValbpakUb0bVh5JwyUtlDS9ULa8GbFUI2lf\nSZMy2i0vtL+7WhtJW/RiXBWvU6ivGbe/L2bVadPr74/iOXNeb0k7S5orqUPSXZIGZRxT87ySVuVH\nvPqY70p6TNIlkraSNE/SAkn75Ly2mX29VNJSSYsk/UzSZoW6X0qaL+lDjcYeKmlrdgB9RFuzA+gT\nYo9DnshTnshTnshTnshT4xqZwVoX13McBcwws0MKZetinDkxWZXHjZ6nEfXO12jczZBz/euAM81s\nF+AO4MxeOG9P+n0CsLOZfQM4AFhsZqPN7FeZ58tpMwP4RzPbFXgK+Obqg83GAguAwxqOPIQQQggh\nZGlkgLWBpCmSlki6VdLA8gaSxkta7F8Xe1k/SZO8rEPSaV4+UtID/pv2RyVt34P4NwdeLCt7qRDP\ncX7NdkmTvWySpCskPSTpt5KO9vJNJM30WDokHeHlI3xGYJKkZZKmSjrQj18maXdvt7Gk6wuzEod7\nGG8Bf8noy0uFx4Ml3SPpSUlXFcpXr/GUdLrPhiwu5HRjP67dy8d5+RiPd5HHt0nxwpLu9ZnAdkkr\nJX0+M+53gZf9HP0KMzSLJJ1c3tjzNtdzfIvHe7CkWwttVs+sSTqovH2dvFXzER/EAMwEPpVxzEse\nwzBJsz0/iyXtvSZUXeB9nSvpg144qfSe8uer/PtdwCBggaQzgUuAo/y8A+n62k6Q9Guvu1pafceJ\nun01s5lm1ulP5wF/V9ZkBenfTfibtTQ7gD6ipdkB9AktLS3NDqFPiDzliTzliTzliTw1rpG7CO4A\n/JuZzZN0PXAScHmpUtLWwMXAbsBK4AEfpDwLbGtmO3u70pKlqcB3zGyapAH0bD9Yf6CzWGBme/h1\ndgTOAvYys1ckFT9UDjOzvSV9FJgG3A68ARxlZq9K2pL04XSatx8JfMrMlkh6FPisH3+EX+No4Gzg\nQTP7oqTBwHxJM83sYeBhj2k0cKKZfam8I6W43Rjgo8AfgfslHW1mt5cqlZbHHe/t+gO/ltTmcT5n\nZp/0dptK2gD4CTDOzBYqLY97vezahxXO+/8Bd5rZqlLc1ZjZs8Ax/vRLwAjSDI2V5RvP6beAj5vZ\n6z7IOB24CPiRpI3M7HXgM8BN3v7sCu0vqJY3SfcCXzSzFWWhPiHpCDObBnya7oOOSn0rnfdzwH1m\ndpEPdEqDvE2AuWb2LUmXkGanvlPpVH6+IyX9n5mVlja+AIw2s1P9eakP/+A5+Gcze1fSD4EJwJTM\nvhb9O+m1L+okvWfqaC08biE+JIcQQghhfdfW1pa1ZLKRQc0fzWyeP54C7FNWPwaYZWYv+2/QpwJj\ngd8D2/us0cHAKv+Qv41/4MXM3jKzNxqIBf+wuwtpAFfJ/sBtZvaKX2Nloe5OL1sKlPajCLhIUgdp\nlmMbrdmrstzMlvjjJ7we4DFgO398EDBRUjtp08EAYHgxIDNbUGlwVcF8M/uDpdts3Uz3XO8D3GFm\nb5jZa6QB4r94PAdKukjSPj5I2gF43swWegyvFmY4VpO0FXAjMN6Pa9QBwI885vJ8A+wJ7Ag85Dk6\nDhhuZu8C9wGHS+pPWr42rVr7WgGY2WFVBhz/Dpws6RHSwOitBvr1CPBvks4hDR5f8/I3zezn/ngB\na94H5XLvd15a/vdxYBTwiPd7f+DD3RpX72u6qHQ28LaZ3VRW9Rywc/1wWgtfLfWbr5famh1AH9HW\n7AD6hNjjkCfylCfylCfylCfytEZLSwutra2rv6ppZAarfP9Hpf0g3T5MmtlKSbsABwNfBsYBX63U\ntsuJpJNIswIGHFr8MCmpH2ng9iZwbwN9KHmzQswTgK2A3cysU+mGAgMrtO8sPO9kTQ5FmuV6qgfx\nlMvJdfeDzJ7yWahDgfMlPUgaTNbLdT/SQK7VB51rg0j75SZUqLsF+ArwCvCImb3mA+hq7RtiZr8h\nvf+Q9BEa2INkZnMkjfVjbpB0mZlNAd4uNHuXNe+Dd/BfXHgfNmgwXAGTzezsBo9bcwLpC6T3wP4V\nqm8HzpG0xMx27Ok1QgghhBBCZY3MYI2QVFw2Naesfj4wVtIWPhMxHpjtS736m9kdpCVio8zsVeAZ\nSUcCSBogaaPiyczsKjPbzcxGlf+m3sw6zWw74FHScqpKfgGMk9+ZTdKQKu1Kg4/BwIs+uNqPtNyt\nvE0t9wOnrj5A2jXjmGr2UNr71Y/Uv/JczyHt3xmotJ/qX4E5vkzzdZ+1+B5pJmQZMMyXJyJpkL8+\nRZcAHWZ2W6VglPZwTa4T8wPAiaVzV8j3PGBvSSO9fmMf7ADM9lhPYM2StlrtG1LYH9WP9B68xp9v\nI2lmnWOHk94X15NullG6I2K198TTwO7++Ei6DrBqvY9KdQ8CxxRiHuIxZJH0CeAM4Agze7NCk+OA\n6TG46g0tzQ6gj2hpdgB9QuxxyBN5yhN5yhN5yhN5alwjA6wnScuslpA2yV/j5aUlYSuAiaT1IO2k\nmYi7gW2BNl/udKO3gfRB71RfkvcQMLQH8f8GqHhra1/SdyFpkNcOXFaMt9jUv08Fxng8xwJLK7Sp\ndHzJ+aQbgSxWuqX9t8sbSBot6doa/SmZD1xJWo74OzO7s3htM2sHbiAtX3sYuNbMOoCdSHu/2oFz\ngAvM7G3SIO1KSYtId5nbsOx6XwcOUrrJxUJJnyyrHw78tU7M1wHPAIv9+uPLYv4T8AXgZs/xXNLy\nRXzJ4j3AJ/x7zfZUeQ2UbtYxrELVeEnLgCWkPWo3ePnWdJ2JqqQF6JC0kLR/6/u1YgB+DOzrOdgT\neK1QV2smspSnpaRB4Azv9wygW59q9PUHpJtpPOCv5VVl9UNIdxcMIYQQQghrgXzLTJ8k6QxgSzOb\nWLdx6DG/icONZvZ4s2PpTUp3OvyDmd3T7FjeK37TjMVm9qMabaz5d9/vC9qI2ZkcbXTPk+jL//es\nDW1tbfFb4gyRpzyRpzyRpzyRp+okYWbdVig1sgdrXXQ7aV/M9LK/hRV6kf/dpvcdM/ths2N4L0ma\nTdo3WOluhyGEEEIIoRf06RmsEELvSzNYIaw9Q4eOYMWKp5sdRgghhPA3eb/OYIUQ1oL4xUsIIYQQ\nQs/05I/7hhDCei/+LkieyFOeyFOeyFOeyFOeyFOeyFPjYoAVQgghhBBCCL0k9mCFELqQZPFzIYQQ\nQgihtmp7sGIGK4QQQgghhBB6SQywQgihB2JNep7IU57IU57IU57IU57IU57IU+NigBVCCCGEEEII\nvST2YIUQuoi/g9V3DN12KCueXdHsMEIIIYT1UrU9WDHACiF0IclobXYUIUtr/M2yEEIIoVniJhch\nhNCbljc7gL4h1u7niTzliTzliTzliTzliTw1LmuAJWmEpMeq1M2SNKp3w8ojabikhZKmF8rWqY89\nkvaVNCmj3fJC+7urtZG0RS/GVfE6hfqacfv7YladNr3+/iieM+f1lrSzpLmSOiTdJWlQxjE1zytp\nVX7Eq4/5rqTHJF0iaStJ8yQtkLRPzmub2dchkmZIWibpfkmDC3UTog2GAAAgAElEQVS/lDRf0oca\njT2EEEIIIeRpZAZrXVyHchQww8wOKZSti3HmxGRVHjd6nkbUO1+jcTdDzvWvA840s12AO4Aze+G8\nPen3CcDOZvYN4ABgsZmNNrNfZZ4vp81EYKaZ7QD8Avjm6oPNxgILgMMajjx0t32zA+gbWlpamh1C\nnxB5yhN5yhN5yhN5yhN5alwjA6wNJE2RtETSrZIGljeQNF7SYv+62Mv6SZrkZR2STvPykZIekLRI\n0qOSevJxZXPgxbKylwrxHOfXbJc02csmSbpC0kOSfivpaC/fRNJMj6VD0hFePkLSUj9umaSpkg70\n45dJ2t3bbSzp+sKsxOEexlvAXzL68lLh8WBJ90h6UtJVhfLVazwlne6zIYsLOd3Yj2v38nFePsbj\nXeTxbVK8sKR7fSawXdJKSZ/PjPtd4GU/R7/CDM0iSSeXN/a8zfUc3+LxHizp1kKb1TNrkg4qb18n\nb9V8xAcxADOBT2Uc85LHMEzSbM/PYkl7rwlVF3hf50r6oBdOKr2n/Pkq/34XMAhYIOlM4BLgKD/v\nQLq+thMk/drrrpZUqsvp65HAZH88mfRLiKIVpH83IYQQQghhLWhkgLUDcKWZ7QisAk4qVkraGrgY\naAF2Bcb4IGVXYFsz29lnEErLzqYCPzCzXYF/Bv63B/H3BzqLBWa2h8ezI3AW0GJmuwGnFZoNM7O9\ngcNJH3QB3gCOMrPdgf2BywrtRwLf9VmBHYDP+vFn+DUAzgYeNLM9/fjvSdrIzB42s695TKMlXVup\nI6W43RjgZOCjwN8XP7D7eUYBx3u7vYATJO0CfAJ4zsx2M7OdgfskbQD8BDjFc30A8HrZtQ8zs1HA\nF4GngTuLcVdjZs+a2TH+9EvACNIMza6k17cY85bAt4CPe44XAKeTBjz/JGkjb/oZ4CZvf3aF9lXz\n5gPFYRVCfaI0YAY+DfxdrX6VnfdzwH2en12ARV6+CTDX+zqHNDtV8VR+viOBv5rZKDO7FDgH+Ik/\nf6PQh3/wHPyzX7MTmNBAXz9kZi94+xVA+XLATtK/m9pmFb7WqUW365DIS5ZYu58n8pQn8pQn8pQn\n8pQn8rRGW1sbra2tq7+q+UAD5/yjmc3zx1OAU4DLC/VjgFlmVprRmAqMBS4Atpd0BfBzYIbSHpht\nzGwagJm91UAc+PlF+sA7pUqT/YHbzOwVv8bKQt2dXrZUa/ajCLhI0ljSh9BtCnXLzWyJP36CNCgA\neAzYzh8fBBwu6Qx/PgAYDiwrXdTMFpAGIvXMN7M/eD9vBvYBbi/U7wPcUfpgLul24F+A+0kDu4uA\ne83sV5I+BjxvZgs9hlf9mC4XlLQVcCNwjJk1vL+INHC72vyWZmX5BtgT2BF4yF+7DUgDlHcl3UfK\n3c9Iy9fOIA3Uu7WvFYCZVVv69u/ADyT9FzCNNDuX6xHgeh+o3mVmHV7+ppn93B8vIPW/km53lqmi\ntPzv48Ao4BHv90DghW6Nq/e12nlLniPltrb9Ms8eQgghhLCeaGlp6bJk8rzzzqvYrpEBVvkHtUr7\nQbp9mDSzlT67cjDwZWAc8NVKbbucSDqJNCtgwKH+2/hSXT/g98CbwL0N9KHkzQoxTwC2AnYzs06l\nGwoMrNC+s/C8kzU5FPApM3uqB/GUy8l194PMnvLZrUOB8yU9SBpM1st1P+BmoNXMlvYg3hwi7Zeb\nUKHuFuArwCvAI2b2mg8uqrVviJn9hvT+Q9JHaGAPkpnN8UH3YcANki4zsynA24Vm77LmffAOPjNc\nGBg2QsBkMzu7weNKXpA01Mxe8Bmu8iW0twPnSFris9Ghp2IPVpZYu58n8pQn8pQn8pQn8pQn8tS4\nRpYIjpBUXDY1p6x+PjBW0haS+gPjgdm+1Ku/md1BWiI2ymdRnpF0JICkAYUlYgCY2VW+1G1UcXDl\ndZ1mth3wKGk5VSW/AMbJ78wmaUiVdqXBx2DgRR9c7Uda7lbeppb7gVNXHyDtmnFMNXso7f3qR+pf\nea7nkPbvDPT9VP8KzPFlmq+b2U3A90gzIcuAYZJGe1yD/PUpugToMLPbKgWjtIdrcqW6ggeAE0vn\nrpDvecDekkZ6/cY+2AGY7bGeQFrOWK99Qwr7o/qR3oPX+PNtJM2sc+xw0vvietLNMkp3RKz2nnga\n2N0fH0nXAVat91Gp7kHgmELMQzyGXNOAL/jj44G7yuqPA6bH4CqEEEIIYe1oZID1JHCypCWkTfLX\neHlpSdgK0h3M2oB20kzE3cC2QJukdtIStIl+3HHAqZI6gIeAoT2I/zdAxVtb+5K+C0mDvHbW7Kmq\nNjs0lbRvrAM4FlhaoU2l40vOJ90IZLHSLe2/Xd6g1h6sMvOBK0nLEX9nZncWr21m7cANpOVrDwPX\n+tK1nYD53t9zgAvM7G3SIO1KSYuAGcCGZdf7OnCQ0k0uFkr6ZFn9cOCvdWK+DngGWOzXH18W859I\nH/xv9hzPJe1nw8w6gXtIe8juqdeeKq9BjX1J4yUtA5aQ9qjd4OVb03UmqpIWoEPSQtL+re/XigH4\nMbCv52BP4LVCXa2ZyFKelpIGgTO83zOAbn2q0ddLgAO9vx8n7YssGgL0xixriD1YWWLtfp7IU57I\nU57IU57IU57IU+PkW2b6JN/vtKWZTazbOPSYpEuAG83s8WbH0puU7nT4BzO7p9mxvFck/ZB0e/gf\n1WhjtL53MfVZy2n+MsFWWNd/hre1tcXykgyRpzyRpzyRpzyRpzyRp+okYWbdVij19QHWSNJMzqtl\nfwsrhFBG0mzSvsFjzey5Gu367g+F9czQbYey4tkV9RuGEEIIode9LwdYIYTeJ8ni50IIIYQQQm3V\nBliN7MEKIYTgYk16nshTnshTnshTnshTnshTnshT42KAFUIIIYQQQgi9JJYIhhC6iCWCIYQQQgj1\nxRLBEEIIIYQQQljLYoAVQgg9EGvS80Se8kSe8kSe8kSe8kSe8kSeGhcDrBBCCCGEEELoJbEHK4TQ\nRfwdrBDeX4YOHcGKFU83O4wQQnjfib+DFULIkgZY8XMhhPcPEf/XhxBC74ubXIQQQq9qa3YAfURb\nswPoI9qaHUCfEHtB8kSe8kSe8kSeGpc1wJI0QtJjVepmSRrVu2HlkTRc0kJJ0wtly5sRSzWS9pU0\nKaPd8kL7u6u1kbRFL8ZV8TqF+ppx+/tiVp02vf7+KJ4z5/WWdK6kZ/29slDSJzKOqXleSavyI159\nzHclPSbpEklbSZonaYGkfXJe28y+XippqaRFkn4mabNC3S8lzZf0oUZjDyGEEEIIeRqZwVoX1xcc\nBcwws0MKZetinDkxWZXHjZ6nEfXO12jczZB7/cvNbJR/3dcL5+1Jv08AdjazbwAHAIvNbLSZ/Srz\nfDltZgD/aGa7Ak8B31x9sNlYYAFwWMORhwpamh1AH9HS7AD6iJZmB9AntLS0NDuEPiHylCfylCfy\n1LhGBlgbSJoiaYmkWyUNLG8gabykxf51sZf1kzTJyzokneblIyU94L9pf1TS9j2If3PgxbKylwrx\nHOfXbJc02csmSbpC0kOSfivpaC/fRNJMj6VD0hFePsJnBCZJWiZpqqQD/fhlknb3dhtLur4wK3G4\nh/EW8JeMvrxUeDxY0j2SnpR0VaF89RpPSaf7bMjiQk439uPavXycl4/xeBd5fJsULyzpXp/ZaZe0\nUtLnM+N+F3jZz9GvMEOzSNLJ5Y09b3M9x7d4vAdLurXQZvXMmqSDytvXyVst3dbH1vGSxzBM0mzP\nz2JJe68JVRd4X+dK+qAXTiq9p/z5Kv9+FzAIWCDpTOAS4Cg/70C6vrYTJP3a666WVKqr21czm2lm\nnf50HvB3ZU1WkP7dhBBCCCGEtcHM6n4BI4BOYE9/fj1wuj+eBYwCtgb+AGxBGrg9CBzhdTMK59rM\nv88DjvDHA4CBObGUxXUe8NUqdTsCTwJD/Pnm/n0ScIs//ijwlD/uDwzyx1sWykeQBhs7+vNHgev9\n8RHA7f74QuBz/ngwsAzYqCym0cC1dfq0L/BXv65IMxJHe91yz+8ooAMYCGwCPA7sAhwN/Khwrk2B\nDYDfAaO8bJC/PvsC08quPQpYBGzag9fiy8CtrLlxSinfpffHlsDsUk6AM4Fved6fLpRfBYyv1r54\nzgox3AsMq1B+ruduEXAdMLiBfp0OfNMfC9jEH3cCh/rjS4CzCu+vowvH/1+Vx8cD/114Xnpt/wGY\nBvT38h8Cx+b2tazNtNJ7slD2X8B/1jnO4NzC1ywDi69uX5GXyFNfyRP2fjFr1qxmh9AnRJ7yRJ7y\nRJ7WmDVrlp177rmrv/znK+VfHyDfH81snj+eApwCXF6oHwPMMrPSjMZUYCxwAbC9pCuAnwMzJA0C\ntjGzaaTI3mogDvz8Ig0qplRpsj9wm5m94tdYWai708uWFvajCLhI0ljSh+dtCnXLzWyJP34CmOmP\nHwO288cHAYdLOsOfDwCGkwZa+PUWAF/K6N58M/uD9/NmYB/g9kL9PsAdZvaGt7kd+BfgfuB7ki4C\n7jWzX0n6GPC8mS30GF71Y7pcUNJWwI3AMWbW8P4i0pK3q83M/Dory+r3JA16H/LXbgNgrpm9K+k+\nUu5+Rlq+dgZpvUy39rUCMLNqS9+uAr5tZibpAtL79ouZ/XoEuF7SBsBdZtbh5W+a2c/98QJS/yvJ\nnTkz//5x0oD0Ee/3QOCFbo2r9zVdVDobeNvMbiqreo6stUit9ZuEEEIIIaxHWlpauiyZPO+88yq2\na2SAZXWeQ4UPk2a2UtIuwMGkWY5xwFcrte1yIukk0p4VI80UrCjU9QN+D7xJ+k1+o96sEPMEYCtg\nNzPrVLqhwMAK7TsLzztZk0MBnzKzp3oQT7mcXHc/yOwppZs/HAqcL+lB0mCyXq77ATcDrWa2tAfx\n5hBpJnNChbpbgK8ArwCPmNlrPrio1r4hZlZcWvdjoOrNPSocO8cH3YcBN0i6zMymAG8Xmr3LmvfB\nO/jS28LAsBECJpvZ2Q0et+YE0hdI74H9K1TfDpwjaYmZ7djTawSIPTO5WpodQB/R0uwA+oTYC5In\n8pQn8pQn8tS4RvZgjZC0hz/+HDCnrH4+MFbSFpL6k5Z5zZa0JWm50x2kJWGjfBblGUlHAkgaIGmj\n4snM7Coz283STQlWlNV1mtl2pOV6n6kS7y+AcfI7s0kaUqVdafAxGHjRB1f7kZbolbep5X7g1NUH\nSLtmHFPNHkp7v/qR+lee6zmk/TsDfT/VvwJzJG0NvO6zFt8jzYQsA4ZJGu1xDfLXp+gSoMPMbqsU\njO/hmlwn5geAE0vnrpDvecDekkZ6/caSPuJ1sz3WE4CfZLRviKRhhadHk5ZUImkbSTMrH7X62OGk\n98X1pOWFpTsiVntPPA3s7o+PpOsAq9b7qFT3IHBMYU/XEI8hi9IdEs8gLb99s0KT44DpMbgKIYQQ\nQlg7GhlgPQmcLGkJaZP8NV5eWhK2AphI+mMe7aSZiLuBbYE2Se2kJWgT/bjjgFMldQAPAUN7EP9v\nSPtWuvElfReSBnntwGXFeItN/ftUYIzHcyywtEKbSseXnE+6EchipVvaf7u8gaTRkq6t0Z+S+cCV\npOWIvzOzO4vXNrN24AbS8rWHSfu6OoCdgPne33OAC8zsbdIg7UpJi0h7ujYsu97XgYOUbnKxUNIn\ny+qHk/aF1XId8Ayw2K8/vizmPwFfAG72HM8FdvC6TuAe4BP+vWZ7qrwGSjfrGFah6lJ/XRaR9p59\nzcu3putMVCUtQIekhcCnge/XioE0Q7av52BP4LVCXa2ZyFKelpJ+ETHD+z0D6NanGn39AWmf3QP+\nWl5VVj+EdHfB8Ddra3YAfURbswPoI9qaHUCfEH+PJ0/kKU/kKU/kqXGlGxL0Sb7faUszm1i3cegx\nSZcAN5rZ482OpTcp3enwD2Z2T7Njea9I+iHp9vA/qtHGMlelrufaiGVdOdqIPOVoY+3lSfTl/+uL\n2traYrlShshTnshTnshTdZIws24rlPr6AGskaSbnVev6t7BCCGUkzSbtGzzWzJ6r0a7v/lAIIXQz\ndOgIVqx4utlhhBDC+877coAVQuh9kix+LoQQQggh1FZtgNXIHqwQQggu1qTniTzliTzliTzliTzl\niTzliTw1LgZYIYQQQgghhNBLYolgCKGLWCIYQgghhFBfLBEMIYQQQgghhLUsBlghhNADsSY9T+Qp\nT+QpT+QpT+QpT+QpT+SpcTHACiGEEEIIIYReEnuwQghdxN/BCs0wdNuhrHh2RbPDCCGEELLF38EK\nIWSRZLQ2O4qw3mmF+P8ohBBCXxI3uQghhN60vNkB9BGRpyyxxyFP5ClP5ClP5ClP5KlxWQMsSSMk\nPValbpakUb0bVh5JwyUtlDS9ULZO/XcuaV9JkzLaLS+0v7taG0lb9GJcFa9TqK8Zt78vZtVp0+vv\nj+I5c15vSedKetbfKwslfSLjmJrnlbQqP+LVx3xX0mOSLpG0laR5khZI2ifntc3s6xBJMyQtk3S/\npMGFul9Kmi/pQ43GHkIIIYQQ8jQyg7Uurt04CphhZocUytbFOHNisiqPGz1PI+qdr9G4myH3+peb\n2Sj/uq8XztuTfp8A7Gxm3wAOABab2Wgz+1Xm+XLaTARmmtkOwC+Ab64+2GwssAA4rOHIQ3fbNzuA\nPiLylKWlpaXZIfQJkac8kac8kac8kafGNTLA2kDSFElLJN0qaWB5A0njJS32r4u9rJ+kSV7WIek0\nLx8p6QFJiyQ9Kqkn/w1vDrxYVvZSIZ7j/JrtkiZ72SRJV0h6SNJvJR3t5ZtImumxdEg6wstHSFrq\nxy2TNFXSgX78Mkm7e7uNJV1fmJU43MN4C/hLRl9eKjweLOkeSU9KuqpQvnqNp6TTfTZkcSGnG/tx\n7V4+zsvHeLyLPL5NiheWdK/P7LRLWinp85lxvwu87OfoV5ihWSTp5PLGnre5nuNbPN6DJd1aaLN6\nZk3SQeXt6+Stlm7rY+t4yWMYJmm252expL3XhKoLvK9zJX3QCyeV3lP+fJV/vwsYBCyQdCZwCXCU\nn3cgXV/bCZJ+7XVXSyrV5fT1SGCyP55M+iVE0QrSv5sQQgghhLAWNDLA2gG40sx2BFYBJxUrJW0N\nXAy0ALsCY3yQsiuwrZntbGa7AKVlZ1OBH5jZrsA/A//bg/j7A53FAjPbw+PZETgLaDGz3YDTCs2G\nmdnewOGkD7oAbwBHmdnuwP7AZYX2I4Hv+qzADsBn/fgz/BoAZwMPmtmefvz3JG1kZg+b2dc8ptGS\nrq3UkVLcbgxwMvBR4O+LH9j9PKOA473dXsAJknYBPgE8Z2a7mdnOwH2SNgB+ApziuT4AeL3s2oeZ\n2Sjgi8DTwJ3FuKsxs2fN7Bh/+iVgBGmGZlfS61uMeUvgW8DHPccLgNOBmcA/SdrIm34GuMnbn12h\nfdW8+UBxWJVwv+KDoetUWDZXo2+l834OuM/zswuwyMs3AeZ6X+eQZqcqnsrPdyTwV59BuxQ4B/iJ\nP3+j0Id/8Bz8s1+zE5jQQF8/ZGYvePsVQPlywE7Sv5vaZhW+1qlFt+uQyEueyFOW2OOQJ/KUJ/KU\nJ/KUJ/K0RltbG62trau/qvlAA+f8o5nN88dTgFOAywv1Y4BZZlaa0ZgKjAUuALaXdAXwc2CGpEHA\nNmY2DcDM3mogDvz8In3gnVKlyf7AbWb2il9jZaHuTi9bqjX7UQRcJGks6UPoNoW65Wa2xB8/QRoU\nADwGbOePDwIOl3SGPx8ADAeWlS5qZgtIA5F65pvZH7yfNwP7ALcX6vcB7ih9MJd0O/AvwP2kgd1F\nwL1m9itJHwOeN7OFHsOrfkyXC0raCrgROMbMGt5fRBq4XW1+G7CyfAPsCewIPOSv3QakAcq7ku4j\n5e5npOVrZ5AG6t3a1wrAzKotfbsK+LaZmaQLSO/bL2b26xHgeh+o3mVmHV7+ppn93B8vIPW/ktyZ\ns9Lyv48Do4BHvN8DgRe6Na7e12rnLXmOlNva9ss8ewghhBDCeqKlpaXLksnzzjuvYrtGBljlH9Qq\n7Qfp9mHSzFb67MrBwJeBccBXK7XtciLpJNKsgAGH+m/jS3X9gN8DbwL3NtCHkjcrxDwB2ArYzcw6\nlW4oMLBC+87C807W5FDAp8zsqR7EUy4n190PMnvKZ7cOBc6X9CBpMFkv1/2Am4FWM1vag3hziLRf\nbkKFuluArwCvAI+Y2Ws+uKjWviFmVlxa92Og6s09Khw7xwfdhwE3SLrMzKYAbxeavcua98E7+Mxw\nYWDYCAGTzezsBo8reUHSUDN7wWe4ypfQ3g6cI2mJz0aHnoq9RXkiT1lij0OeyFOeyFOeyFOeyFPj\nGlkiOEJScdnUnLL6+cBYSVtI6g+MB2b7Uq/+ZnYHaYnYKJ9FeUbSkQCSBhSWiAFgZlf5UrdRxcGV\n13Wa2XbAo6TlVJX8AhgnvzObpCFV2pUGH4OBF31wtR9puVt5m1ruB05dfYC0a8Yx1eyhtPerH6l/\n5bmeQ9q/M9D3U/0rMMeXab5uZjcB3yPNhCwDhkka7XEN8ten6BKgw8xuqxSM0h6uyZXqCh4ATiyd\nu0K+5wF7Sxrp9RtL+ojXzfZYTyAtZ6zXviFlS+mOBh738m0kzax81Opjh5PeF9cD13mcUP098TSw\nuz8+kq4DrFrvo1Ldg8AxhT1dQzyGXNOAL/jj44G7yuqPA6bH4CqEEEIIYe1oZID1JHCypCWkTfLX\neHlpSdgK0h3M2oB20kzE3cC2QJukdtIStIl+3HHAqZI6gIeAoT2I/zdAxVtb+5K+C0mDvHbW7Kmq\nNjs0lbRvrAM4FlhaoU2l40vOJ90IZLHSLe2/Xd6g1h6sMvOBK0nLEX9nZncWr21m7cANpOVrDwPX\n+tK1nYD53t9zgAvM7G3SIO1KSYuAGcCGZdf7OnCQ0k0uFkr6ZFn9cOCvdWK+DngGWOzXH18W859I\nH/xv9hzPJe1nw8w6gXtIe8juqdeeKq9BjX1Jl/rrsgjYFyjtLduarjNRlbQAHZIWAp8Gvl8rBtIM\n2b6egz2B1wp1tWYiS3laSvpFxAzv9wygW59q9PUS4EBJy0jLDS8uqx8C9MYsa4i9RXkiT1lij0Oe\nyFOeyFOeyFOeyFPj5Ftm+iTf77SlmU2s2zj0mKRLgBvN7PFmx9KblO50+Aczu6fZsbxXJP2QdHv4\nH9VoY7S+dzH1WcuJ5W85cvPUCn35/6O/VVtbWyzDyRB5yhN5yhN5yhN5qk4SZtZthVJfH2CNJM3k\nvFr2t7BCCGUkzSbtGzzWzJ6r0a7v/lAIfdbQbYey4tkV9RuGEEII64j35QArhND7JFn8XAghhBBC\nqK3aAKuRPVghhBBcrEnPE3nKE3nKE3nKE3nKE3nKE3lqXAywQgghhBBCCKGXxBLBEEIXsUQwhBBC\nCKG+WCIYQgghhBBCCGtZDLBCCKEHYk16nshTnshTnshTnshTnshTnshT42KAFUIIIYQQQgi9JPZg\nhRC6iL+DFULzDB06ghUrnm52GCGEEDLE38EKIWRJA6z4uRBCc4j4fzmEEPqGuMlFCCH0qrZmB9BH\ntDU7gD6irdkB9AmxFyRP5ClP5ClP5KlxWQMsSSMkPValbpakUb0bVh5JwyUtlPT/s3fncXJVdd7H\nP98EkG0IEDUBhMBklEdUCAEUBZMGd9kxLAEGhgEdXzqigDgoahIWBQXGCKLymAloMgiMKLuEpTsT\ngjEhZAGyyBIWcQI8QjQ4ipj+PX/cU8nt6lpONR26m3zfr1e/+tY55977u7+qdOrUOefW7aWyFX0R\nSz2SxkqamtFuRan9zfXaSNq2F+OqeZ5SfcO40+uivUmbXn99lI+Z83xLGifpIUlrcmNpdlxJq/Oi\n7bLPtyU9KOkiSW+UNEfSfEn75zy3mdf6LUlLJS2U9DNJW5Xq/lvSXElvbjV2MzMzM8vTyghWf5yz\ncDgwIyI+Virrj3HmxBR1tls9TiuaHa/VuPtCzvkfBI4AZvbicXty3Z8Edo+IfwM+CCyOiL0i4t7M\n4+W0mQG8IyJGAY8AX167c8QYYD5wUMuRWw1tfR3AANHW1wEMEG19HcCA0NbW1tchDAjOUx7nKY/z\n1LpWOlgbS5omaYmk6yRtWt1A0nhJi9PPhalskKSpqWyRpM+n8pGS7kyftN8vaZcexL818FxV2fOl\neE5M51wg6epUNlXSZEmzJT0q6chUvoWku1IsiyQdmspHpBGBqZKWS5ou6UNp/+WS9k7tNpc0pTQq\ncUgK46/AHzKu5fnS9hBJt0haJumKUvnaOZ6SzkijIYtLOd087bcglR+VyvdJ8S5M8W1RPrGkW9NI\n4AJJqyT9Y2bca4AX0jEGlUZoFkr6bHXjlLf7Uo6vTfF+RNJ1pTZrR9Ykfbi6fZO81RQRyyPikXL+\nMjyfYhguaWbKz2JJ+60LVeena71P0ptS4dTKayo9Xp1+3whsCcyX9CXgIuDwdNxN6frcHi/p16nu\n+5IqdTnXeldEdKaHc4C3VDVZSfHvxszMzMzWh4ho+gOMADqBfdPjKcAZabsdGA1sBzwJbEvRcbsb\nODTVzSgda6v0ew5waNreBNg0J5aquCYBX6hTtxuwDNgmPd46/Z4KXJu23w48krYHA1um7aGl8hEU\nnY3d0uP7gSlp+1DghrR9AXBc2h4CLAc2q4ppL+DKJtc0FvjfdF5RjEgcmepWpPyOBhYBmwJbAA8B\newBHAj8sHevvgI2Bx4DRqWzL9PyMBW6qOvdoYCHwdz14Lj4NXMe6G6dU8l15fQylGEHaLJV/Cfhq\nyvsTpfIrgPH12pePWSOGW4HhDWKsuV+T6zoD+HLaFrBF2u4EPp62LwK+Unp9HVna/491tk8Cvlt6\nXHlu/w9wEzA4lX8POKHVa01tbqq8JktlXwO+2GS/gAmln/aA8E+3H+fFeVofeSKsvvb29r4OYUBw\nnvI4T3mcp3Xa29tjwoQJa3/S32yqfzYi31MRMSdtTwM+BwsvocoAACAASURBVFxaqt8HaI+IyojG\ndGAMcD6wi6TJwG3ADElbAttHxE0Ukf21hThIxxdFp2JanSYHAtdHxIvpHKtKdb9IZUtL61EEfFPS\nGIo3z9uX6lZExJK0/TBwV9p+ENg5bX8YOETSWenxJsBOFB0t0vnmA5/KuLy5EfFkus5rgP2BG0r1\n+wM/j4i/pDY3AO8H7gAulvRN4NaIuFfSO4HfRcQDKYaX0j5dTijpjcBPgHER0fL6Ioopb9+PiEjn\nWVVVvy9Fp3d2eu42Bu6LiDWSfkmRu59RTF87i2K+TLf2jQKIiPUx9W0eMEXSxsCNEbEolb8cEbel\n7fkU119L7qhZpN8foOiQzkvXvSnwbLfGTa5V0jnAKxHxn1VVz5A1F2li8yZmZmZmG5C2trYuUyYn\nTZpUs10rHaxo8hhqvJmMiFWS9gA+QjHKcRTwhVptuxxI+gzFmpWgGClYWaobBDwOvEzxSX6rXq4R\n8/HAG4E9I6JTxQ0FNq3RvrP0uJN1ORTwiSimor1aObnuvlPEIypu4vBx4DxJd1N0JpvlehBwDTAx\nIpb2IN4cohjJPL5G3bXAvwIvAvMi4k+pc1Gv/WsmImalTvdBwFWSLomIacArpWZrWPc6+Btp6m2p\nY9gKAVdHxDk9jVnSP1G8Bg6sUX0D8HVJSyJit56ew8BrZnK19XUAA0RbXwcwIHgtSB7nKY/zlMd5\nal0ra7BGSHpP2j4OmFVVPxcYI2lbSYMppnnNlDSUYrrTzymmhI1OoyhPSzoMQNImkjYrHywiroiI\nPSNidLlzleo6I2Jniul6x9SJ9x7gKKU7s0napk67SudjCPBc6lwdQDFFr7pNI3cAp63dQRqVsU89\n71Gx9msQxfVV53oWxfqdTdN6qiOAWZK2A/6cRi0uphgJWQ4Ml7RXimvL9PyUXQQsiojrawWT1nBd\n3STmO4F/qRy7Rr7nAPtJGpnqN5f01lQ3M8X6SeCnGe1fjfJap+0l3dWwsbQTxetiCvCjFGeX41R5\nAtg7bR9G1w5Wo9dRpe5uYFxpTdc2KYYskj5KMQJ4aES8XKPJicDt7lyZmZmZrR+tdLCWAZ+VtIRi\nkfwPUnllSthK4GyKL/NYQDEScTOwA9AhaQHFFLSz034nAqdJWgTMBob1IP7fUKxb6SZN6buAopO3\nALikHG+5afo9HdgnxXMCsLRGm1r7V5xHcSOQxSpuaX9udQNJe0m6ssH1VMwFLqeYjvhYRPyifO6I\nWABcRTF97VcU67oWAe8C5qbr/TpwfkS8QtFJu1zSQoo1XW+oOt+ZwIdV3OTiAUkHV9XvRLEurJEf\nAU8Di9P5x1fF/P+AfwKuSTm+D9g11XUCtwAfTb8btqfOc6DiZh3Da5QfLulpimmKt2jdbf23o+tI\nVC1twCJJDwBHA99pFAPwf4GxKQf7An8q1TUaiazkaSnFBxEz0nXPAGpdU81rBS6jWGd3Z3our6iq\n34bi7oL2qnX0dQADREdfBzBAdPR1AAOCv48nj/OUx3nK4zy1rnJDggEprXcaGhFnN21sPSbpIuAn\nEfFQX8fSm1Tc6fDJiLilr2N5rUj6HsXt4X/YoE1kzkrdwHXgaV05OnCecnRQ5EkM5P+X17eOjg5P\nV8rgPOVxnvI4T/VJIiK6zVAa6B2skRQjOS9F1+/CMrMqkmZSrBs8ISKeadDOHSyzPuMOlpnZQPG6\n7GCZWe8rOlhm1heGDRvBypVP9HUYZmaWoV4Hq5U1WGa2gaj1nQ7+6frT3t7e5zEMhB/nqbU8uXPV\nmNeC5HGe8jhPeZyn1rmDZWZmZmZm1ks8RdDMupAU/rtgZmZm1pinCJqZmZmZma1n7mCZmfWA56Tn\ncZ7yOE95nKc8zlMe5ymP89Q6d7DMzMzMzMx6iddgmVkXXoNlZmZm1ly9NVgb9UUwZta/Sd3+Vmyw\nhu0wjJW/XdnXYZiZmdkA4REsM+tCUjCxr6PoRyYW3wtWraOjg7a2ttc8nIHGecrjPOVxnvI4T3mc\npzzOU32+i6CZmZmZmdl6ltXBkjRC0oN16tolje7dsPJI2knSA5JuL5Wt6ItY6pE0VtLUjHYrSu1v\nrtdG0ra9GFfN85TqG8adXhftTdr0+uujfMyc51vSOEkPSVqTG0uz40panRdtl32+LelBSRdJeqOk\nOZLmS9o/57nNvNZtJM2QtFzSHZKGlOr+W9JcSW9uNXbrzp/m5XGe8jhPeZynPM5THucpj/PUulZG\nsPrjXMLDgRkR8bFSWX+MMyemqLPd6nFa0ex4rcbdF3LO/yBwBDCzF4/bk+v+JLB7RPwb8EFgcUTs\nFRH3Zh4vp83ZwF0RsStwD/DltTtHjAHmAwe1HLmZmZmZZWmlg7WxpGmSlki6TtKm1Q0kjZe0OP1c\nmMoGSZqayhZJ+nwqHynpTkkLJd0vaZcexL818FxV2fOleE5M51wg6epUNlXSZEmzJT0q6chUvoWk\nu1IsiyQdmspHSFqa9lsuabqkD6X9l0vaO7XbXNKU0qjEISmMvwJ/yLiW50vbQyTdImmZpCtK5Wvn\neEo6I42GLC7ldPO034JUflQq3yfFuzDFt0X5xJJuTSOBCyStkvSPmXGvAV5IxxhUGqFZKOmz1Y1T\n3u5LOb42xfsRSdeV2qwdWZP04er2TfJWU0Qsj4hHyvnL8HyKYbikmSk/iyXtty5UnZ+u9T5Jb0qF\nUyuvqfR4dfp9I7AlMF/Sl4CLgMPTcTel63N7vKRfp7rvS2vvONH0WoHDgKvT9tUUH0KUraT4d2Ov\nkr8XJI/zlMd5yuM85XGe8jhPeZyn1rVyF8FdgZMjYo6kKcBngEsrlZK2Ay4E9gRWAXemTspvgR0i\nYvfUbqu0y3TgGxFxk6RN6Nl6sMFAZ7kgIt6TzrMb8BXgvRHxoqTym8rhEbGfpLcDNwE3AH8BDo+I\nlyQNBeakOoCRwCciYomk+4Fj0/6HpnMcCZwD3B0Rp6RpWXMl3RURvwJ+lWLaC/iXiPhU9YVU4k72\nAd4OPAXcIenIiLihUqlimttJqd1g4NeSOlKcz0TEwand30naGPgpcFREPCBpS+DPVec+qHTc/wB+\nERGrK3HXExG/Bcalh58CRlCM0ERVvkk5/SrwgYj4c+pknAF8E/ihpM0i4s/AMcB/pvbn1Gh/fr28\nSboVOCUiXvUt30rHPQ74ZUR8M3V0Kp28LYD7IuKrki6iGJ36Rq1DpeMdJumPEVGZ2vgssFdEnJYe\nV67h/6QcvC8i1kj6HnA8MC3zWt8cEc+mc65U9+mAnRSvmcbKEz93Bnry8YeZmZnZ60hHR0dWh7OV\nDtZTETEnbU8DPkepg0XxZr89IiojGtOBMRRviHeRNBm4DZiR3uRvHxE3AUTEX1uIg3R8AXukWGo5\nELg+Il5M51hVqvtFKltaegMq4JuSxlC8Cd2+VLciIpak7YeBu9L2gxRvPwE+DBwi6az0eBNgJ2B5\n5aQRMZ+iI9LM3Ih4Ml3nNcD+FJ3Aiv2Bn0fEX1KbG4D3A3cAF0v6JnBrRNwr6Z3A7yLigRTDS2mf\nLieU9EbgJ8C41Llq1QeB71e+QKkq3wD7ArsBs9NztzFFB2WNpF9S5O5nFNPXzgLaarVvFEClo9jL\n5gFTUkf1xohYlMpfjojb0vZ8iuuvJXfUrDL97wPAaGBeuu5NgWe7Nc6/1upphc9Q5LaxAzKPvgHz\nnPQ8zlMe5ymP85THecrjPOVxntZpa2vrko9JkybVbNdKB6v6jVqt9SDd3kxGxCpJewAfAT4NHAV8\noVbbLgeSPkMxKhDAx8uf1EsaBDwOvAzc2sI1VLxcI+bjgTcCe0ZEp4obCmxao31n6XEn63IoilGu\nR3oQT7WcXHffKeKRNAr1ceA8SXdTdCab5XoQcA0wMSKW9iDeHKJYL3d8jbprgX8FXgTmRcSfUuei\nXvvXTETMSp3ug4CrJF0SEdOAV0rN1rDudfA30mhsqWPYCgFXR8Q5PQz5WUnDIuJZScPpPoX2BuDr\nkpZExG49PIeZmZmZ1dHKtLwRksrTpmZV1c8FxkjaVtJgYDwwM031GhwRP6eYIjY6jaI8LekwAEmb\nSNqsfLCIuCIi9oyI0dXToCKiMyJ2Bu6nmE5Vyz3AUUp3ZpO0TZ12lc7HEOC51Lk6gGK6W3WbRu4A\nTlu7gzQqY5963qNi7dcgiuurzvUsivU7m6pYT3UEMCtN0/xzRPwncDHFSMhyYHianoikLdPzU3YR\nsCgirq8VjIo1XFfXqiu5E/iXyrFr5HsOsJ+kkal+c0lvTXUzU6yfpJjO2Kz9q1Fe67S9pLsaNpZ2\nonhdTAF+lOLscpwqTwB7p+3D6NrBavQ6qtTdDYwrrenaJsWQ6ybgn9L2ScCNVfUnAre7c/XqeU56\nHucpj/OUx3nK4zzlcZ7yOE+ta6WDtQz4rKQlFIvkf5DKK1PCVlLcwawDWEAxEnEzsAPQIWkBxRS0\ns9N+JwKnSVoEzAaG9SD+3wA1b22dpvRdQNHJWwBcUo633DT9ng7sk+I5AVhao02t/SvOo7gRyGIV\nt7Q/t7qBpL0kXdngeirmApdTTEd8LCJ+UT53RCwArqKYvvYr4Mo0de1dFGu/FgBfB86PiFcoOmmX\nS1oIzADeUHW+M4EPq7jJxQOSDq6q3wn43yYx/wh4Gliczj++Kub/R/HG/5qU4/so1vUREZ3ALcBH\n0++G7anzHKi4WcfwGuWHS3qaYpriLVp3W//t6DoSVUsbsEjSA8DRwHcaxQD8X2BsysG+wJ9KdY1G\nIit5WkrxQcSMdN0zgFrXVPNaKTrLH5K0nGK64YVV9dsAvTHKamZmZmY1KC2ZGZDSeqehEXF208bW\nY+kmDj+JiIf6OpbepOJOh09GxC19HctrJd00Y3FE/LBBm2DiaxdTvzcRBvLfSTMzM1s/JBER3WYo\nDfQO1kiKkZyXqr4Ly8yqSJpJsW7whIh4pkG7gftHYT0YtsMwVv72Vd+Y0szMzF5n6nWwenJr9H4j\nIh6LiPe7c2XWXESMjYgDGnWuSm39k37qda48Jz2P85THecrjPOVxnvI4T3mcp9YN6A6WmZmZmZlZ\nfzKgpwiaWe+TFP67YGZmZtbY63KKoJmZmZmZWX/iDpaZWQ94Tnoe5ymP85THecrjPOVxnvI4T61z\nB8vMzMzMzKyXeA2WmXXhNVhmZmZmzdVbg7VRXwRjZv2b1O1vhfUzw4aNYOXKJ/o6DDMzM6viKYJm\nVkP4p+lPe5+e/9lnn6z/9PUjnrufx3nK4zzlcZ7yOE95nKfWNexgSRoh6cE6de2SRq+fsBqTtJOk\nByTdXipb0Rex1CNprKSpGe36VdxlObE1ayNpgqQzei+qrseUNFXSmCbtt5Z0g6RFkuZI2i3jHO2S\ndmpS39LrX9I4SUsk3Z0eXyNpoaTPp+s4ssn+Odd6XLrORZLulbR7qe4SSQ9LGttK3GZmZmaWL2cE\nqz8uxjgcmBERHyuV9cc4c2Lqj3FXDPT4K74CLIiIPYCTgO/2URynAKdGxAckDQf2johRETG5F8/x\nODAmXev5wJWViog4EzgX+OdePN8GrK2vAxgQ2tra+jqEAcF5yuM85XGe8jhPeZyn1uV0sDaWNC19\n8n6dpE2rG0gaL2lx+rkwlQ1Kn7gvTp+mfz6Vj5R0Z/rk/n5Ju/Qg7q2B56rKni/Fc2I65wJJV6ey\nqZImS5ot6dHKaIGkLSTdlWJZJOnQVD5C0tK033JJ0yV9KO2/XNLeqd3mkqakkZH5kg5JYfwV+EPG\ntTyfjjNc0sw0MrdY0n6pfLWk81O+7pP0plR+cOmcM0rlEyT9OLVdLunUVD42Hf8WScskXaHCyZL+\nvZS7UyVdUp3TZvHXy3uZpL+XdLukeSmWt0naStITpTabS3pK0uBa7WucfxVFrhvZDbgHICKWAztX\n8tXA74E19V7HydGSfp3yWXm+TpJ0Wel6bpY0RtLXgP2BKZK+BdwB7JCe7/2r8jRaUke67tslDcu9\n1oiYExGV190cYIeqJisp/v2YmZmZ2foQEXV/gBFAJ7BvejwFOCNttwOjge2AJ4FtKTpsdwOHproZ\npWNtlX7PAQ5N25sAmzaKoU5ck4Av1KnbDVgGbJMeb51+TwWuTdtvBx5J24OBLdP20FL5CIo3s7ul\nx/cDU9L2ocANafsC4Li0PQRYDmxWFdNewJVNrukM4MtpW8AWabsT+Hjavgj4SuVcpX1PAb6dticA\nC1JuhwJPAcOBscD/pusSMAM4EtgCeBQYnPafDbyjB89JvbxPKL1m7gJGpu13A3en7Z8DY9P20ZVc\nNWi/9pg1XhcH1yi/ALikdJy/AntmXle913F7KecfA+5M2ycB3y21v5liRKmyz56l19fiUrup6fnY\nKD0HQ0v5mJJ7rVVtvlj9ugPeD9zSZL+A8E/Tn/Y+Pj8xELS3t/d1CAOC85THecrjPOVxnvI4T/Wl\n/4u7vZfKuYvgUxExJ21PAz4HXFqq3wdoj4gXACRNB8ZQTE/aRdJk4DZghqQtge0j4iaKiJqNPHQj\nScAeKZZaDgSuj4gX0zlWlep+kcqWSnpz5ZDAN1WsbekEti/VrYiIJWn7YYo3/AAPAjun7Q8Dh0g6\nKz3eBNiJoqNFOt984FNNLm0exejGxsCNEbEolb8cEbel7fnAB9P2jpKuo+jgbgysKB3rxpTb30u6\nh6JT8QdgbkQ8CcX6H2D/iLhBxZqggyUtAzaKiIebxFpLo7wjaQvgfcD16TkkxQ1wHXAMMBM4Fvhe\nk/Y1RcSEOlUXApMlPUDx3C0A1mRe1+NUvY5LdTek3/MpOkw5mt2eb1fgncCd6boHAb+rbtTgWouT\nSAcAJ1OMmpU9A7xN0hsi4uX6R5hY2m7D0+HMzMxsQ9fR0ZF104+cDlY0eQw13jRGxCpJewAfAT4N\nHAV8oVbbLgeSPgN8Mp3n4xGxslQ3iOIN78vArRmxVyu/oazEcTzwRoqRhU4VN23YtEb7ztLjTtbl\nTsAnIuKRHsSzVkTMSp28g4CrJF0SEdOAV0rN1pTOexlwcUTcquKmBeU33OXnSNR+zsrtplCsU1pG\nMZKyPgwCXoyIWjeGuAm4QNI2FCNG9wBbNmjfkohYTWndUXqOH8/ct9br+NRUXXk9lJ+Xv9F16m23\nKbVNCHgoIvZrcb91ByhubHEl8NFKh7ciIh6XtBR4UtIH6nemJ/b09BuQtr4OYEDw3P08zlMe5ymP\n85THecrjPK3T1tbWJR+TJk2q2S5nDdYISe9J28cBs6rq5wJjJG0raTAwHpgpaSjFtLOfA18FRkfE\nS8DTkg4DkLSJpM3KB4uIKyJiz4gYXe5cpbrOiNiZYrreMXXivQc4StK26Rzb1GlX6WANAZ5LnasD\n6DoSkfNlQHcAp63dQRqVsU/3YIo71j0XEVOAH1F0NBrFsBXrRjZOqqo7LOV2KMXUwHmpfB8Va8sG\nUeTvXoCImAvsSPHcXVMnvqVNLqFh3lMnZ4WkcaVj7p7q/kTxnE6mmL4Wjdq3StKQNDKIpE8CM9Nr\nERXr77ZrsG+313G9pun3E8CotL5tR4rRw7qHr1G2HHiTpH3T+TdSxl0PS/HuBPwM+MeIeKxG/e7A\nLhQjyT0ZqTQzMzOzBnI6WMuAz0paQrE4/gepvFisUXSCzgY6KKZezYuImykW13dIWgD8JLUBOBE4\nTdIiirUmlQX8rfgNxZqvbtKUvgsoOnkLgMoNG+qNxE2n6HgsAk4AltZoU2v/ivMobgSyWMUt7c+t\nbiBpL0lXdt+1izZgUZrGdjTwnSbnnQT8l6R5dL8ZxWKK5+M+4NxSR/V+4HKK6Y6PpU5DxXXA7Fh3\ng4Ry/EObxN4o72UnAKeouGHHQxRr2SqupRhN/Gmp7PgG7buRNEnSwTWq3g48lDqJHwEqN1wRMBJ4\nocFh672Oa76eImI2RSfrYYrncH51mzqPK/u/AowDLpK0kOLf1HtbuNavUfzbuELFzUbmVtVvAzwR\nEZ019rWWdPR1AAOCvz8lj/OUx3nK4zzlcZ7yOE+tU7E+a2BJ652GRsTZTRtvYCRNAFZHxKVV5WOB\nMyOiZidF0s3ApRHRXqPuIGCXiLh8fcTcVyS9Azg5Ir7Y17G8ViQdDRwREeMbtIn6/Xpbp4O+nSYo\nBsLf746ODk8vyeA85XGe8jhPeZynPM5TfZKIiG4zkgZqB2skcBXwUnT9LqwNXqsdLElDKKZ5LoiI\nY1+7SO21puL2+++nuFvl3Q3auYM1IAyMDpaZmdnr1euqg2Vm60/RwbL+btiwEaxc+URfh2FmZrbB\nqtfBylmDZWYbmFrf6eCfrj/t7e19ev6B0rny3P08zlMe5ymP85THecrjPLXOHSwzMzMzM7Ne4imC\nZtaFpPDfBTMzM7PGPEXQzMzMzMxsPXMHy8ysBzwnPY/zlMd5yuM85XGe8jhPeZyn1rmDZWZmZmZm\n1ku8BsvMuvAaLDMzM7Pm6q3B2qgvgjGz/k3q9rfC+sCwHYax8rcr+zoMMzMza4FHsMysC0nBxL6O\nYgBYAeyyns8xsfhOsoGso6ODtra2vg6j33Oe8jhPeZynPM5THuepvh7dRVDSCEkP1qlrlzS6twJs\nhaSdJD0g6fZS2Yq+iKUeSWMlTc1o16/iLsuJrVkbSRMkndF7UXU9pqSpksY0ab+1pBskLZI0R9Ju\nGedol7RTk/qWXv+SxklaIunu9PgaSQslfT5dx5FN9m96randdyU9ko49qlR+iaSHJY1tJW4zMzMz\ny5dzk4v++PHp4cCMiPhYqaw/xpkTU3+Mu2Kgx1/xFWBBROwBnAR8t4/iOAU4NSI+IGk4sHdEjIqI\nyb11AkkfA0ZGxFuBfwF+UKmLiDOBc4F/7q3zbdDW9+jV64Q/9czjPOVxnvI4T3mcpzzOU+tyOlgb\nS5qWPnm/TtKm1Q0kjZe0OP1cmMoGpU/cF6eRg8+n8pGS7kyfrt8vqSdvU7YGnqsqe74Uz4npnAsk\nXZ3KpkqaLGm2pEcrowWStpB0V4plkaRDU/kISUvTfsslTZf0obT/ckl7p3abS5qSRkbmSzokhfFX\n4A8Z1/J8Os5wSTPTyNxiSful8tWSzk/5uk/Sm1L5waVzziiVT5D049R2uaRTU/nYdPxbJC2TdIUK\nJ0v691LuTpV0SXVOm8VfL+9lkv5e0u2S5qVY3iZpK0lPlNpsLukpSYNrta9x/lUUuW5kN+AegIhY\nDuxcyVcDvwfW1HsdJ0dL+nXKZ+X5OknSZaXruVnSGElfA/YHpkj6FnAHsEN6vvevytNoSR3pum+X\nNKyFaz0M+HG61l8DQ0r7A6yk+PdjZmZmZutBTgdrV+DyiNgNWA18plwpaTvgQqANGAXskzopo4Ad\nImL3NHJQmS43HbgsIkYB7wP+pwdxDwY6ywUR8Z4Uz24UIxZtEbEnUH5DPDwi9gMOAS5KZX8BDo+I\nvYEDgUtK7UcC346IXVMejk37n5XOAXAOcHdE7Jv2v1jSZhHxq4g4PcW0l6Qra11IJW7gOOCXETEa\n2ANYmMq3AO5L+ZoFfDKVz4qIfSNiL+Ba4Eulw76L4vl4H/B1FaMlAPsAnwXeDvwDcARwHXCIpMGp\nzcnAf1TFVldm3iuuBP41IvahyOH3I+KPwAKtm7Z2cMrDmlrta5z/9IiYk2KYJOngGuddBFQ61O8G\ndgLe0uS6xkXEM9R/HQMMTtd/OnRZtdRtVC8izgPuB46LiC8BhwKPRsToiLi30k7SRsBlwCfSdU8F\nvtHCte4APF16/Ewqq+ik+Pdjr1a/ndzbv/j7U/I4T3mcpzzOUx7nKY/z1Lqcuwg+VXlTB0wDPgdc\nWqrfB2iPiBcAJE0HxgDnA7tImgzcBsyQtCWwfUTcBBARzT6N70aSKDog0+o0ORC4PiJeTOdYVar7\nRSpbKunNlUMC31SxtqUT2L5UtyIilqTth4G70vaDwM5p+8MUHZSz0uNNKN7AL6+cNCLmA59qcmnz\nKEY3NgZujIhFqfzliLgtbc8HPpi2d5R0HbAdsDFd3+7dmHL7e0n3AO+mGE2bGxFPQrH+B9g/Im5Q\nsSboYEnLgI0i4uEmsdbSKO9I2oKiw3d9eg5JcUPRyTsGmAkcC3yvSfuaImJCnaoLgcmSHqB47hYA\nazKv63GqXseluhvS7/nAiMzjNbs9367AO4E703UPAn5X3ajBtTbzDPA2SW+IiJfrtmovbe+Mp8OZ\nmZnZBq+joyOrw5nTwar+NL7WmptubxojYpWkPYCPAJ8GjgK+UKttlwNJn6EYpQng4xGxslQ3iOIN\n78vArRmxVyu/oazEcTzwRmDPiOhUcdOGTWu07yw97mRd7kQx2vBID+JZKyJmpU7eQcBVki6JiGnA\nK6Vma0rnvQy4OCJuTaM/5Tfc5edI1F8nVSmfQjH6tIyuIzS9aRDwYhqhq3YTcIGkbYDRFNP5tmzQ\nviURsZrSuqP0HD+euW+t1/Gpqbryeig/L3+j68hwtym1TQh4KI2U9sQzwI6lx29JZQBExOOSlgJP\nSvpA3c70AT08+4bEnc4snrufx3nK4zzlcZ7yOE95nKd12trauuRj0qRJNdvlTBEcIak8jW1WVf1c\nYIykbdM0s/HATElDKaZQ/Rz4KjA6Il4CnpZ0GICkTSRtVj5YRFwREXumqVMrq+o6I2JniqlWx9SJ\n9x7gKEnbpnNsU6ddpYM1BHguda4OoOtIRM6XAd0BnLZ2h9Jd21qh4o51z0XEFOBHFB2NRjFsxbqR\njZOq6g5LuR0KjKUYHYNi+uaI1FE9BrgXICLmUrwpHw9cUye+pU0uoWHeUydnhaRxpWPunur+RPGc\nTgZuiULd9q2SNCSNDCLpk8DM9FpExfq77Rrs2+11XK9p+v0EMEqFHSlGD+sevkbZcuBNkvZN599I\nGXc9LLkJODHtuy+wKiKeLV3P7hRdg+17OFJpZmZmZg3kdLCWAZ+VtIRicXzlrmQBkDpBZwMdFFOv\n5kXEzRTrPjokLQB+ktpA8ebvNEmLgNlAeQF+rt8A9uLOlAAAIABJREFU29aqSFP6LqDo5C1g3Zqq\neiNx0yk6HouAE4ClNdrU2r/iPIobgSxWcUv7c6sbNFqDVdIGLErT2I4GvtPkvJOA/5I0j+43o1hM\n8XzcB5xb6qjeD1xOMd3xsdRpqLgOmB0R3W7MkToZDTXIe9kJwCkqbtjxEMU6pIprKUYTf1oqO75B\n+24arEt6O/BQ6iR+hLQ+LE3BGwm80OCw9V7HNV9PETGbopP1MMVzOL+6TZ3Hlf1fAcYBF0laSPFv\n6r2515qmk66Q9CjwQ6rWTALbAE9ERGf1vtYir8HK4rn7eZynPM5THucpj/OUx3lq3YD8ouG03mlo\nRJzdtPEGRtIEYHVEXFpVPhY4MyJqdlIk3QxcGhHtNeoOAnaJiMvXR8x9RdI7gJMj4ot9HctrRdLR\nwBERMb5BG3/RcA5/0XAWf0FlHucpj/OUx3nK4zzlcZ7qU50vGh6oHayRwFXAS1XfhbXBa7WDJWkI\nxTTPBRFx7GsXqb3WVNx+//3AlyPi7gbt3MHqLyYO/A6WmZnZ69XrqoNlZuuPJP9R6CeG7TCMlb9d\n2byhmZmZvebqdbBy1mCZ2QYmIvzT5Ke9vX29n+P10Lny3P08zlMe5ymP85THecrjPLXOHSwzMzMz\nM7Ne4imCZtaFpPDfBTMzM7PGPEXQzMzMzMxsPXMHy8ysBzwnPY/zlMd5yuM85XGe8jhPeZyn1rmD\nZWZmZmZm1ku8BsvMuvAaLDMzM7PmvAbLzLJJqvkzfPjOfR2amZmZWb/mDpaZ1RA1f5599sk+jao/\n8Zz0PM5THucpj/OUx3nK4zzlcZ5a17CDJWmEpAfr1LVLGr1+wmpM0k6SHpB0e6lsRV/EUo+ksZKm\nZrTrV3GX5cTWrI2kCZLO6L2ouh5T0lRJYzL2+a6kRyQtlDQqo327pJ2a1Lf0+pc0TtISSXenx9ek\neD6fruPIJvs3vVZJx0lalH7ulbR7qe4SSQ9LGttK3GZmZmaWb6OMNv1xMcbhwIyIOLtU1h/jzImp\nP8ZdMdDjB0DSx4CREfFWSe8BfgDs2wehnAKcGhH3SRoO7B0Rb00xNu2MZ3ocGBMRf5D0UeBK0rVG\nxJmS5gL/DMzspfNtsNra2vo6hAHBecrjPOVxnvI4T3mcpzzOU+typghuLGla+uT9OkmbVjeQNF7S\n4vRzYSoblD5xX5w+Tf98Kh8p6c70yf39knbpQdxbA89VlT1fiufEdM4Fkq5OZVMlTZY0W9KjldEC\nSVtIuivFskjSoal8hKSlab/lkqZL+lDaf7mkvVO7zSVNkTRH0nxJh6Qw/gr8IeNank/HGS5pZhqZ\nWyxpv1S+WtL5KV/3SXpTKj+4dM4ZpfIJkn6c2i6XdGoqH5uOf4ukZZKuUOFkSf9eyt2pki6pzmmz\n+OvlvUzS30u6XdK8FMvbJG0l6YlSm80lPSVpcK32Nc6/iiLXjRwG/BggIn4NDJE0rMk+vwfW1Hsd\nJ0dL+nXKZ+X5OknSZaXruVnSGElfA/YHpkj6FnAHsEN6vvevytNoSR3pum8vxdr0WiNiTkRUXndz\ngB2qmqyk+PdjZmZmZutBTgdrV+DyiNgNWA18plwpaTvgQqANGAXskzopo4AdImL3iNgDqHxCPx24\nLCJGAe8D/qcHcQ8GOssFEfGeFM9uwFeAtojYEyi/IR4eEfsBhwAXpbK/AIdHxN7AgcAlpfYjgW9H\nxK4pD8em/c9K5wA4B7g7IvZN+18sabOI+FVEnJ5i2kvSlbUupBI3cBzwy4gYDewBLEzlWwD3pXzN\nAj6ZymdFxL4RsRdwLfCl0mHfRfF8vA/4uorREoB9gM8Cbwf+ATgCuA44RNLg1OZk4D+qYqsrM+8V\nVwL/GhH7UOTw+xHxR2CB1k1bOzjlYU2t9jXOf3pEzEkxTJJ0cI3z7gA8XXr8DN07HtXHHRcRz1D/\ndQwwOF3/6cDE8u41jncecD9wXER8CTgUeDQiRkfEvZV2kjYCLgM+ka57KvCNFq617FTg9qqyTop/\nP/YqeU56Hucpj/OUx3nK4zzlcZ7yOE+ty5ki+FTlTR0wDfgccGmpfh+gPSJeAJA0HRgDnA/sImky\ncBswQ9KWwPYRcRNARDQbeehGkig6INPqNDkQuD4iXkznWFWq+0UqWyrpzZVDAt9UsbalE9i+VLci\nIpak7YeBu9L2g8DOafvDFB2Us9LjTYCdgOWVk0bEfOBTTS5tHsXoxsbAjRGxKJW/HBG3pe35wAfT\n9o6SrgO2AzYGymuhbky5/b2ke4B3U4ymzY2IJ6FY/wPsHxE3qFgTdLCkZcBGEfFwk1hraZR3JG1B\n0eG7Pj2HpLih6OQdQzFt7Vjge03a1xQRE3oQdzOPU/U6LtXdkH7PB0ZkHq/brTyr7Aq8E7gzXfcg\n4HfVjZpdq6QDKDrL+1dVPQO8TdIbIuLl+keYWNpuSz9mZmZmG66Ojo6sDmdP1mDVWnPT7U1jRKyS\ntAfwEeDTwFHAF2q17XIg6TMUozQBfDwiVpbqBlG84X0ZuDUj9mrlN5SVOI4H3gjsGRGdKm7asGmN\n9p2lx52sy50oRhse6UE8a0XErNTJOwi4StIlETENeKXUbE3pvJcBF0fErWn0p/yGu/wcifrrpCrl\nUyhGn5bRdYSmNw0CXkwjdNVuAi6QtA0wGrgH2LJB+1Y9A+xYevyWVNZUndfxqam68nooPy9/o+vI\ncLcptU0IeCiNlPaIihtbXAl8tNLhrYiIxyUtBZ6U9IH6nemJPT39BsNz0vM4T3mcpzzOUx7nKY/z\nlMd5Wqetra1LPiZNmlSzXc4UwREqbgwAxTS2WVX1c4ExkrZN08zGAzMlDaWYQvVz4KvA6Ih4CXha\n0mEAkjaRtFn5YBFxRUTsmaZOrayq64yInSmmWh1TJ957gKMkbZvOsU2ddpUO1hDgudS5OoCuIxHN\nRhugWEtz2todMu5QVzOY4o51z0XEFOBHFB2NRjFsxbqRjZOq6g5LuR0KjKUYHYNi+uaI1FE9BrgX\nICLmUnRAxgPX1IlvaZNLaJj3iFgNrJA0rnTM3VPdnyie08nALVGo274HbgJOTMfYF1gVEc+mx3el\naa411Xod12uafj8BjFJhR4rRw7qHr1G2HHhTihNJG6Xpl1nS6+hnwD9GxGM16ncHdqEYSe7JSKWZ\nmZmZNZDTwVoGfFbSEorF8T9I5QGQOkFnAx3AAmBeRNxMscalQ9IC4CepDRRvdE+TtAiYDTS72UAt\nvwG2rVWRpvRdQNHJW8C6NVX1RuKmU3Q8FgEnAEtrtKm1f8V5FDcCWazilvbnVjdotAarpA1YJOkB\n4GjgO03OOwn4L0nz6H4zisUUz8d9wLmljur9wOUU0x0fS52GiuuA2aUbJJTjH9ok9kZ5LzsBOEXF\nDTseoliHVHEtxWjiT0tlxzdo3029dUlpiuUKSY8CPyStI0xT8EYCLzQ4bL3Xcc3XU0TMpuhkPUzx\nHM6vblPncWX/V4BxwEWSFlL8m3pv7rUCX6P4t3GFipuNzK2q3wZ4IiI6u+9qrfCc9DzOUx7nKY/z\nlMd5yuM85XGeWqeIfn+X7W7SeqehVbdpN4q7CAKrI+LSqvKxwJkRUbOTIulm4NKIaK9RdxCwS0Rc\nvj5i7iuS3gGcHBFf7OtYXiuSjgaOiIjxDdpE/X69GIh/M9aHjo4OT5vI4DzlcZ7yOE95nKc8zlMe\n56k+SUREtxlJA7WDNRK4CngpIj7Wx+H0K612sCQNoZjmuSAijn3tIrXXmorb778f+HJE3N2gnTtY\nZmZmZk28rjpYZrb+uINlZmZm1ly9DlbOGiwz2+Co5s+wYbl3o3/985z0PM5THucpj/OUx3nK4zzl\ncZ5al3ObdjPbwHiUyszMzKxnPEXQzLqQFP67YGZmZtaYpwiamZmZmZmtZ+5gmZn1gOek53Ge8jhP\neZynPM5THucpj/PUOnewzMzMzMzMeonXYJlZF16DZWZmZtac12CZmZmZmZmtZ+5gmVk3kmr+DH/L\n8L4Ord/wnPQ8zlMe5ymP85THecrjPOVxnlrn78Eys+4m1i5+duKzr2kYZmZmZgNNwxEsSSMkPVin\nrl3S6PUTVmOSdpL0gKTbS2Ur+iKWeiSNlTQ1o12/irssJ7ZmbSRNkHRG70XV9ZiSpkoak7HPdyU9\nImmhpFEZ7dsl7dSkvqXXv6RxkpZIujs9vibF8/l0HUc22f9VXaukSyQ9LGlsK3FbbW1tbX0dwoDg\nPOVxnvI4T3mcpzzOUx7nqXU5UwT742r3w4EZEfGxUll/jDMnpv4Yd8VAjx8ASR8DRkbEW4F/AX7Q\nR6GcApwaER+QNBzYOyJGRcTk3jpBo2uNiDOBc4F/7q3zmZmZmVlXOR2sjSVNS5+8Xydp0+oGksZL\nWpx+Lkxlg9In7oslLZL0+VQ+UtKd6dP1+yXt0oO4twaeqyp7vhTPiemcCyRdncqmSposabakRyuj\nBZK2kHRXimWRpENT+QhJS9N+yyVNl/ShtP9ySXundptLmiJpjqT5kg5JYfwV+EPGtTyfjjNc0sw0\nMrdY0n6pfLWk81O+7pP0plR+cOmcM0rlEyT9OLVdLunUVD42Hf8WScskXaHCyZL+vZS7UyVdUp3T\nZvHXy3uZpL+XdLukeSmWt0naStITpTabS3pK0uBa7WucfxVFrhs5DPgxQET8GhgiaViTfX4PrKn3\nOk6OlvTrlM/K83WSpMtK13OzpDGSvgbsD0yR9C3gDmCH9HzvX5Wn0ZI60nXfXoq1N651JcW/H3uV\nPCc9j/OUx3nK4zzlcZ7yOE95nKfW5XSwdgUuj4jdgNXAZ8qVkrYDLgTagFHAPqmTMgrYISJ2j4g9\ngMp0uenAZRExCngf8D89iHsw0FkuiIj3pHh2A74CtEXEnkD5DfHwiNgPOAS4KJX9BTg8IvYGDgQu\nKbUfCXw7InZNeTg27X9WOgfAOcDdEbFv2v9iSZtFxK8i4vQU016Srqx1IZW4geOAX0bEaGAPYGEq\n3wK4L+VrFvDJVD4rIvaNiL2Aa4EvlQ77Lorn433A11WMlgDsA3wWeDvwD8ARwHXAIZIGpzYnA/9R\nFVtdmXmvuBL414jYhyKH34+IPwILtG7a2sEpD2tqta9x/tMjYk6KYZKkg2ucdwfg6dLjZ1JZo+sa\nFxHPUP91DDA4Xf/pdF211G1ULyLOA+4HjouILwGHAo9GxOiIuLfSTtJGwGXAJ9J1TwW+0YvX2knx\n78fMzMzM1oOcm1w8VXlTB0wDPgdcWqrfB2iPiBcAJE0HxgDnA7tImgzcBsyQtCWwfUTcBBARzT6N\n70aSKDog0+o0ORC4PiJeTOdYVar7RSpbKunNlUMC31SxtqUT2L5UtyIilqTth4G70vaDwM5p+8MU\nHZSz0uNNgJ2A5ZWTRsR84FNNLm0exejGxsCNEbEolb8cEbel7fnAB9P2jpKuA7YDNgbKa6FuTLn9\nvaR7gHdTjKbNjYgnoVj/A+wfETeoWBN0sKRlwEYR8XCTWGtplHckbUHR4bs+PYekuKHo5B0DzASO\nBb7XpH1NETGhB3E38zhVr+NS3Q3p93xgRObxun1XQpVdgXcCd6brHgT8rrrRq7jWZ4C3SXpDRLxc\nt1V7aXtnoCfjzK9znpOex3nK4zzlcZ7yOE95nKc8ztM6HR0dWSN6OR2s6k/ja6256famMSJWSdoD\n+AjwaeAo4Au12nY5kPQZilGaAD4eEStLdYMo3vC+DNyaEXu18hvKShzHA28E9oyIThU3bdi0RvvO\n0uNO1uVOFKMNj/QgnrUiYlbq5B0EXCXpkoiYBrxSaramdN7LgIsj4tY0+lN+w11+jkT9dVKV8ikU\no0/L6DpC05sGAS+mEbpqNwEXSNoGGA3cA2zZoH2rngF2LD1+Syprqs7r+NRUXXk9lJ+Xv9F1ZLjb\nlNomBDyURkp7ouG1RsTjkpYCT0r6QN3O9AE9PLuZmZnZ61RbW1uXDuekSZNqtsuZIjhCUnka26yq\n+rnAGEnbpmlm44GZkoZSTKH6OfBVYHREvAQ8LekwAEmbSNqsfLCIuCIi9kxTp1ZW1XVGxM4UU62O\nqRPvPcBRkrZN59imTrtKB2sI8FzqXB1A15GIZqMNUKylOW3tDhl3qKsZTHHHuuciYgrwI4qORqMY\ntmLdyMZJVXWHpdwOBcZSjI5BMX1zROqoHgPcCxARcynelI8HrqkT39Iml9Aw7xGxGlghaVzpmLun\nuj9RPKeTgVuiULd9D9wEnJiOsS+wKiKeTY/vStNca6r1Oq7XNP1+Ahilwo4Uo4d1D1+jbDnwphQn\nkjZK0y9z1b3WVLY7xXjU9j0cqbTEc9LzOE95nKc8zlMe5ymP85THeWpdTgdrGfBZSUsoFsdX7koW\nAKkTdDbQASwA5kXEzRTrPjokLQB+ktpA8ebvNEmLgNlAs5sN1PIbYNtaFWlK3wUUnbwFrFtTVW8k\nbjpFx2MRcAKwtEabWvtXnEdxI5DFKm5pf251g0ZrsEragEWSHgCOBr7T5LyTgP+SNI/uN6NYTPF8\n3AecW+qo3g9cTjHd8bHUaai4DpgdEd1uzJE6GQ01yHvZCcApKm7Y8RDFOqSKaylGE39aKju+Qftu\n6q1LSlMsV0h6FPghaR1hmoI3EnihwWHrvY5rvp4iYjZFJ+thiudwfnWbOo8r+78CjAMukrSQ4t/U\ne1/ttZZsAzwREZ3V+5qZmZnZq6eIfn+X7W7SeqehEXF208YbGEkTgNURcWlV+VjgzIio2UmRdDNw\naUS016g7CNglIi5fHzH3FUnvAE6OiC/2dSyvFUlHA0dExPgGbaLeFw0zEQbi3wwzMzOz3iaJiOg2\nIylnBKs/ugHYT6UvGraekTRE0nLgT7U6VwARcevrrXMFEBEPb2Cdq0uAL1JMQTUzMzOz9WBAjmCZ\n2fojqe4fhWE7DGPlb1fWq96gdHR0+M5KGZynPM5THucpj/OUx3nK4zzVV28EK+cugma2gfEHL2Zm\nZmY94xEsM+tCUvjvgpmZmVljr7c1WGZmZmZmZv2OO1hmZj3g7wXJ4zzlcZ7yOE95nKc8zlMe56l1\n7mCZmZmZmZn1Eq/BMrMuvAbLzMzMrDmvwTIzMzMzM1vPfJt2M+tG6vZhjJmZbQA2pO879Pc75XGe\nWucOlpl1N7GvAxgAVgC79HUQA4DzlMd5yuM85XkVeXp24rO9GorZhshTBK3PSVrd1zFUSBot6UFJ\nU0plK/oolj0kfaz0+CRJEzL2u13Si5JuqiofL2mZpNPXR7wbHL/Jy+M85XGe8jhPeZynLB6VyeM8\ntc4dLOsP+tMdFU4AvhcRp5TKWopPUm/9uxoFfLyqLCeWb1FcR9cdI64BxgLuYJmZmZmtJ+5gWb8h\naZKkBZIekPRbSVMkjZC0VNJUScslTZf0IUmz0+O90777SLpP0nxJ90p6aw/D2Bp4rqrs+XSOsZJm\nSroljQRdUYp9taSLJS0A9k0jYR2S5qURpWGp3WmSHpa0UNJ/prLN07XOSfEfImlj4Fzg6JSPo4D/\nBV5qdgER0V6vXUQ8CwxpOSvWXZ+Maw5AzlMe5ymP85THecri73fK4zy1zmuwrN+IiAnABElDgP8G\nLktVI4FPRMQSSfcDx0bEfpIOBc4BjgCWAvtHRKekDwDfBMb1IIzBQGdVXO8pPdwHeDvwFHCHpCMj\n4gZgC+BXEfFFSRsBM4FDI+L3ko4GvgGcAvwbsHNEvCJpq3TMc4C7I+KUdO1zgbuArwN7RcRp1UFK\nOiTVTezBNfqDFTMzM7P1xB0s64+mAZdExEJJI4AVEbEk1T1M0fkAeBAYkba3Bn6cRq6CHry2U8fo\nHazr2NUyNyKeTO2vAfYHbgDWpN8AuwLvBO5UcTu+QcDvUt0i4D8l/QL4RSr7MHCIpLPS402AnRrF\nGhE3AzfnX10XL0gaGRGP1W3RXtreGc/nr8U5yeM85XGe8jhPeZynLF5blMd5WqejoyNrRM8dLOtX\nJE0EnoqIH5eKXy5td5Yed7LuNXwecE9EHJk6ZeUuQuXY5wMHARERo6vq3kIxcvRoRNzfIMTqNVCV\nx38ufTuvgIciYr8a+x8EjAEOBc6R9K7U/hMR8UhVTPs2iOPVmAwslPS5iLiqZosD1tOZzczMzAao\ntra2Lh3OSZMm1WznqULWHwjWTnv7IPD5WvVNDAGeSdsn12oQEV+NiD2rO1ep7rfADkUYamtwnnen\ndWGDgGOAWTViXA68qdJBkrSRpN1S3U4RMRM4G9iKYmrhHcDaaYCSRqXN1alNT4j6efsK8A91O1eW\nx2sc8jhPeZynPM5THucpi9cW5XGeWucOlvUHlZGf04HtgXnpxg4Tq+qrt8u+BVwoaT49fF2nEahH\ngW0bNLsfuJxiquJjEVGZ5rc2roh4hWL910WSFgILgPemKYjTJC0C5gOTI+KPFKNvG0taLOlBiptb\nQDEKt1vpJhdrpRthTKwVoKT/Bq4FDpT0lKQPVTXZJN3swszMzMx6mdbNajIzSd8DHoyIH9SoGwuc\nGRGHvvaR9Q5JbwYWRcR2DdqEv2jYzGwDNRH83tAsjyQiotuMIY9gmXX1Y+Dk8hcNv15IGg/MoBjt\nMzMzM7P1wCNYZtaFJP9RMDPbQA3bYRgrf7uyr8N4TXR0dPgOeRmcp/rqjWD5LoJm1o0/eGnO/+Hk\ncZ7yOE95nKc8zpNZ3/IIlpl1ISn8d8HMzMysMa/BMjMzMzMzW8/cwTIz6wF/L0ge5ymP85THecrj\nPOVxnvI4T61zB8vMzMzMzKyXeA2WmXXhNVhmZmZmzXkNlpmZmZmZ2Xrm27SbWTdStw9jzMzMzAac\nvvhuN08RNLMuJAUT+zqKAWAFsEtfBzEAOE95nKc8zlMe5ymP85RnoOdp4vr7fk9PETTrZySNkPRg\nC+2/JWmppIWSfiZpqxbPt6uk+yT9RdIZrUdsXQzk/2xeS85THucpj/OUx3nK4zzlcZ5a5g6WWd9q\n5SOVGcA7ImIU8Ajw5RbP9Xvgc8C3W9zPzMzMzDK5g2XWtzaWNE3SEv3/9u49yJK6POP490FQEQUV\nw1Jh5RZBRQV2uYUC4mAUbwHRRAyKAhpjFZRsAhgTk8IlmggYJQQvkYgrohhBUYHyhsAoKyLI7nJH\no1wES1YtAVeNROTNH6cHembOzPYZZvfMst9P1anp/p3uc9556uzsvNO/7k7OTfL4JLslWZ5kWZLr\nkvweoKq+XlUPNvtdCcwf5I2q6udVdQ3wwCx/D+un24ZdwDrCnLoxp27MqRtz6sacujGngdlgScP1\nTOADVbUTsAo4qqquqaoFVbUQ+Ar9jzi9EfjyWqxTkiRJHXgVQWm4flRVVzbLn6Q3he/9AEleAywA\nDmjvkOQfgd9V1TlrrKrLWsvb4vzrfsykG3Pqxpy6MaduzKkbc+rGnB4yOjrK6OjoarezwZKGa+I5\nWAWQ5LnACcB+7bv+JjkCeBnwgn4vluTdwMuBao6Azcz+M95TkiTpUWlkZISRkZGH1k888cS+2zlF\nUBqubZLs1Sy/FliaZDPgHOANVfWLsQ2TvAR4G3BQVd3f78Wq6p9a0wun442uHinnpHdjTt2YUzfm\n1I05dWNO3ZjTwDyCJQ3XLcDRSZYANwAfBg4Btgb+K707/o4djTodeCxwcXMj4Cur6qiub5RkHvBd\n4EnAg0kWATtV1a9m8xuSJElan3mjYUnjeKNhSZL0qLHYGw1LkiRJ0jrLI1iSxkniDwVJkvSoMG+r\nedx9191r5LWnOoLlOViSJvEPL6s3Ojo67kpC6s+cujGnbsypG3Pqxpy6MafBeQRL0jhJyp8LkiRJ\n0/McLEmSJElaw2ywJGkGutzJXebUlTl1Y07dmFM35tSNOQ3OBkuSJEmSZonnYEkax3OwJEmSVs9z\nsCRJkiRpDbPBkjRJEh8+fMzBx5bztxz2j4eh8lyQbsypG3PqxpwG532wJE22eNgFrANuA7YbdhHr\nAHPqpmNOKxevXOOlSJIeGc/BkuaYJLcBu1XVL5Israp9kzwfOL6qDnwEr3sm8GfAyqraeZrtygZL\nmqMWeyNwSZorEs/BktYVD/32VFX79hufoSXAix/ha0iSJGkaNljSkCR5S5LlSZYluTXJJWNPtbZZ\n1dplsyQXJbklyYcGfb+qWgrc8wjL1pjbhl3AOsKcujGnTjwXpBtz6sacujGnwdlgSUNSVR+pqgXA\nnsCdwPv6bdZa3gM4Gng28Iwkr1rzVUqSJGkQXuRCGr7/AC6tqi+tZrurquoOgCSfBvYFzl8jFV3W\nWt4WL1LQj5l0Y07dmFMnIyMjwy5hnWBO3ZhTN+b0sNHR0U5H9GywpCFKcgTw9Ko6qsPmE8/BGree\nZE/gI834CVV10YwL23/Ge0qSJD0qjYyMjGs4TzzxxL7bOUVQGpIkuwHHAYdNt1lrea8k2yTZAHgN\nsLS9YVVdVVULqmrhNM1VJrymZspzZroxp27MqRPPBenGnLoxp27MaXA2WNLwHA08BbisudDFGc14\n+8hUe/kq4APAjcAPq+rzg7xZknOAK4Adk/woyZEzL12SJEn9eB8sSeN4HyxpDlvsfbAkaa7wPliS\nJEmStIbZYEnSTHjOTDfm1I05deK5IN2YUzfm1I05Dc6rCEqabPGwC5DUz7yt5g27BEnSangOlqRx\nkpQ/FyRJkqbnOViSJEmStIbZYEnSDDgnvRtz6sacujGnbsypG3PqxpwGZ4MlSZIkSbPEc7AkjeM5\nWJIkSavnOViSJEmStIbZYEnSDDgnvRtz6sacujGnbsypG3PqxpwGZ4MlaZIkfR9bzt9y2KVJkiTN\naZ6DJWmcJDXljYYXgz8zJEmSPAdL6iTJg0ne21o/LskJQ6plm6aeo1tjpyd5wzDqkSRJ0urZYEnj\n3Q+8KslTh11I46fAoiQbDrsQjeec9G7MqRtz6sacujGnbsypG3ManA2WNN4DwBnAsROfaI4oXZJk\nRZKLk8xvxpckOS3Jt5L8IMmrWvscn+SqZp8OkRXqAAAKwklEQVR3zqCenwGXAEf0qWfXJN9uXvtz\nSTZrxi9LclKS7yS5Jck+zfgGSU5pxlckefMM6pEkSdI0bLCk8Qr4IPC6JE+a8NzpwJKq2hU4p1kf\ns2VV7QMcCJwMkORFwA5VtSewANg9yb4zqOdk4PgkE+f4ngW8rannBqDdwD2mqvYC/hYeOqPqTcC9\nzfiewF8n2WbAetQYGRkZdgnrBHPqxpy6MaduzKkbc+rGnAbntCNpgqr6VZKzgEXA/7ae2ht4ZbN8\nNk0j1fhCs+/NSbZoxg4AXpRkGRBgE2AHYOmA9dye5ErgdWNjSTYFNquqsdc6Czi3tdv5zddrgLEm\n6gDgeUle3axv2tRzx6Q3vay1vC2w3SAVS5IkPfqMjo52mjLpESypv9PoHfHZpDU23eXz7m8tp/X1\nPVW1sKoWVNWOVbWkvVOSg5MsT7IsycJpXv89wNsnjE26ak2fen7Pw39ICfDWppYFVfVHVfX1vnvv\n33rYXPXlnPRuzKkbc+rGnLoxp27MqRtzetjIyAiLFy9+6DEVGyxpvABU1T30jgi9qfXcFcChzfJh\nwOXTvQbwVeCNSTYBSPKHSf6gvWFVfaFpdhZW1bJp6vkecBNwULP+S+AXY+dXAa8HvtGhnqPGLpiR\nZIckG0+xjyRJkmbA+2BJLUl+WVWbNstbALcCJ1fVu5JsDSwBNqd38Ykjq+quJB8DLqqq8/u8xluB\nsYtJrAIOq6rbOtayDXBhVe3crO8MLAPeWFWfSLIL8J/Axk2dR1bVfUkuBY6vqmVJNgeurqrtm3O4\n3k3vPLHQu0LhwVW1asL7eh8sSZKk1ZjqPlg2WJLGscGSJElaPW80LEmzyDnp3ZhTN+bUjTl1Y07d\nmFM35jQ4GyxJky3u/5i31bwhFTT3rFixYtglrBPMqRtz6sacujGnbsypG3ManJdplzSJ0wBX7957\n7x12CesEc+rGnLoxp27MqRtz6sacBucRLEmSJEmaJTZYkjQDt99++7BLWCeYUzfm1I05dWNO3ZhT\nN+Y0OK8iKGmcJP5QkCRJ6sDLtEuSJEnSGuQUQUmSJEmaJTZYkiRJkjRLbLAkSZIkaZbYYEkCIMlL\nktyS5PtJ3j7seuaSJGcmWZnkutbYU5J8Lcn3knw1yWbDrHHYksxPcmmSG5Ncn+SYZtycWpI8Lsl3\nkixvsvrXZtyc+kiyQZJlSS5o1s2pjyS3J7m2+Vxd1YyZ1QRJNktyXpKbm39/e5nTeEl2bD5Hy5qv\n9yU5xpwGY4MliSQbAB8AXgw8Bzg0ybOGW9WcsoReNm1/D3y9qp4JXAr8w1qvam55ADi2qp4D7A0c\n3XyGzKmlqu4H9q+qBcDOwAuS7IM5TWURcFNr3Zz6exAYqaoFVbVnM2ZWk50GfKmqng3sAtyCOY1T\nVd9vPkcLgd2AXwOfx5wGYoMlCWBP4H+q6o6q+h3w38ArhlzTnFFVS4F7Jgy/AjirWT4LOHitFjXH\nVNXdVbWiWf4VcDMwH3OapKp+0yw+jt7/w/dgTpMkmQ+8DPhoa9ic+guTf6czq5YkmwL7VdUSgKp6\noKruw5ym80Lgh1V1J+Y0EBssSQBbAXe21u9qxjS1LapqJfSaC2CLIdczZyTZFtgVuBKYZ07jNdPe\nlgN3A6NVdRPm1M+pwNuA9v1kzKm/Ai5OcnWSv2rGzGq87YCfJ1nSTH87I8kTMKfpvAY4p1k2pwHY\nYEnS7PCmgkCSJwKfBRY1R7Im5rLe51RVDzZTBOcD+yUZwZzGSfJyYGVzVHTSTTxb1uucWvZppnS9\njN703P3wMzXRhsBC4INNVr+mN+3NnPpIshFwEHBeM2ROA7DBkgTwY2Dr1vr8ZkxTW5lkHkCSLYGf\nDrmeoUuyIb3m6uyq+mIzbE5TqKpfAl8CdsecJtoHOCjJrcCn6Z2rdjZwtzlNVlU/ab7+DPgCvWnf\nfqbGuwu4s6q+26x/jl7DZU79vRS4pqp+3qyb0wBssCQBXA08I8k2SR4L/CVwwZBrmmvC+L+kXwAc\n0SwfDnxx4g7roY8BN1XVaa0xc2pJ8rSxq28l2Rh4EbAccxqnqt5RVVtX1fb0fh5dWlWvBy7EnMZJ\n8oTmyDFJNgEOAK7Hz9Q4zfS2O5Ps2Az9KXAj5jSVQ+n9cWOMOQ0gVR7hk9S7TDu9KyxtAJxZVScN\nuaQ5I8k5wAiwObASeCe9vxKfBzwduAM4pKruHVaNw9ZcCe+b9H6xq+bxDuAq4FzMCYAkz6N3gvjY\nRQnOrqp/S/JUzKmvJM8Hjquqg8xpsiTb0bvKW9GbBvepqjrJrCZLsgu9i6ZsBNwKHAk8BnMapzk3\n7Q5g+6pa1Yz5eRqADZYkSZIkzRKnCEqSJEnSLLHBkiRJkqRZYoMlSZIkSbPEBkuSJEmSZokNliRJ\nkiTNEhssSZIkSZolNliSJGmdkuSiJJuuxffbJclLW+sHJvm7tfX+ktYt3gdLkiStFUkeU1W/H3Yd\n/UxXW5LDgd2r6q1ruSxJ6yCPYEmStJ5J8oYk1yZZnuSsZmybJJckWZHk4iTzm/ElST6U5NtJfpBk\nJMnHk9yU5GOt11yV5P1Jbmj237wZvyzJqUmuBo5J8rQkn03yneaxd7Pd85t6liW5JskmSbZM8o1m\n7Lok+zTb3pbkqc3ysUmub55f1PpebkpyRlPPV5I8rk8OS5J8OMmVwMlJ9khyRfP+S5PskGQj4J+B\nQ5o6Xp3k8CSnT5ebpPWXDZYkSeuRJDsB7wBGqmoBsKh56nRgSVXtCpzTrI95clXtDRwLXACcUlU7\nATsn2bnZZhPgqqp6LvBN4J2t/Teqqj2q6lTgNOD9VbUX8BfAmc02xwFHVdVCYD/gt8Brga80Y7sA\nK5ptq/leFgKHA3sAewNvTrJLs80zgNObeu4D/nyKSLaqqj+uquOBm4F9q2q3pv73VNXvgBOAz1TV\nwqo6r13DanKTtB7acNgFSJKkteoFwHlVdQ9AVd3bjO8NvLJZPhs4ubXPhc3X64GfVNVNzfqNwLbA\ndcCDwLnN+CeBz7X2/0xr+YXAs5OkWX9ikicA3wJOTfIp4Pyq+nFz1OvM5ijSF6vq2gnfy77A56vq\ntwBJzqfXnF0I3FZV1zfbXdPU2c95reUnA59IsgO9BqrL70kTczulwz6SHsU8giVJkuDhIzL93N98\nfbC1PLY+VRPSfr1ft5YD7FVVC5rH1lX1m6o6GXgTsDHwrSQ7VtXlwJ8APwY+nuSwAb6fdp2/n6bO\ndm3vAi6tqucBBwKP7/A+E3Pz5HZpPWeDJUnS+uVS4NWtc5ie0oxfARzaLB8GXD7F/plifAN6U/4A\nXgcsnWK7r/HwtETGpvQl2b6qbqyqU4CrgWcl2Rr4aVWdCXwUWDihhsuBg5M8Pskm9I4kXT5hm0Fs\nSq+ZAziyNb6qea6frrlJWk/YYEmStB5ppvf9C/CNJMuB9zVPHQMcmWQFvQZprAma7gjNxKNUeya5\nHhihd2GIfvsvAnZvLrJxA/CWZvxvmotVXAv8H/Dl5nWuTbIMOAT49/ZrVtVy4OP0GrJvA2e0phF2\nOZI0cZv3AicluYbxvyNdBuw0dpGLCftMlZuk9ZSXaZckSY9YklVV9aRh1yFJw+YRLEmSNBv8i60k\n4REsSZIkSZo1HsGSJEmSpFligyVJkiRJs8QGS5IkSZJmiQ2WJEmSJM0SGyxJkiRJmiX/DxZWyxUh\naxJjAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "ratios = compression_ratios() \n", - "labels = ['%s - %s' % (c, o)\n", - " for c, o in compression_configs]\n", - "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Compression ratio', fontsize=14, y=1.01)\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", - "x = [ratios[i] for i in y]\n", - "ax.barh(bottom=np.array(y)+.2, width=np.array(x), height=.6, label='bit shuffle', color='b')\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", - "x = [ratios[i] for i in y]\n", - "ax.barh(bottom=np.array(y)+.2, width=np.array(x), height=.6, label='no shuffle', color='g')\n", - "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", - "ax.set_yticklabels(labels, rotation=0)\n", - "\n", - "ax.set_xlim(0, max(ratios)+3)\n", - "ax.set_ylim(0, len(ratios))\n", - "ax.set_xlabel('compression ratio')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", - "\n", - "fig.tight_layout();\n" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "@functools.lru_cache(maxsize=None)\n", - "def compression_decompression_times(repeat=3, number=1):\n", - " c = list()\n", - " d = list()\n", - " for compression, compression_opts in compression_configs:\n", - " \n", - " def compress():\n", - " zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", - " \n", - " t = timeit.Timer(stmt=compress, globals=locals())\n", - " compress_times = t.repeat(repeat=repeat, number=number)\n", - " c.append(compress_times)\n", - " \n", - " z = zarr.array(genotype_sample, chunks=chunks, compression=compression, \n", - " compression_opts=compression_opts)\n", - " \n", - " def decompress():\n", - " z[:]\n", - " \n", - " t = timeit.Timer(stmt=decompress, globals=locals())\n", - " decompress_times = t.repeat(repeat=repeat, number=number)\n", - " d.append(decompress_times)\n", - " \n", - " log(compression, compression_opts, compress_times, decompress_times)\n", - " \n", - " return c, d\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAMWCAYAAADszSe0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcVcWZ//HPlwaCoAhowqYsEjEuAWk3FAIoanSiSAwi\niMsoxjjmF3FLhnGjiRE1I05MoiZmEDUaI86QDOCggtI4GgnKqoCorIriwqKiEZV+fn+cunD69l2b\nPrdbeN6v13n1vafqVNV57hFP3ao6V2aGc84555xzzrmd16i+G+Ccc84555xzuwrvYDnnnHPOOedc\nHfEOlnPOOeecc87VEe9gOeecc84551wd8Q6Wc84555xzztUR72A555xzzjnnXB3xDpZzzrmvFEmz\nJP26vttRXyRdLWlVfbfDOedcZt7Bcs65BkzSNyTdKekNSZ9JelPS45JOre+21aPvA/9W342oZ/4j\nls4510A1ru8GOOecy0xSZ+BvwIfAvwKLib4YOxG4B+hSb43LQlITM/siyTrMbHOS5TvnnHM7w0ew\nnHOu4boHqAKOMLP/NrPXzWy5md0F9EhlkrS/pL9I+ihs/y2pYyx9jKSXJZ0vaZWkLZImSGoi6Sdh\nVOwDSf8erzzkHSPpj5I+lvSOpKvT8lRJuizUuQW4Oew/RNK00J53Jf1JUtvYcYdJminpw1D2Akn9\nQ1pjSb+WtC6M2q2RNC52bLUpgpJaSXpA0kZJn0qaIemQWPoFoY4TQhy2SHomdGCzkvQjScsl/UPS\n+5KmS2oU0iZKmirpOknrQ/n3SfpaWhk/C6OPn0paJGlEWnoHSX8Obd8YYvbNDGW8E2J5P7BnrnY7\n55yrX97Bcs65BkhSa+C7wG/N7B/p6Wb2UcgnYArwdaA/MADoAPwl7ZAuwCDge0RT7IYCjwOHE42I\njQRGSToj7bgrgSVAL+BGYJykwWl5bgxlHQbcJakdMJtoxO1IYCDQAvif2DF/At4O6T2BCuCzkDYK\nOCO08ZvA2cDymlHa7gHgKOD08PdT4Im0zs7XgNHAPwO9gVbA77IVKOkI4LfAGKA7cALwRFq2/kQd\n3ROAM4GTgdtiZdwMXAj8C3AwcAvwu9T0Tkl7ALOAT4DvhHa9DcyU1CzkGQrcBNwAlAOvAVfliIVz\nzrn6Zma++eabb741sI2oo1AFnJEn30nAF8D+sX1dgW3ACeH9GKKb+D1jeR4D3gUax/bNAn4de78K\neDKtvj8Az8beVwG/SsszFpiRtq91yHtkeP8hcF6Wc7oz/fi09O3tBA4M5faJpbcENgMXhfcXhHh8\nM5bnHOAfOer4PrAJaJElfSKwEdgjtm8E8A9gD6A5UUevT9px/wFMC68vApanpZcBHwBDwvvngd+l\n5ZkBrKzva9Q333zzzbfMm49gOedcw6QC830LeNvM3kztMLNVRCMhh8TyrTWzLbH37wKvmdmXafu+\nkVb+CxneH5K2b17a+yOA/mHa3MeSPgbWEj2YoVvIcwcwQdLTkq6VdFDs+PuBXpJek/RbSf8URuoy\n+RZR52lOaodFo3svp7Vzq5m9EXv/NtBUUqss5c4A1gCrJT0UplemT81bbNVHF18AmoZzPARoRjSS\nFo/DpcABIX85cEBa+mai0bVUnA6On1usHueccw2UP+TCOecapteJOiQHU31qXTHiT5pLf/CEZdlX\nVot6Pkl73wiYBlxNzY7iuwBmNlbSQ8CpwCnAGEk/MrP7zWxBWB/1XaLphQ8AC4lG64oRP/8vs6Rl\n/KLRzLZIKgf6hXpHE02PPNLM1ueoM3W+qXJPA95My/NFLM8CoimQ6XHamKMO55xzDZiPYDnnXANk\nZpuAJ4H/J6l5erqkvcPLZUAHSZ1iaQcQrcNaUgdN6Z32/thQZy7zgUOJRs1Wpm3bO2NmtsLMfmtm\npwETgItjaZ+Y2WQz+zHRurGB6Q9/CJYR/b/s2NQOSS2Bb7OT529mVWZWaWbXEa0Ta0HUYUr5dlhH\nlXIssBVYASwNr7tkiEGqwzWfaI3Zhgx5Uk9KXEbmz8A551wD5R0s55xruH5MNLLxkqQhkrpLOkjS\nvwCLAMxsJtF0uIclHSHpSOAh4CUzq6yDNvSW9K+Svinph8C5RNP7crkL2BuYJOloSV0lnSjp95Ja\nSGoWpv71l9RZ0jFAX0KHSNKVkoZJ+lboVI0gWrP1VnpFYdrfFOD3kvpK+nY4/w+BR/K0M+s0TEnf\nk3S5pMND53UE0dP7lsayNQbuU/TExJOIHmJxr5n9I0zHvB24XdKFkrpJ6hmeTJjqSD5MNKL3P5L6\nSeoS/t4uKTVF8E7gAkkXh8/g34Cj85yXc865euRTBJ1zroEys1Vhmtq1wK1AR2AD8ApwRSzrIODX\nwDPh/Qzg8jpqxh1ET8q7HtgC3GBm8ScU1vjBWzN7R1Ifog7HdKK1SGuBp4hGdUT00IuJQPtwTlOB\nn4YiPg6vvxnKXwCcamappwym1/nPwK+IplI2A54DTjGzrXnOLdeP9W4GBhM9va850ajUSDP7WyzP\nbKJO4SyiB1v8F9HvlaXicIOk9URTJe8GPiKa6vjLkP4PSf2IPttJRJ3St0N5m0KeSZK6Ar8I7ZgC\njA/n7JxzrgGSmf8YvHPOuZokrQJ+Y2b5Rqx2O5ImAvuY2aD6botzzrmGxacIOuecc84551wd8Q6W\nc865bHyKg3POOVcknyLonHPOOeecc3XER7Ccc84555xzro54B8s555xzzjnn6oh3sJxzzjnnnHOu\njvjvYDnnqpHkCzOdc8455wpgZjV+tN5HsJxzNZiZb3W8XXDBBfXehl1189h6XL9Km8fVY/tV2zyu\n2bdsvIPlnHMl0KVLl/puwi7LY5sMj2syPK7J8dgmw+NaPO9gOeecc84551wd8Q6Wc64GSXWytWvX\npb5PpcFo1apVfTdhl+WxTYbHNRke1+R4bJPhcS2eP+TCOZdB3Tzn4t13a6z73G0dfvjh9d2EXZbH\nNhke12R4XJPjsU2Gx7V4yrVAyzm3+4meIlhX/y4o5yJQ55xzzkXrnNasWVPfzXBZdO7cmdWrV9fY\nLwmrz6cISuos6eUsabMklZeqLWl1d5I0X9L02L5V9dGWbCT1lzSxgHw52y3p4/C3vaRJ4fUFkn5T\nm/IKrHOMpKvylVOMeJmSJkrqlyd/f0mbw+c8X9L1BdQxS1KnPOlFXbOShkhaKunp8P4RSQsljQrn\ncWae4ws513MkLQrbc5J6xNLGS1oiqX8x7XbOOedcstasWVPvT8TzLftWbOe31GuwGuJX2YOBp8zs\n1Ni+htjOQtqUL48BmNk7Zja0gOPqos6G4lkzKw/bL+qpDSOBi81soKR2wJFmdriZ3VmHdawE+plZ\nT+AXwL2pBDO7Gvg5cFEd1ucKVFlZWd9N2GV5bJPhcU2GxzU5HlvXUJS6g9VE0kPhW/xJkpqlZ5A0\nXNLisN0a9jUK394vDt/Mjwr7u0maEUYBXpLUtRZtagW8l7bv/Vh7zg91LpD0QNg3UdKdkp6X9EZq\n5EFSC0kzQ1sWSRoU9neWtCwct1zSw5JOCscvl3RkyNdc0gRJcyTNk3R6aMbnwIcFnMv7oZyxob3z\nJb0laULqdGLtiY8mdgojMssl3ZgpDvnqzBarOEkHSJou6UVJsyV1l9RS0upYnuaS1koqy5Q/Q/2b\nieKTT7GLgTYA27Jde8FQSX+X9KqkPqH91UYEJU2V1E/SDUBfYIKkXwJPAh3DZ9S3WkOlckmV4byn\nS2pb6Lma2RwzS10rc4COaVnWE13zzjnnnHMuAaV+yMVBwIVmNifc9F8G3JFKlNQeuBXoRXQzOSN0\nUt4COppZj5CvZTjkYWCcmU2R1JTadRjLgKr4DjM7JtRzCHAtcKyZbZIUvzFtZ2Z9JB0MTAEmA58B\ng81si6R9iG5wp4T83YAfmNlSSS8Bw8Lxg0IdZwLXAU+b2UhJewNzJc00sxeAF0KbjgB+ZGaXpJ9I\nqt1mNgYYE8p4Fkjd8MdHm+KvjwIODe1/UdI0M5ufKi+XAmOVcm9o+wpJRwP3hNGcBZL6m9ls4DTg\nCTPbJqlGfmBgWv1Xpl5LGgu8aGbTMtR9rKSFwDrgp2a2NM95DQlllpP52gMoM7NjJJ0KVAAnpQ7P\nUN5Nkk4ArjKzBZLuAqaaWXkod2T425jo8xpkZhskDQXGASOLONeUi4HpafuqiK75PCpirweEze2M\nAQMG1HcTdlke22R4XJPhcU2Ox9YlrbKysqCR0lJ3sNaa2Zzw+iHgJ8Q6WEQ3+rPMbCOApIeBfkRT\nnbpKuhP4X+ApSXsCHcxsCoCZFTKKUY0kAT1DWzI5AXjMzDaFOjbH0v4a9i2T9I1UkcAtitbJVAEd\nYmmrYjf1S4CZ4fXLQJfw+mTgdEk/De+bAp2A5alKzWweUKNzlcVDwB1mtjBPvhmpc5M0mWikZX6B\ndaTkihWSWgDHAY+FuAM0CX8nAWcDs4FhwF158mcUOpaZzAM6mdmnoTP0VyDTaFgmK0m79mJpk2Pl\ndy6wvHwjaQcBhxF9uSCiLw3eTs+U41yjSqTjgQuJPsu4dUB3SV8zs63ZS6jI00znnHPOud3LgAED\nqnXkx44dmzFffa/ByrR+p8YNaLhZ7wlUApcCf8iWt1pB0mWxqXLt0tIaAauAg4HHC2p9dfGb01Q7\nRgD7Ar3MrBfR1MNmGfJXxd5XsaOjK6JRrl5h62pmy6kFSRVEHdoaU/UyKORz2VmNgE1hDVTq/A4L\naVOAUyS1BsqBZ/LkL4qZbTGzT8Pr6URTVdsUeGy2aw92fIbb2PEZfkn1/65qTIPNQ8ArsfPumbY+\nMH8B0YMt7iUaBdsUTzOzlcAyYI2kQ4tsm9sJvjYgOR7bZHhck+FxTc6uFtt27brU2e9SZtoK/a3K\nrl278swzz2RMe+655zj44IPr5Hxz1ZPPZ599xumnn06rVq04++yzAbj++uv5+te/TocOHVizZg2N\nGjWiqqoqT0l1o9QdrM6SUtPOzgH+Ly19LtBPUhtJZcBwYHaYbldmZn8BrgfKzWwL8KakMwAkNZW0\nR7wwM7s73KSWm9n6tLQqM+sCvEQ0epLJM8BZqZvx0AHIJNXB2ht4z8yqwghC5wx5cnkSuHz7AVKt\nfnhA0dqtE4FR6UlZDjlJUqsQv8HA8xnKXJan2pyxMrOPgVWShsTK7BHSPiH6HO4Eplkka/5ixdYw\nEaYaKjZKOjNMTc12bI1rL1vW8Hc1cLgi+wNH52pahn3Lga9L6h3qbxymXxZE0ZMP/xs4z8xWZEjv\nAXQlGv1dUmi5zjnnnCutd99dQ/SddzJbVP7O6du3L8uW7bhF3JlO0s74r//6L95//302bdrEo48+\nyptvvskdd9zBq6++yttvRxOBdkyISl6pO1ivAj+WtJRoof3vwv7U0+3WA6OJRgsWEK0xmUq0UL9S\n0gLgjyEPwPnA5ZIWEXUKtt9IF+E1IONoRpjSdzNRJ28BMD7e3njW8Pdh4KjQnnOJRgrS82Q6PuUm\notGVxYoeQvHz9AySjghrk3K5EuhAtJ5qfhjNylXvXKLpbguJpvlVmx4YOhk55YhV3LnASEUPJXkF\nGBRLe5RoBPDPsX0jcuSvQdHDPU7LkDRE0iuhXb8imoaYmiLaDdiYo9hs117Ga8DMnifqZC0Jdc1L\nz5Plfer4L4AhwG2K1owtAI4t4lxvILqe7w6jt3PT0lsDq82sNF/huO18bUByPLbJ8Lgmw+OaHI/t\n7mvNmjV07959eydqzZo17LvvvuyzT95b2GTkeub77rABPwVure92NOQN+B7w/+q7HQmc16HA7fXd\njhKf81DgkTx5DKyONsw555xzuWX6/2Xd/v+49v+P7tKli91yyy12yCGHWJs2beyiiy6yrVu3mplZ\nZWWl7bfffmZmdt5551mjRo2sefPmttdee9m///u/1yjrgw8+sNNOO81atWplbdq0sX79+lWr5/bb\nb7cePXpYq1atbNiwYdvruf/++61v377VypJkK1assDFjxljTpk2tSZMmttdee9nvf/9722OPPays\nrMz22msvu/DCC2316tXWqFEj27Ztm5mZffjhhzZy5Ehr37697bfffnb99ddbVVVVUZ9PbH+Ne6lS\nj2A1RJOBPor90LCrzsweN7Pf1nc76pqZLTGza+q7HaUiaTxwDfCfBeSuk61t20Kf/bHr29XWBjQk\nHttkeFyT4XFNjsc2OX/605+YMWMGK1asYPny5fziFzt+UjQ1avTggw/SqVMnpk2bxkcffcQ119S8\nxRo/fjz7778/GzZs4L333mPcuHHV0h977DGeeuopVq1axaJFi7j//vtr1JP+vqKigmuvvZZhw4bx\n0UcfcckllzB9+nQ6dOjARx99xH333VejHRdccAFNmzZl5cqVLFiwgBkzZvCf/1nA7VGBdvsOlpmt\nMLPvWJEPEnDuq8bMrjazo83s6QLy1sm2fv3qEpyZc84555L0k5/8hA4dOtCqVSuuu+46Hnnkkax5\no4GdzJo0acI777zDqlWrKCsro0+fPtXSR40aRdu2bWnVqhWnn346CxdmfxB2rnpyeffdd5k+fTr/\n8R//QbNmzdh333254oorcp5TsXb7DpZzzpWCrw1Ijsc2GR7XZHhck+OxTc5+++23/XXnzp23Pzii\nWD/72c/o1q0bJ598Mt/85je57bbbqqW3bbvjcQrNmzdny5YttWtwDmvXruWLL76gffv2tGnThtat\nW3PppZfywQcf1Fkdpf4dLOecc84559xXyJtvvrn99Zo1a+jQoUPGfPme1NeiRQtuv/12br/9dpYu\nXcrxxx/P0UcfzfHHH5/3uE8//XT7+/Xr19f6qYD7778/zZo1Y8OGDYk9WdBHsJxzrgR8bUByPLbJ\n8Lgmw+OaHI9tcu666y7WrVvHxo0bGTduHMOGDcuYr127dqxcuTJrOY8//jgrVkS/IrPXXnvRuHFj\nysrK8tbfs2dPlixZwuLFi9m6dWvWH/jNJTWlsF27dpx88slceeWVfPzxx5gZK1eu5Nlnny26zGy8\ng+Wcc84551wDEz0oqm4eOrUzD6KSxDnnnLN9Wt+BBx7IddddlzHv6NGjuemmm2jTpg133HFHjfTX\nX3+dE088kb322os+ffrw4x//mH79+m2vJ5sDDzyQG2+8kYEDB9K9e3e+853vFNT29PNIefDBB/n8\n88855JBDaNOmDWeddRbr16/PcXSRddV2gZhzbtckyfzfBeecc650JNX6oQ0uedk+n7C/Rs/QR7Cc\nc84555xzro54B8s5V4Okndra7deuvk+hwfG1Acnx2CbD45oMj2tyPLauofCnCDrnaqrYucPfrXi3\nTprhnHPOOfdV42uwnHPVSLKd7WBRUfsfAHTOOed2N74Gq2FrsGuwJHWW9HKWtFmSykvVlrS6O0ma\nL2l6bN+q+mhLNpL6S5pYQL6c7Zb0cfjbXtKk8PoCSb+pTXkF1jlG0lX5yilGvExJEyX1y5O/v6TN\n4XOeL+n6AuqYJalTnvSirllJQyQtlfR0eP+IpIWSRoXzODPP8XnPNeT7taTXQ9mHx/aPl7REUv9i\n2u2cc8455wpX6jVYDbFrPhh4ysxOje1riO0spE358hiAmb1jZkMLOK4u6mwonjWz8rD9op7aMBK4\n2MwGSmoHHGlmh5vZnXVVgaRTgW5mdiDwI+B3qTQzuxr4OXBRXdXnCudrA5LjsU2GxzUZHtfkeGxd\nQ1HqDlYTSQ+Fb/EnSWqWnkHScEmLw3Zr2NcofHu/WNIiSaPC/m6SZoRv6l+S1LUWbWoFvJe27/1Y\ne84PdS6Q9EDYN1HSnZKel/RGauRBUgtJM0NbFkkaFPZ3lrQsHLdc0sOSTgrHL5d0ZMjXXNIESXMk\nzZN0emjG58CHBZzL+6GcsaG98yW9JWlC6nRi7YmPJnYKIzLLJd2YKQ756swWqzhJB0iaLulFSbMl\ndZfUUtLqWJ7mktZKKsuUP0P9m4nik0+xP9W9AdiW7doLhkr6u6RXJfUJ7a82IihpqqR+km4A+gIT\nJP0SeBLoGD6jvtUaKpVLqgznPV1S2yLO9QzgQQAz+zuwd+x4gPVE17xzzjnnnEtAqR9ycRBwoZnN\nCTf9lwHbf4VMUnvgVqAX0c3kjNBJeQvoaGY9Qr6W4ZCHgXFmNkVSU2rXYSwDquI7zOyYUM8hwLXA\nsWa2SVL8xrSdmfWRdDAwBZgMfAYMNrMtkvYB5oQ0gG7AD8xsqaSXgGHh+EGhjjOB64CnzWykpL2B\nuZJmmtkLwAuhTUcAPzKzS9JPJNVuMxsDjAllPAukbvjjo03x10cBh4b2vyhpmpnNT5WXS4GxSrk3\ntH2FpKOBe8JozgJJ/c1sNnAa8ISZbZNUIz8wMK3+K1OvJY0FXjSzaRnqPlbSQmAd8FMzW5rnvIaE\nMsvJfO0BlJnZMWHUqAI4KXV4hvJuknQCcJWZLZB0FzDVzMpDuSPD38ZEn9cgM9sgaSgwDhhZ4Ll2\nBN6MvV8X9qWeOlFFdM3nNiv2ugtQm68uXDUDBgyo7ybssjy2yfC4JsPjmhyPrUtaZWVlQSOlpR7B\nWmtmc8Lrh4i+0Y87CphlZhvNrIqoA9UPWAl0DaNG3wU+lrQn0MHMpgCY2edm9lkxjZEkoCdRBy6T\nE4DHzGxTqGNzLO2vYd8y4BupIoFbJC0CZgIdJKXSVsVu6peEdICXiW5hAU4GRktaAFQCTYFq64DM\nbF6mzlUWDwF3mNnCPPlmmNnmEL/J1PxcCpErVkhqARwHPBbO7/dAamRlEnB2eD0MeDRP/ozMbEyW\nztU8oJOZHQ78lvDZFajGtRdLmxwrv7CfQ88/knYQcBjRlwsLiDrdHdIz5TjXfNYB3SV9LWeu42Ob\nd66cc865kmu3X7ud/tmUXFtD+kmV448/nvvuu6/Wx1944YW0adOG3r17A3DPPffQrl07WrZsycaN\nG2nUqBErV67c6XYOGDCAioqK7Vs2pR7BSv9mP9P6nRo3oGa2WVJP4LvApcBZwBWZ8lYrSLoM+GGo\n55/MbH0srRHRzfNW4PEiziFla4Y2jwD2BXqZWZWiB0A0y5C/Kva+ih2fg4hGuV6vRXuqkVRB1KGt\nMVUvg0I+l53VCNiUGrFJMwW4WVJroBx4BtgzR/6imNmW2Ovpku6W1MbMNhZwbKZr7+KQnPoMt7Hj\nM/yS6l9c1JgGm4eAV8ysT5HHpawD9o+93y/sA8DMVkpaBqyRNNDMltSyHlekyspK/3Y1IR7bZHhc\nk+FxTc6uFtt317270z+bkrP8XeQnVZ577jmefvpp3n77bZo1a8aXX37J1Vdfzdy5cznssMOA6Gl/\npVTqEazOklLTzs4B/i8tfS7QT1IbSWXAcGB2mG5XZmZ/Aa4HysNN85uSzgCQ1FTSHvHCzOxuM+sV\nHmywPi2tysy6AC+xY/Qk3TPAWZLahDpaZ8mX+tT2Bt4LnavjqT6qUcgn+yRw+fYDYk+AK4aitVsn\nAqPSk7IccpKkViF+g4HnM5S5LE+1OWNlZh8DqyQNiZXZI6R9QvQ53AlMs0jW/MWKr0EKUw2V6lwp\nWjPXPsexNa69bFnD39XA4YrsDxydq2kZ9i0Hvi6pd6i/cZh+WagpwPnh2N7AZjPb/i9oiGFXotFf\n71w555xz7itt9erVdOnShWbNou+0169fz9atWzn44IO35yn1I/BL3cF6FfixpKVEC+1TTzhLPd1u\nPTCaaHrcAqI1JlOJ1pBUhilTfwx5ILqRvDxMyXuePFPIsngNaJMpIUzpu5mok7cAGB9vbzxr+Psw\ncFRoz7nAsgx5Mh2fchPRg0AWK3oIxc/TM0g6IqxNyuVKomllLyp6iEJFnnrnEk13W0g0zW9+Wp37\n5KkvV6zizgVGKnooySvAoFjao0QjgH+O7RuRI38Nih7ucVqGpCGSXgnt+hXRNMTUFNFuQK6RrGzX\nXsZrwMyeJ+pkLQl1zUvPk+V96vgvgCHAbYrWjC0Aji30XM3sf4k6pm8QTau8LC1La2B1mILrSmhX\n+la1ofHYJsPjmgyPa3I8tsno2rUr48ePp2fPnrRu3Zrhw4fz+ec7nrn1hz/8gQMPPJB9992XwYMH\n884772QsZ+vWrZx33nnsu+++tG7dmmOOOYb339/xPLXVq1fTt29fWrZsySmnnMLGjdHt2ezZs9l/\n//2rldW1a1eeeeYZ7rvvPn74wx/ywgsv0LJlS0aMGMG3vvUtAFq3bs2JJ55Yox2ff/4511xzDZ07\nd6Z9+/ZcdtllbN26tUa+nbHb/9CwpJ8C+5jZ6LyZd1OSvgd0NbPf1ndb6pKkQ4keunJNfbelVBQ9\nNOP7ZjY8Rx7/oWHnnHOuhJThh2wlJTpFsND/V3ft2pW2bdvyP//zP3zta1/juOOO44orruCSSy7h\nmWee4eyzz2bmzJkccsghXH311SxatIjZs2fXKOfee+/l8ccfZ9KkSTRt2pSFCxdy4IEHsueee3L8\n8cfz1ltv8cQTT7DffvtxyimncOyxxzJu3Dhmz57Neeedx9q1a6u1acKECZxwwgk88MADTJgwgWef\nfRaANWvWcMABB/Dll19unxrYqFEj3njjDQ444ACuvPJKVq1axQMPPEDjxo0555xzOOyww7j55puz\nxiDT5xPbX2NGUqnXYDVEk4H7JU1P+y0sF5hZbdaoNXhhitzu1LkaD3wH+Le8mSt2rq62HWszmLxr\n29XWBjQkHttkeFyT4XFNjsc2OaNGjaJt2+j/7aeffjoLF0bPT/vTn/7EyJEj6dmzJwC33HILrVu3\nZu3atXTqVO05bTRp0oQNGzbw2muv8e1vf5tevXpVS7/wwgvp1q0bAEOHDmXq1Kk71WYzy7j26g9/\n+AMvv/wye++9NwCjR49mxIgROTtYxdrtO1hmtoLoptO5XVr4oeFC8ybZFOecc859haQ6VwDNmzff\nPg3w7bff5ogjjtie1qJFC/bZZx/WrVtXo4N1/vnn89ZbbzFs2DA+/PBDRowYwbhx4ygri349pl27\ndtXq2LJlC3Xt/fff59NPP63W5qqqqjq/7yn1GiznnNst+beqyfHYJsPjmgyPa3I8tqXXoUMH1qxZ\ns/39J5/ThHwuAAAgAElEQVR8woYNG+jYsWONvGVlZdxwww0sWbKEv/3tb0ybNo0HH3wwbx0tWrTg\n008/3f5+27Zt1dZuFWPfffelefPmLFmyhI0bN7Jx40Y2b97Mhx9+WKvysvEOlnPOOeecc65ow4cP\nZ+LEiSxevJitW7dy7bXX0rt37xqjVxBN4XzllVeoqqpizz33pEmTJttHr3Lp3r07n332GdOnT+fL\nL7/kF7/4RbWHbGSSbURKEj/84Q+54oortnfS1q1bx1NPPVXA2RZut58i6JxzpeBrA5LjsU2GxzUZ\nHtfk7GqxbduxbaK/VVXoeulcvyE1cOBAbrrpJs4880w2b97Mcccdx5///OeMedevX8+ll17KunXr\n2HPPPRk2bBjnnntu3jpatmzJ3XffzciRI6mqquJnP/sZ++23X1Ftjr+/7bbbGDt2LL17994+2vYv\n//IvnHzyyTnLLMZu/xRB51x1ksz/Xah7u9r/+BsSj20yPK7J8Lgm56sc22xPqXMNQ7FPEfQOlnOu\nGu9gOeecc6XlHayGrdgOlq/Bcs4555xzzrk64h0s51wNknaJrV27LvUdyu0qKyvruwm7LI9tMjyu\nyfC4Jsdj6xoKf8iFcy6DXWOawrvvZl8065xzzjmXBF+D5ZyrRpLtKh0s8DntzjnnGj5fg9WwNdg1\nWJI6S3o5S9osSeWlakta3Z0kzZc0PbZvVX20JRtJ/SVNLCBfznZL+jj8bS9pUnh9gaTf1Ka8Ausc\nI+mqfOUUI16mpImS+uXJP0jSIkkLJL0k6YQC6pglqeaPOFRPL+qalTRE0lJJT4f3j0haKGlUOI8z\n8xxfyLmeE851kaTnJPWIpY2XtERS/2La7Zxzzrlkde7cud6n1fuWfevcuXNRn2ep12A1xK75YOAp\nMzs1tq8htrOQNuXLYwBm9o6ZDS3guLqosyGYaWY9zawXcCFwbz21YyRwsZkNlNQOONLMDjezO+uw\njpVAPzPrCfyC2Lma2dXAz4GL6rA+VyBfG5Acj20yPK7J8Lgm56sc29WrV2NmDXKbNWtWvbehvrfV\nq1cX9XmWuoPVRNJD4Vv8SZKapWeQNFzS4rDdGvY1Ct/eLw7fzI8K+7tJmqFoFOAlSV1r0aZWwHtp\n+96Pted87Rj9eCDsmyjpTknPS3pDYeRBUgtJM0NbFkkaFPZ3lrQsHLdc0sOSTgrHL5d0ZMjXXNIE\nSXMkzZN0emjG58CHBZzL+6GcsaG98yW9JWlC6nRi7YmPJnZSNCKzXNKNmeKQr85ssYqTdICk6ZJe\nlDRbUndJLSWtjuVpLmmtpLJM+TPUv5koPlmZ2aext3sCHxRwXhuAbdmuvWCopL9LelVSn9D+aiOC\nkqZK6ifpBqAvMEHSL4EngY7hM+qbFqdySZXhvKdLSv0SYCHnOsfMUtfKHKBjWpb1RNe8c84555xL\nQKkfcnEQcKGZzQk3/ZcBd6QSJbUHbgV6Ed1MzgidlLeAjmbWI+RrGQ55GBhnZlMkNaV2HcYyoCq+\nw8yOCfUcAlwLHGtmmyTFb0zbmVkfSQcDU4DJwGfAYDPbImkfohvcKSF/N+AHZrZU0kvAsHD8oFDH\nmcB1wNNmNlLS3sBcSTPN7AXghdCmI4Afmdkl6SeSareZjQHGhDKeBVI3/PHRpvjro4BDQ/tflDTN\nzOanysulwFil3BvavkLS0cA9Fo3mLJDU38xmA6cBT5jZNkk18gMD0+q/MvVa0ljgRTObll6xpMHA\nLUA74LsFnNeQcFw5ma89gDIzO0bSqUAFcFLq8Azl3aRoauJVZrZA0l3AVDMrD+WODH8bE31eg8xs\ng6ShwDhgZKHnGnMxMD1tXxXRNZ9HRez1gLC5nfFV/fHLrwKPbTI8rsnwuCbHY5sMj+sOlZWVBY2U\nlrqDtdbM5oTXDwE/IdbBIrrRn2VmGwEkPQz0I5rq1FXSncD/Ak9J2hPoYGZTAMws5zf7mUgS0DO0\nJZMTgMfMbFOoY3Ms7a9h3zJJ30gVCdyiaJ1MFdAhlrbKzJaG10uAmeH1y0CX8Ppk4HRJPw3vmwKd\ngOWpSs1sHlCjc5XFQ8AdZrYwT74ZqXOTNJlopGV+gXWk5IoVkloAxwGPhbgDNAl/JwFnA7OBYcBd\nefJnFDqW2dL+Cvw1jBb9kaizX4iVpF17sbTJ4e88oNDJufkea3cQcBjRlwsi+tLg7fRMuc4VQNLx\nRNMh+6YlrQO6S/qamW3NXkJFnmY655xzzu1eBgwYUK3DOXbs2Iz56nsNVqb1OzVuQMPNek+gErgU\n+EO2vNUKki6LTZVrl5bWCFgFHAw8XlDrq4vfnKbaMQLYF+hl0Xqf94BmGfJXxd5XsaOjK6JRrl5h\n62pmy6kFSRVEHdoaU/UyKORz2VmNgE1mVh47v8NC2hTgFEmtgXLgmTz5a83MngMahxHGQvJnu/Zg\nx2e4jR2f4ZdU/++qxjTYPAS8EjvvnlZ9fWD+AqIHW9xLNAq2KZ5mZiuBZcAaSYcW2Ta3E77KawMa\nOo9tMjyuyfC4JsdjmwyPa/FK3cHqLCk17ewc4P/S0ucC/SS1kVQGDAdmh5vhMjP7C3A9UG5mW4A3\nJZ0BIKmppD3ihZnZ3eEmtdzM1qelVZlZF+AlotGTTJ4BzpLUJtTROku+VAdrb+A9M6sKIwidM+TJ\n5Ung8u0HSIcXcEzNxkRrt04ERqUnZTnkJEmtQvwGA89nKHNZnmpzxsrMPgZWSRoSK7NHSPuE6HO4\nE5hmkaz5iyWpW+x1eahzQ3g/M0xNzXZsjWsvW9bwdzVwuCL7A0fnalqGfcuBr0vqHepvHKZfFkTR\nkw//GzjPzFZkSO8BdCUa/V1SaLnOOeecc64wpe5gvQr8WNJSooX2vwv7U0+3Ww+MJhotWEC0xmQq\n0UL9SkkLiKZ3jQ7HnQ9cLmkRUacg9TCAYrwGtMmUEKb03UzUyVsAjI+3N541/H0YOCq051yikYL0\nPJmOT7mJ6EEgixU9hOLn6RkkHRHWJuVyJdCBaD3V/DCalaveuUTT3RYSTfOrNj2wkNGeHLGKOxcY\nqeihJK8Ag2JpjxKNAP45tm9Ejvw1KHq4x2kZkn4g6RVJ84k6ccNCfhGtjduYo9hs117Ga8DMnifq\nZC0BfkU0fZBcx6Qd/wUwBLhN0kKi/w6OLeJcbyC6nu8Oo7dz09JbA6vNrKrmoS5JPoc9OR7bZHhc\nk+FxTY7HNhke1+Lt9j80HNY77WNmo/Nm3k1J+h7Q1cx+W99tqUthityFZnZNfbelVMJDM75vZsNz\n5PEfGnbOOeecy0P1/UPDDdhkoI9iPzTsqjOzx3e1zhWAmS3ZzTpX44FrgP+s77bsjnwOe3I8tsnw\nuCbD45ocj20yPK7FK/VTBBucsE7lO/XdDueSZtEPDReokCWDDV/btsX98rpzzjnn3M7a7acIOueq\nk2T+74JzzjnnXG4+RdA555xzzjnnEuYdLOecKwGfw54cj20yPK7J8Lgmx2ObDI9r8byD5Zxzzjnn\nnHN1xNdgOeeq8TVYzjnnnHP5+Ros55xzzjnnnEuYd7Ccc64EfA57cjy2yfC4JsPjmhyPbTI8rsXb\n7X8HyzlXk/TV/B2sth3bsv6t9fXdDOecc87txnwNlnOuGklGRX23opYqwP9Nc84551wp+Bos55xz\nzjnnnEtYyTpYkjpLejlL2ixJ5aVqS1rdnSTNlzQ9tm9VfbQlG0n9JU0sIF/Odkv6OPxtL2lSeH2B\npN/UprwC6xwj6ap85RQjXqakiZL65ck/SNIiSQskvSTphALqmCWpU570oq5ZSUMkLZX0dHj/iKSF\nkkaF8zgzz/F5zzXk+7Wk10PZh8f2j5e0RFL/Ytrt6obPYU+OxzYZHtdkeFyT47FNhse1eKVeg9UQ\n5+4MBp4ys9GxfQ2xnYW0KV8eAzCzd4ChBRxXF3U2BDPNbAqApG8DfwG+WQ/tGAlcbGZ/k9QOONLM\nDgztytuBLoSkU4FuZnagpGOA3wG9AczsaklzgYuA2XVRn3POOeecq67UUwSbSHoofIs/SVKz9AyS\nhktaHLZbw75G4dv7xWEkYlTY303SjPBN/UuSutaiTa2A99L2vR9rz/mx0Y8Hwr6Jku6U9LykN1Ij\nD5JaSJoZ2rJI0qCwv7OkZeG45ZIelnRSOH65pCNDvuaSJkiaI2mepNNDMz4HPizgXN4P5YwN7Z0v\n6S1JE1KnE2tPfDSxUxiRWS7pxkxxyFdntljFSTpA0nRJL0qaLam7pJaSVsfyNJe0VlJZpvwZ6t9M\nFJ+szOzT2Ns9gQ8KOK8NwLZs114wVNLfJb0qqU9of7URQUlTJfWTdAPQF5gg6ZfAk0DH8Bn1TYtT\nuaTKcN7TJbUt9FyBM4AHw3n/Hdg7djzAeqJr3pXYgAED6rsJuyyPbTI8rsnwuCbHY5sMj2vxSj2C\ndRBwoZnNCTf9lwF3pBIltQduBXoR3UzOCJ2Ut4COZtYj5GsZDnkYGGdmUyQ1pXYdxjKgKr7DzI4J\n9RwCXAsca2abJMVvTNuZWR9JBwNTgMnAZ8BgM9siaR9gTkgD6Ab8wMyWSnoJGBaOHxTqOBO4Dnja\nzEZK2huYK2mmmb0AvBDadATwIzO7JP1EUu02szHAmFDGs0Dqhj8+2hR/fRRwaGj/i5Kmmdn8VHm5\nFBirlHtD21dIOhq4x8wGhg5ZfzObDZwGPGFm2yTVyA8MTKv/ytRrSWOBF81sWnrFkgYDtwDtgO8W\ncF5DwnHlZL72AMrM7BhFo0YVwEmpwzOUd5OiqYlXmdkCSXcBU82sPJQ7MvxtTPR5DTKzDZKGAuOA\nkQWea0fgzdj7dWHfu+F9FdE1n9us2OsuQG2+unDOOeec24VUVlYWNGWy1CNYa81sTnj9ENE3+nFH\nAbPMbKOZVRF1oPoBK4GuYdTou8DHkvYEOqSmfpnZ52b2WTGNkSSgJ1EHLpMTgMfMbFOoY3Ms7a9h\n3zLgG6kigVskLQJmAh0kpdJWmdnS8HpJSAd4megWFuBkYLSkBUAl0BSotg7IzOZl6lxl8RBwh5kt\nzJNvhpltDvGbTM3PpRC5YoWkFsBxwGPh/H4PpEZWJgFnh9fDgEfz5M/IzMZk6lyFtL+a2cHA6cAf\nizivGtdeLG1y+DsP6Fxgefmef34QcBjRlwsLiDrdHdIz5TrXPNYB3SV9LWeu42Obd67qhM9hT47H\nNhke12R4XJPjsU2Gx3WHAQMGUFFRsX3Lpr7XYGVav1PjBtTMNkvqSTTycClwFnBFprzVCpIuA34Y\n6vknM1sfS2tEdPO8FXi8iHNI2ZqhzSOAfYFeZlal6AEQzTLkr4q9r2LH5yCiUa7Xa9GeaiRVEHVo\na0zVy6CQz2VnNQI2pUZs0kwBbpbUGigHniGaypctf62Z2XOSGkvax8w2FJA/07V3cUhOfYbb2PEZ\nfkn1Ly5qTIPNQ8ArZtanyONS1gH7x97vF/YBYGYrJS0D1kgaaGZLalmPc84555zLoNQjWJ0VLbwH\nOAf4v7T0uUA/SW0klQHDgdlhul2Zmf0FuB4oN7MtwJuSzgCQ1FTSHvHCzOxuM+tlZuXxzlVIqzKz\nLsBL7Bg9SfcMcJakNqGO1lnypTpYewPvhc7V8VQf1Sjkl1ufBC7ffkDsCXDFULR260RgVHpSlkNO\nktQqxG8w8HyGMpflqTZnrMzsY2CVpCGxMnuEtE+IPoc7gWkWyZq/WJK6xV6Xhzo3hPczw9TUbMfW\nuPayZQ1/VwOHK7I/cHSupmXYtxz4uqTeof7GYfploaYA54djewObzSw1PTAVw65Eo7/euSohn8Oe\nHI9tMjyuyfC4JsdjmwyPa/FK3cF6FfixpKVEC+1/F/annm63HhhNND1uAdEak6lEa0gqw5SpP4Y8\nEN1IXh6m5D1PnilkWbwGtMmUEKb03UzUyVsAjI+3N541/H0YOCq051xgWYY8mY5PuYnoQSCLFT2E\n4ufpGSQdEdYm5XIl0bSyF8NDFCry1DuXaLrbQqJpfvPT6twnT325YhV3LjBS0UNJXgEGxdIeJRoB\n/HNs34gc+WtQ9HCP0zIk/UDSK5LmE3XihoX8IlobtzFHsdmuvYzXgJk9T9TJWgL8imj6ILmOSTv+\nC2AIcJukhUT/HRxb6Lma2f8SdUzfIJpWeVlaltbA6jAF1znnnHPO1TGZfRWesp0cST8F9kl7TLuL\nkfQ9oKuZ/ba+21KXJB1K9NCVa+q7LaUSHprxfTMbniOPUVG6NtWpCmio/6ZVVlb6t4AJ8dgmw+Oa\nDI9rcjy2yfC4ZicJM6sxI6nUa7AaosnA/ZKmm9mp9d2YhsjMarNGrcELU+R2p87VeOA7wL/lzVyR\ndGuS0bZjbQaxnXPOOefqzm4/guWcq06S+b8LzjnnnHO5ZRvBKvUaLOecc84555zbZXkHyznnSsB/\nRyQ5HttkeFyT4XFNjsc2GR7X4nkHyznnnHPOOefqiK/Bcs5V42uwnHPOOefy8zVYzjnnnHPOOZcw\n72A551wJ+Bz25Hhsk+FxTYbHNTke22R4XIvnv4PlnKtBqjHavUtq27Yz69evru9mOOecc24X4muw\nnHPVSDLYXf5dEP5voHPOOedqw9dgOeecc84551zCStbBktRZ0stZ0mZJKi9VW9Lq7iRpvqTpsX2r\n6qMt2UjqL2liAflytlvSx+Fve0mTwusLJP2mNuUVWOcYSVflK6cY8TIlTZTUL0/+gyT9TdJnhbYl\nXJOd8qQXdc1KGiJpqaSnw/tHJC2UNCqcx5l5ji/kXM+RtChsz0nqEUsbL2mJpP7FtNvVDZ/DnhyP\nbTI8rsnwuCbHY5sMj2vxSr0GqyHOxRkMPGVmo2P7GmI7C2lTvjwGYGbvAEMLOK4u6mwINgA/Ifqs\n69NI4GIz+5ukdsCRZnYgRJ2nOqpjJdDPzD6UdApwL9AbwMyuljQXuAiYXUf1Oeecc865mFJPEWwi\n6aHwLf4kSc3SM0gaLmlx2G4N+xqFb+8Xh2/mR4X93STNCKMAL0nqWos2tQLeS9v3fqw954c6F0h6\nIOybKOlOSc9LeiM18iCphaSZoS2LJA0K+ztLWhaOWy7pYUknheOXSzoy5GsuaYKkOZLmSTo9NONz\n4MMCzuX9UM7Y0N75kt6SNCF1OrH2xEcTO4URmeWSbswUh3x1ZotVnKQDJE2X9KKk2ZK6S2opaXUs\nT3NJayWVZcqfof7NRPHJysw+MLN5wJcFnE/KBmBbtmsvGCrp75JeldQntL/aiKCkqZL6SboB6AtM\nkPRL4EmgY/iM+qbFqVxSZTjv6ZLaFnGuc8wsda3MATqmZVlPdM27EhswYEB9N2GX5bFNhsc1GR7X\n5Hhsk+FxLV6pR7AOAi40sznhpv8y4I5UoqT2wK1AL6KbyRmhk/IW0NHMeoR8LcMhDwPjzGyKpKbU\nrsNYBlTFd5jZMaGeQ4BrgWPNbJOk+I1pOzPrI+lgYAowGfgMGGxmWyTtQ3SDOyXk7wb8wMyWSnoJ\nGBaOHxTqOBO4DnjazEZK2huYK2mmmb0AvBDadATwIzO7JP1EUu02szHAmFDGs0Dqhj8+2hR/fRRw\naGj/i5Kmmdn8VHm5FBirlHtD21dIOhq4x8wGhg5ZfzObDZwGPGFm2yTVyA8MTKv/ytRrSWOBF81s\nWr52F3BeQ0KZ5WS+9gDKzOwYSacCFcBJqcMzlHeTpBOAq8xsgaS7gKlmVh7KHRn+Nib6vAaZ2QZJ\nQ4FxwMhanOvFwPS0fVVE13weFbHXA8LmnHPOObf7qqysLGjKZKlHsNaa2Zzw+iGib/TjjgJmmdlG\nM6si6kD1I5r21DWMGn0X+FjSnkAHM5sCYGafm9lnxTRGkoCeRB24TE4AHjOzTaGOzbG0v4Z9y4Bv\npIoEbpG0CJgJdJCUSltlZkvD6yUhHeBloEt4fTIwWtICoBJoClRbB2Rm8zJ1rrJ4CLjDzBbmyTfD\nzDaH+E2m5udSiFyxQlIL4DjgsXB+vwdSIzOTgLPD62HAo3nyZ2RmY+qic5WmxrUXS5sc/s4DOhdY\nXr7nnx8EHEb05cICok53h/RM+c5V0vHAhcC/piWtA7pL+lruZlTEtgF5muwK4XPYk+OxTYbHNRke\n1+R4bJPhcd1hwIABVFRUbN+yqe81WJnW79S4ATWzzZJ6At8FLgXOAq7IlLdaQdJlwA9DPf9kZutj\naY2Ibp63Ao8XcQ4pWzO0eQSwL9DLzKoUPQCiWYb8VbH3Vez4HEQ0yvV6LdpTjaQKog5tjal6GRTy\nueysRsCm1IhNminAzZJaA+XAM8CeOfKXTJZr7+KQnPoMt7HjM/yS6l9c1JgGm4eAV8ysT+1aDIoe\nbHEvcEqqw5tiZislLQPWSBpoZktqW49zzjnnnKup1CNYnSWlpp2dA/xfWvpcoJ+kNpLKgOHA7DDd\nrszM/gJcD5Sb2RbgTUlnAEhqKmmPeGFmdreZ9TKz8njnKqRVmVkX4CV2jJ6kewY4S1KbUEfrLPlS\nHay9gfdC5+p4qo9qFPLLrU8Cl28/QDq8gGNqNiZau3UiMCo9KcshJ0lqFeI3GHg+Q5nL8lSbM1Zm\n9jGwStKQWJk9QtonRJ/DncA0i2TNv5OqxUDRmrn2WTNnuPbylLsaOFyR/YGjC21LsBz4uqTeof7G\nYfplQRQ9+fC/gfPMbEWG9B5AV6LRX+9clZDPYU+OxzYZHtdkeFyT47FNhse1eKXuYL0K/FjSUqKF\n9r8L+1NPt1sPjCaaHreAaI3JVKKF+pVhytQfQx6A84HLw5S858kzhSyL14A2mRLClL6biTp5C4Dx\n8fbGs4a/DwNHhfacCyzLkCfT8Sk3ET0IZLGih1D8PD2DpCPC2qRcriSaVvZieIhCRZ565xJNd1tI\nNM1vflqd++SpL1es4s4FRip6KMkrwKBY2qNEI4B/ju0bkSN/DYoe7nFahv1tJb1JFJfrFD1EY88w\nRbQbsDFHsdmuvYzXgJk9T9TJWgL8imj6ILmOSTv+C2AIcJukhUT/HRxb6LkCNxBdz3eHtW1z09Jb\nA6vDFFznnHPOOVfHZPZVeMp2ciT9FNgn7THtLkbS94CuZvbb+m5LXZJ0KNFDV66p77aUSnhoxvfN\nbHiOPPbVePp+XRCl+jewsrLSvwVMiMc2GR7XZHhck+OxTYbHNTtJmFmNGUmlXoPVEE0G7pc03cxO\nre/GNERmVps1ag1emCK3O3WuxgPfAf6tgNxJN6dBaNu20GeTOOecc84VZrcfwXLOVSfJ/N8F55xz\nzrncso1glXoNlnPOOeecc87tsryD5ZxzJeC/I5Icj20yPK7J8Lgmx2ObDI9r8byD5ZxzzjnnnHN1\nxNdgOeeq8TVYzjnnnHP5+Ros55xzzjnnnEuYd7Ccc64EfA57cjy2yfC4JsPjmhyPbTI8rsXz38Fy\nztUg7R6/gxXXtmNb1r+1vr6b4ZxzzrmvOF+D5ZyrRpJRUd+tqAcV4P8eOuecc65QvgbLOeecc845\n5xJWsg6WpM6SXs6SNktSeanaklZ3J0nzJU2P7VtVH23JRlJ/SRMLyJez3ZI+Dn/bS5oUXl8g6Te1\nKa/AOsdIuipfOcWIlylpoqR+efIfJOlvkj4rtC3hmuyUJ72oa1bSEElLJT0d3j8iaaGkUeE8zsxz\nfN5zDfl+Len1UPbhsf3jJS2R1L+Ydru64XPYk+OxTYbHNRke1+R4bJPhcS1eqddgNcT5N4OBp8xs\ndGxfQ2xnIW3Kl8cAzOwdYGgBx9VFnQ3BBuAnRJ91fRoJXGxmf5PUDjjSzA6EqPNUFxVIOhXoZmYH\nSjoG+B3QG8DMrpY0F7gImF0X9TnnnHPOuepKPUWwiaSHwrf4kyQ1S88gabikxWG7NexrFL69Xyxp\nkaRRYX83STPCN/UvSepaiza1At5L2/d+rD3nhzoXSHog7Jso6U5Jz0t6IzXyIKmFpJmhLYskDQr7\nO0taFo5bLulhSSeF45dLOjLkay5pgqQ5kuZJOj0043PgwwLO5f1QztjQ3vmS3pI0IXU6sfbERxM7\nhRGZ5ZJuzBSHfHVmi1WcpAMkTZf0oqTZkrpLailpdSxPc0lrJZVlyp+h/s1E8cnKzD4ws3nAlwWc\nT8oGYFu2ay8YKunvkl6V1Ce0v9qIoKSpkvpJugHoC0yQ9EvgSaBj+Iz6psWpXFJlOO/pktoWeq7A\nGcCD4bz/DuwdOx5gPdE170pswIAB9d2EXZbHNhke12R4XJPjsU2Gx7V4pR7BOgi40MzmhJv+y4A7\nUomS2gO3Ar2IbiZnhE7KW0BHM+sR8rUMhzwMjDOzKZKaUrsOYxlQFd9hZseEeg4BrgWONbNNkuI3\npu3MrI+kg4EpwGTgM2CwmW2RtA8wJ6QBdAN+YGZLJb0EDAvHDwp1nAlcBzxtZiMl7Q3MlTTTzF4A\nXghtOgL4kZldkn4iqXab2RhgTCjjWSB1wx8fbYq/Pgo4NLT/RUnTzGx+qrxcCoxVyr2h7SskHQ3c\nY2YDQ4esv5nNBk4DnjCzbZJq5AcGptV/Zeq1pLHAi2Y2LV+7CzivIaHMcjJfewBlZnaMolGjCuCk\n1OEZyrtJ0gnAVWa2QNJdwFQzKw/ljgx/GxN9XoPMbIOkocA4YGSB59oReDP2fl3Y9254X0V0zec2\nK/a6C1Cbry6cc84553YhlZWVBU2ZLPUI1tr/z97dB1lV3fn+f39AGUVRwQcezIiG0hhujIpRY3S0\n9RcxMdF4URIfMliJo0lhqZiYKn6Z3HSjRIzGRJ0px2gcEgVrkHtJBjWKQLodRRGE5kFB1AkasYJ6\no6hMER3le/9Y3wO7T5/d54He3QjfV1VX77P32nt993fvA2edtdZuM1voy9NI3+hnHQu0mtlbZraZ\n1IA6GfgjcIj3Gp0BvCdpT2CYmc0GMLMPzOyv9QQjScCRpAZcJacBM83sba9jQ2bb73zdauCA0iGB\nKZKWA/OAYZJK29aa2Spffs63A6wkfYQFGA1MlNQOtAH9gA7zgMxsSaXGVY5pwM/NbFmVcnPNbIPn\nbxUygQ8AACAASURBVBadr0stusoVkvYAvgDM9PP7JVDqWbkf+IYvnw/MqFK+IjNr7o7GVZlO915m\n2yz/vQQYXuPxqj3//FPAZ0hfLrSTGt3Dygttw7m+Bhwm6W+6LHVq5icaV90ixrAXJ3JbjMhrMSKv\nxYncFiPyulVTUxMtLS1bfvL09hysSvN3On0ANbMNko4EzgC+C4wFJlQq2+FA0njgUq/nTDNbn9nW\nh/Th+X3goTrOoeT9CjFfBOwHHG1mm5UeALFbhfKbM683s/U6iNTL9WID8XQgqYXUoO00VK+CWq7L\ntuoDvF3qsSkzG/iJpIHAKOAPwJ5dlO8xOffeP/jm0jX8iK3X8EM6fnHRaRhsFQKeNbMTG4uY14C/\nzbz+hK8DwMz+KGk18Iqk/8/MnmuwnhBCCCGEUEFP92ANV5p4D3Ah8HjZ9kXAyZIGSeoLXAA85sPt\n+prZb4EfAaPMbCPwqqSvAUjqJ2n37MHM7HYzO9rMRmUbV75ts5kdDDzD1t6Tcn8Axkoa5HUMzClX\namDtDbzhjatT6dirUctfbp0DXLllh8wT4OqhNHfri8BV5Ztydjld0j6ev3OABRWOubpKtV3mysze\nA9ZKOi9zzM/6tv8iXYdbgQctyS2/jTrkQGnO3NDcwhXuvSrHfRk4SsnfAsfVGotbA+wv6fNe/y4+\n/LJWs4Fxvu/ngQ1mVhoeWMrhIaTe32hc9aAYw16cyG0xIq/FiLwWJ3JbjMhr/Xq6gfU8cLmkVaSJ\n9nf4+tLT7dYDE0nD49pJc0weIM0hafMhU/d6GUgfJK/0IXkLqDKELMcLwKBKG3xI309Ijbx24OZs\nvNmi/ns6cKzH801gdYUylfYvuY70IJAVSg+huLa8gKRjfG5SV64mDStb7A9RaKlS7yLScLdlpGF+\nS8vq3LdKfV3lKuubwCVKDyV5Fjg7s20GqQfw3zLrLuqifCdKD/f4aoX1gyW9SsrLPyo9RGNPHyI6\nAniri8Pm3XsV7wEzW0BqZD0H3EIaPkhX+5Tt/9/AecBPJS0jvQ9OqPVczez3pIbpS6RhlePLigwE\nXvYhuCGEEEIIoZvJ7OPwlO3iSPoBsG/ZY9pDhqSvAIeY2T/3dizdSdL/ID105ZrejqWn+EMz/qeZ\nXdBFGaOl52LabrRAkf8etrW1xbeABYncFiPyWozIa3Eit8WIvOaThJl1GpHU03OwtkezgF9LetjM\nvtzbwWyPzKyROWrbPR8itzM1rm4G/g74/6sWbik6mu3P4AMb6QAPIYQQQuhop+/BCiF0JMni34UQ\nQgghhK7l9WD19BysEEIIIYQQQthhRQMrhBB6QPwdkeJEbosReS1G5LU4kdtiRF7rFw2sEEIIIYQQ\nQugmMQcrhNBBzMEKIYQQQqgu5mCFEEIIIYQQQsGigRVCCD0gxrAXJ3JbjMhrMSKvxYncFiPyWr9o\nYIUQOpFU2M+QIQf39umFEEIIIRQm5mCFEDqQZFDkvwsi/t0JIYQQwsddzMEKIYQQQgghhIIV2sCS\nNFzSypxtrZJGFVl/HkkHSVoq6eHMurW9EUseSadImlpDubrjlnSVpN1ytl0s6TZfbpY0rsqxLpbU\nXKXMe/XGWE3pmH6PtdZQvlXS85La/drvV6V8l/n37Q/UGXM/SXO9/rGSTpL0rL8+PO+9ktm/6rlK\n2l3Sg5JWS1op6frMtsO8vhn1xB26R4xhL07kthiR12JEXosTuS1G5LV+PdGDtT2OBToHeNTMvpxZ\ntz3GWUtMjcQ9AejfwH6NxlBEbi1nuSsXmNnRZjbKzP5vnXU0sr3cKMC8/pnARcD1ZjYK2FTj8Wop\nc5OZfRo4GjhJ0hmkil8ws88AR0g6pM7YQwghhBBCDXqigbWrpGmSVkm6v1LPiaQLJK3wnxt8XR9J\nU33dcklX+foR3guwTNIzDX5Q3Ad4o2zdm5l4xnmd7ZJ+4+umSrpV0gJJL0ka4+v3kDTPY1ku6Wxf\nP9x7EaZKWiNpuqTTff81kj7n5fpLulvSQklLJJ3lYXwAvFPDubzpx5mU6Z1Z58fs770Z7Z7HsZKu\nAIYBrZLm+77f8pgWAidmjr2R9MG/K5u8HJIOkDTLr027pM+XUprJ7TWSFnmZZl83RdL4TJlmSd/L\nK1/mI+CtGvIE9d3vW/LvvVWl3C6RtIeXGSBppl/nezPxr5U0yJeP8d6z/YF7gWP9OJcBXweuy+7r\n+/SRdKOkp/28L631XM1sk5k95ssfAkuBT5QVe530Hgg9qKmpqbdD2GFFbosReS1G5LU4kdtiRF7r\nt0sP1PEp4FtmtlDS3cB44OeljZKGAjeQvm3fAMz1Rso64EAz+6yX28t3mU761n+2pH401kjsC2zO\nrjCz472ekcAPgRPM7G1J2Q+iQ8zsREmfBmYDs4C/AueY2UZJ+wILfRvACOBcM1sl6RngfN//bK9j\nDPCPwHwzu0TS3sAiSfPM7CngKY/pGOA7ZnZZ+YmU4jazZqDZj/EfwD8DXwJeM7Ov+nEGmNl7kq4G\nmvz8hgAtpPy/C7SRPpRjZjdXS6SZ3Z95eRvQZmZjJAnYs1TM6z8dONTMjvPtsyWdBMwAbgFu9/Jf\nB0bnlTezJ/BGm5mtA87z4w8F7iqdbwW/lvTfwCwzm1zlvLbkH/g+MN7MnpLUn3TNAY4CRgLrgQWS\nvmBmT9K5l8nM7E1J/wB838xKjfATgAfMbJak4ZnylwAbzOx4v8cXSHrUzF6p41zxe/csUm6zNpPe\nA11oySw3+U8IIYQQws6rra2tpiGTPdGD9SczW+jL04CTyrYfC7Sa2VtmtpnUgDoZ+CNwiPcanQG8\nJ2lPYJiZzQYwsw/M7K/UwT+oH0lqwFVyGjDTzN72OjZktv3O160GDigdEpgiaTkwDxgmqbRtrZmt\n8uXnfDvASuBgXx4NTJTUTmrc9AMOygZkZksqNa5yTANuNrN2r+d07yE6ycxKc6HE1l6l49ma/w9J\njZ1GnQb8i8dsmfpKRns8S0mNuE+RGlDLgP0lDZH0WeAtM3str3xe5Wb25y4aHBea2RHA3wF/J+mb\ndZzXAuAX3vs30O9TgEVepwHL2HpNOz1Npk6jgXF+TzwNDKLsvKucK5L6AvcBt5jZy2Wb15HeA11o\nyfw01R55yBVj2IsTuS1G5LUYkdfiRG6LEXndqqmpiZaWli0/eXqiB6vTt/kVynT6QGpmGyQdCZwB\nfBcYS5o71OWHVx9qdqnXc6aZrc9s60NquL0PPFTHOZS8XyHmi4D9gKPNbLPSQyd2q1B+c+b1Zrbm\nXqRerhcbiKcDSS2kBu09AGb2otKDRM4EJnvPWKWem21tEJRUmx8kYIqZ3VVh20zSNR7C1kZeV+Xr\nmv9kZn/23/8l6T7gOFJjtJZ9fyrpQeArpN6k0b4pe30/Yus1/ZCtX15UfJhIFQKuMLO5Dexbciew\nxsz+qcK2XwJzJB1nZt/ZhjpCCCGEEEKZnujBGi7peF++EHi8bPsi4GRJg/xb9wuAx3y4XV8z+y3w\nI2CUmW0EXpX0NdjyVLbdswczs9szDzJYX7Zts5kdDDwDfCMn3j8AYzNzaAbmlCs1SvYG3vDG1anA\n8AplujIHuHLLDtJRNezTOZg0d+uLwFWZdUOBTWZ2H3AT6SELkIYCloZcPk3K/0BJu5IaOZWOf3l2\nnlSO+aQhoKV5RANKu/vvOcC3S3OYJA3zuUkA9wPnA+eSGlt55fcrO2ZVkvr6/YSf41eBZ/31Oco8\naS9n/0+a2XNmdiOwGDi8SpVrgWN8+dxa48yYA4yXtIvXf2j5fV4l3snAXmZ2dU6Ra4BLonHVs2IM\ne3Eit8WIvBYj8lqcyG0xIq/164kG1vPA5ZJWkSbW3+HrDcAbQRNJw+PagcVm9gBwINDmw6Tu9TIA\n44ArfUjeAmBwAzG9QBp21YkP6fsJqZHXDpTmIeX1xE0nPbhgOfBNYHWFMpX2L7mO9CCQFUqP6b62\nvIA/KOHOLs4H4GrSwysW+0MUWoAjSHO62oEfA6Xeq7uARyTN9/xPIs0dexxY1enIyeHAX6rEMAE4\nVdIKUiN2pK8vXeu5pGFrT3mZmfg8Lc/7AGCdmb3eRfkB2WNmSRrqPU3l/obUY7OMNNRwnecA0jy5\nag8TmaD0yPPlpIdfPFyhTDaea4HbJC0i9WblybsnfkW6Dkv9nriDst7mvHOVdCBpft9IbX0wx7fL\nig0EXuoirhBCCCGE0CCl6SM7F0k/APY1s4lVCwcAJM0Gxvg8rR2GpHuAq82sWuNxh+BzEFcA55nZ\nmpwyVuxfLRA74787bW1t8S1gQSK3xYi8FiPyWpzIbTEir/kkYWadRlX1RA/W9mgWcKIyf2g4dM3M\nzt7RGlcAZjZuJ2pcHUbqJW4n9eKGEEIIIYRutlP2YIUQ8qUerOIMHjyc9etfLrKKEEIIIYTC5fVg\n9cRTBEMIHzPxxUsIIYQQQmN21iGCIYTQo+LviBQncluMyGsxIq/FidwWI/Jav2hghRBCCCGEEEI3\niTlYIYQOJFn8uxBCCCGE0LV4imAIIYQQQgghFCwaWCGE0ANiDHtxIrfFiLwWI/JanMhtMSKv9YsG\nVgghhBBCCCF0k5iDFULooLv+DtbgAwezft367jhUCCGEEMJ2J28OVjSwQggdSDJauuFALfH3tEII\nIYSw44qHXIQQQi+KMezFidwWI/JajMhrcSK3xYi81q/QBpak4ZJW5mxrlTSqyPrzSDpI0lJJD2fW\nre2NWPJIOkXS1BrK1R23pKsk7Zaz7WJJt/lys6RxVY51saTmKmXeqzfGakrH9HustYbyrZKel9Tu\n136/KuW7zL9vf6DOmPtJmuv1j5V0kqRn/fXhee+VzP61nusoSSskvSDplsz6w7y+GfXEHUIIIYQQ\natcTPVjb4xihc4BHzezLmXXbY5y1xNRI3BOA/g3s12gMReTWcpa7coGZHW1mo8zs/9ZZRyPby40C\nzOufCVwEXG9mo4BNNR6vljL/AlxiZocBh0k6g1TxC2b2GeAISYfUGXvYRk1NTb0dwg4rcluMyGsx\nIq/FidwWI/Jav55oYO0qaZqkVZLur9RzIukC/8Z9haQbfF0fSVN93XJJV/n6Ed4LsEzSMw1+UNwH\neKNs3ZuZeMZ5ne2SfuPrpkq6VdICSS9JGuPr95A0z2NZLulsXz9c0mrfb42k6ZJO9/3XSPqcl+sv\n6W5JCyUtkXSWh/EB8E4N5/KmH2dSpndmnR+zv6QHff0K7zW5AhgGtEqa7/t+y2NaCJyYOfZG0gf/\nrmzyckg6QNIsvzbtkj5fSmkmt9dIWuRlmn3dFEnjM2WaJX0vr3yZj4C3asgT1He/b8m/91aVcrtE\n0h5eZoCkmX6d783Ev1bSIF8+xnvP9gfuBY7141wGfB24Lruv79NH0o2SnvbzvrTWc5U0BBhgZot9\n1T2kLxSyXie9B0IIIYQQQjfbpQfq+BTwLTNbKOluYDzw89JGSUOBG4CjgQ3AXG+krAMONLPPerm9\nfJfppG/9Z0vqR2ONxL7A5uwKMzve6xkJ/BA4wczelpT9IDrEzE6U9GlgNjAL+CtwjpltlLQvsNC3\nAYwAzjWzVZKeAc73/c/2OsYA/wjMN7NLJO0NLJI0z8yeAp7ymI4BvmNml5WfSCluM2sGmv0Y/wH8\nM/Al4DUz+6ofZ4CZvSfpaqDJz28I0ELK/7tAG7DUj3lztUSa2f2Zl7cBbWY2RpKAPUvFvP7TgUPN\n7DjfPlvSScAM4Bbgdi//dWB0XnkzewJvtJnZOuA8P/5Q4K7S+Vbwa0n/Dcwys8lVzmtL/oHvA+PN\n7ClJ/UnXHOAoYCSwHlgg6Qtm9iSde5nMzN6U9A/A982s1Ag/AXjAzGZJGp4pfwmwwcyO93t8gaRH\nzeyVGs71QNJ7p2Sdr8vaTHoP5MsORDwYiP6ubdbW1hbfAhYkcluMyGsxIq/FidwWI/K6VVtbW01z\n0nqigfUnM1voy9OAK8g0sIBjgVYzewtA0nTgZGAycIikW4HfA49K2hMYZmazAczsg3qD8Q/qR3os\nlZwGzDSzt72ODZltv/N1qyUdUDokMEXSyaQPrsMy29aa2Spffg6Y58srSR9bAUYDZ0n6gb/uBxwE\nrClVamZLgE6NqxzTgJvNrF3SRuBnkqYAD3nDpBRzqVfpeDrmfwZwaI11lTsN+HuP2YDyuVejgdMl\nLfX69yA1oKZK2t8bewcAb5nZa5ImVCoPPEEFZvZnIK9xdaGZ/dl7n2ZJ+qaZ5d0D5RYAv/B7c5bH\nBrDI60TSMtI1fZJMj12DRpOG8Y3113uRzvuVUoEq51rNOtJ74JncEqc2eOQQQgghhB1UU1NTh8bm\npEmTKpbriQZWp2/zK5Tp9IHUzDZIOhI4A/guMJY0d6jLD68+1OxSr+dMM1uf2dYH+CPwPvBQHedQ\n8n6FmC8C9gOONrPNSg+d2K1C+c2Z15vZmnuRerlebCCeDiS1kBq09wCY2YtKDxI5E5jsPWOVem62\ntUFQUm1+kIApZnZXhW0zSdd4CKlHq1r5uuY/lRpCZvZfku4DjiO/kV2+708lPQh8hdSbNNo3Za/v\nR2y9ph+ytWe14sNEqhBwhZnNbWDf14C/zbz+hK/L+iUwR9JxZvadBuoIDYhv/4oTuS1G5LUYkdfi\nRG6LEXmtX0/MwRou6XhfvhB4vGz7IuBkSYMk9QUuAB7z4XZ9zey3wI+AUWa2EXhV0tdgy1PZds8e\nzMxuzzzIYH3Zts1mdjDpm/tv5MT7B2BsZg7NwJxypUbJ3sAb3rg6FRheoUxX5gBXbtlBOqqGfToH\nk+ZufRG4KrNuKLDJzO4DbiI9ZAHSUMDSkMunSfkfKGlXUiOn0vEvz86TyjGfNAS0NI9oQGl3/z0H\n+HZpDpOkYT43CeB+4HzgXFJjK6/8fmXHrEpSX7+f8HP8KvCsvz5H0vVV9v+kmT1nZjcCi4HDq1S5\nFjjGl8+tNc6MOcB4Sbt4/YeW3+d5/J5/R1JpWOU44N/Lil1DeghGNK5CCCGEELpZTzSwngcul7SK\nNLH+Dl9vsOUD4UTS3J92YLGZPUCaN9ImqZ30cICJvt844EpJy0lDtwY3ENMLwKBKG3xI309Ijbx2\noDQPKa8nbjrpwQXLgW8CqyuUqbR/yXWkB4GsUHpM97XlBZQelHBnF+cDcDXp4RWLlR6i0AIcQZrT\n1Q78mDTsEuAu4BFJ8z3/k0hzxx4HVnU6cnI48JcqMUwATpW0gtSIHenrS9d6LnAf8JSXmYnP0/K8\nDwDWmdnrXZQfkD1mlqSh3tNU7m9IPTbLSPPL1nkOIM2Tq/YwkQmSVvo1/gB4uEKZbDzXArdJWkTq\nzcqTd0/8inQdlvo9cQdlvc1dnCvA5cDdpPv8RTN7pGz7QOClLuIKBYi/I1KcyG0xIq/FiLwWJ3Jb\njMhr/ZSmyuxcfL7TvmY2sWrhAICk2cAYM+uqwfCxI+ke4Gozq9Z43CF4r9YK4DwzW5NTxmjphspa\nYGf89yVPTBIuTuS2GJHXYkReixO5LUbkNZ8kzKzTqKqdtYE1Avg1sLHsb2GFsMOSdBhpKOYK4GLL\nefNL6pZ/FAYfOJj169ZXLxhCCCGE8DEUDawQQk0k5bW9QgghhBCCy2tg9cQcrBBC2OnFGPbiRG6L\nEXktRuS1OJHbYkRe6xcNrBBCCCGEEELoJjFEMITQQQwRDCGEEEKoLoYIhhBCCCGEEELBooEVQgg9\nIMawFydyW4zIazEir8WJ3BYj8lq/aGCFEEIIIYQQQjeJOVghhA666+9g7QgGDx7O+vUv93YYIYQQ\nQtgOxd/BCiHUJDWw4t+FRMS/kSGEEEKoJB5yEUIIvSjGsBcncluMyGsxIq/FidwWI/Jav0IbWJKG\nS1qZs61V0qgi688j6SBJSyU9nFm3tjdiySPpFElTayhXd9ySrpK0W862iyXd5svNksZVOdbFkpqr\nlHmv3hirKR3T77HWGso/LKld0rOSfiVplyrlu8y/b3+gzpj7SZrr995YSSd5PEslHZ73XsnsX/Vc\nJe0u6UFJqyWtlHR9ZtthXt+MeuIOIYQQQgi164kerO1xfM05wKNm9uXMuu0xzlpiaiTuCUD/BvZr\nNIYicms5y3nGmtnRZvYZYB/gG3XW0cj2cqMAM7NRZjYTuAi43sxGAZtqPF4tZW4ys08DRwMnSTqD\nVPELfv5HSDqkztjDNmpqaurtEHZYkdtiRF6LEXktTuS2GJHX+vVEA2tXSdMkrZJ0f6WeE0kXSFrh\nPzf4uj6Spvq65ZKu8vUjvBdgmaRnGvyguA/wRtm6NzPxjPM62yX9xtdNlXSrpAWSXpI0xtfvIWme\nx7Jc0tm+frj3IkyVtEbSdEmn+/5rJH3Oy/WXdLekhZKWSDrLw/gAeKeGc3nTjzPJ410qaZ0fs7/3\nZrR7HsdKugIYBrRKmu/7fstjWgicmDn2RtIH/65s8nJIOkDSLL827ZI+X0ppJrfXSFrkZZp93RRJ\n4zNlmiV9L698mY+At6olycxKMe4K9AP+UmWXLfn33qpSbpdI2sPLDJA006/zvZn410oa5MvHKPXW\n7g/cCxzrx7kM+DpwXXZf36ePpBslPe3nfWmt52pmm8zsMV/+EFgKfKKs2Ouk90AIIYQQQuhuZlbY\nDzAc2Ax83l/fDXzPl1tJ3+gPBV4BBpEafPOBs33bo5lj7eW/FwJn+3I/YLcG4poETMjZNhJ4Hhjo\nr/fx31OBGb78aeBFX+4L7OnL+2bWDyd9SB/pr58B7vbls4FZvvwT4EJf3htYA+xeFtMxwJ01ntve\nwHJS78UY4JeZbQP89x8z5zckk/9dgCeA2xq83v8GXOnLytT3rv8+vRSPb38AOAk4CmjLHOc54MC8\n8v76vQr1DwUe7CK+R0gNqxl1ntds4ARf7u/36SnA216ngCeBL2TyOyhz7f7gy6cAszPHnQqMydwv\nK3z5UuCHmXt8MTC8nnMt3bvAfwIHl62fD3yui/0MmjM/rQa2k/5g3aW1tbXbjhU6itwWI/JajMhr\ncSK3xYi8btXa2mrNzc1bfvxzQqfPUl3OQ+kmfzKzhb48DbgC+Hlm+7FAq5m9BSBpOnAyMBk4RNKt\nwO+BRyXtCQwzs9mkM/qg3mAkCTjSY6nkNGCmmb3tdWzIbPudr1st6YDSIYEpkk4mNSaHZbatNbNV\nvvwcMM+XVwIH+/Jo4CxJP/DX/YCDSA0tvL4lwGU1nuI04GYza5e0EfiZpCnAQ2b2RCbmUq/S8XTM\n/wzg0BrrKnca8PceswHlc69GA6dLWur17wEcamZTJe0vaQhwAPCWmb0maUKl8qRGYCdm9mfgq3nB\nmdmXJPUD7pc0zszuqfG8FgC/8HtzlscGsMjrRNIy0jV9kkyPXYNGk4bxjfXXe5HO+5XMuXR5rpL6\nAvcBt5jZy2Wb15HeA8/kh9BSf9QhhBBCCDuwpqamDkMmJ02aVLFcTzSwyueMVJpD0ukDqZltkHQk\ncAbwXWAsae5Qlx9efajZpV7PmWa2PrOtD6l34X3goTrOoeT9CjFfBOwHHG1mm5UeOrFbhfKbM683\nszX3As41sxcbiKcDSS2kBu09AGb2otKDRM4EJkuaZ2aTK+26rXW7Ste2vJ4pZnZXhW0zSdd4CDCj\nhvLV6qocoNkHkv4PcBxQUwPLzH4q6UHgK8ACSaN9U/b6fsTWa/ohW4ffVnyYSBUCrjCzuQ3sW3In\nsMbM/qnCtl8CcyQdZ2bf2YY6Qh1iDHtxIrfFiLwWI/JanMhtMSKv9euJOVjDJR3vyxcCj5dtXwSc\nLGmQf+t+AfCYpH2Bvmb2W+BHwChL82helfQ12PJUtt2zBzOz2y09zGBUtnHl2zab2cGkb+7zHnLw\nB2BsZg7NwJxypUbJ3sAb3rg6lTTUq7xMV+YAV27ZQTqqhn06B5Pmbn0RuCqzbiiwyczuA24iDbsE\neJfUKwLwNCn/A31+0lgqkHR5dp5UjvnAeC/fR9KA0u7+ew7w7dIcJknDfG4SwP3A+cC5pMZWXvn9\nyo5ZldI8uSG+vAupobTMX5+jzJP2cvb/pJk9Z2Y3kobrHV6lyrWkoYH4+dRrDjDeY0XSoeX3eZV4\nJ5OG1F6dU+Qa4JJoXIUQQgghdL+eaGA9D1wuaRVpTsgdvj5N9kiNoIlAG9AOLDazB0hzcNoktZMe\nDjDR9xsHXClpOWno1uAGYnqBNOeoEx/S9xNSI68duDkbb7ao/55OenDBcuCbwOoKZSrtX3Id6UEg\nK5Qe031teQF/UMKdXZwPwNWkh1cs9ocotABHAIv8PH5MGnYJcBfwiKT5nv9JpLltjwOrOh05OZzq\nD4aYAJwqaQWpETvS15eu9VzSsLWnvMxMYE/ftgoYAKwzs9e7KD8ge8wsSUO9p6ncHsBsH8a3BHgV\n+FffNoLqDxOZoPTI8+WkeXUPVyiTjeda4DZJi0i9WXny7olfka7DUr8n7qCstznvXCUdCPwQGJl5\nMMe3y4oNBF7qIq5QgPg7IsWJ3BYj8lqMyGtxIrfFiLzWT2mqzM7F5zvta2YTqxYOAEiaTXogQ1cN\nho8dSfcAV5tZtcbjDsHnIK4AzjOzNTllbPv8qwW9QXTXv5FtbW0xzKIgkdtiRF6LEXktTuS2GJHX\nfJIws06jqnbWBtYI4NfARuv4t7BC2GFJOow0FHMFcLHlvPlTAysADB48nPXrX+7tMEIIIYSwHYoG\nVgihJpLy2l4hhBBCCMHlNbB6Yg5WCCHs9GIMe3Eit8WIvBYj8lqcyG0xIq/1iwZWCCGEEEIIIXST\nGCIYQugghgiGEEIIIVQXQwRDCCGEEEIIoWDRwAohhB4QY9iLE7ktRuS1GJHX4kRuixF5rV80sEII\nIYQQQgihm8QcrBBCBzvj38EafOBg1q9b39thhBBCCOFjJP4OVgihJpKMlt6Oooe1QPxbGEIIIYR6\nxEMuQgihF8UY9uJEbosReS1G5LU4kdtiRF7rV2gDS9JwSStztrVKGlVk/XkkHSRpqaSHM+vW0G7j\nMgAAIABJREFU9kYseSSdImlqDeXqjlvSVZJ2y9l2saTbfLlZ0rgqx7pYUnOVMu/VG2M1pWP6PdZa\nQ/mHJbVLelbSryTtUqV8l/n37Q/UGXM/SXP93hsr6SSPZ6mkw/PeK5n9az3XUZJWSHpB0i2Z9Yd5\nfTPqiTuEEEIIIdSuJ3qwtsdxN+cAj5rZlzPrtsc4a4mpkbgnAP0b2K/RGIrIreUs5xlrZkeb2WeA\nfYBv1FlHI9vLjQLMzEaZ2UzgIuB6MxsFbKrxeLWU+RfgEjM7DDhM0hmkil/w8z9C0iF1xh62UVNT\nU2+HsMOK3BYj8lqMyGtxIrfFiLzWrycaWLtKmiZplaT7K/WcSLrAv3FfIekGX9dH0lRft1zSVb5+\nhPcCLJP0TIMfFPcB3ihb92YmnnFeZ7uk3/i6qZJulbRA0kuSxvj6PSTN81iWSzrb1w+XtNr3WyNp\nuqTTff81kj7n5fpLulvSQklLJJ3lYXwAvFPDubzpx5nk8S6VtM6P2V/Sg75+hfeaXAEMA1olzfd9\nv+UxLQROzBx7I+mDf1c2eTkkHSBpll+bdkmfL6U0k9trJC3yMs2+boqk8ZkyzZK+l1e+zEfAW9WS\nZGalGHcF+gF/qbLLlvx7b1Upt0sk7eFlBkia6df53kz8ayUN8uVjlHpr9wfuBY7141wGfB24Lruv\n79NH0o2SnvbzvrTWc5U0BBhgZot91T2kLxSyXie9B0IIIYQQQjfrcphUN/kU8C0zWyjpbmA88PPS\nRklDgRuAo4ENwFxvpKwDDjSzz3q5vXyX6aRv/WdL6kdjjcS+wObsCjM73usZCfwQOMHM3paU/SA6\nxMxOlPRpYDYwC/grcI6ZbZS0L7DQtwGMAM41s1WSngHO9/3P9jrGAP8IzDezSyTtDSySNM/MngKe\n8piOAb5jZpeVn0gpbjNrBpr9GP8B/DPwJeA1M/uqH2eAmb0n6Wqgyc9vCNBCyv+7QBuw1I95c7VE\nmtn9mZe3AW1mNkaSgD1Lxbz+04FDzew43z5b0knADOAW4HYv/3VgdF55M3sCb7SZ2TrgPD/+UOCu\n0vmWk/QIcCwwz8weqXJeW/IPfB8Yb2ZPSepPuuYARwEjgfXAAklfMLMn6dzLZGb2pqR/AL5vZqVG\n+AnAA2Y2S9LwTPlLgA1mdrzf4wskPWpmr9RwrgeS3jsl63xd1mbSeyBfdiDiwUD0d22ztra2+Baw\nIJHbYkReixF5LU7kthiR163a2tpqmpPWEw2sP5nZQl+eBlxBpoFF+sDbamZvAUiaDpwMTAYOkXQr\n8HvgUUl7AsPMbDaAmX1QbzD+Qf1Ij6WS04CZZva217Ehs+13vm61pANKhwSmSDqZ9MF1WGbbWjNb\n5cvPAfN8eSXpYyvAaOAsST/w1/2Ag4A1pUrNbAnQqXGVYxpws5m1S9oI/EzSFOAhb5iUYi71Kh1P\nx/zPAA6tsa5ypwF/7zEbUD73ajRwuqSlXv8epAbUVEn7e2PvAOAtM3tN0oRK5YEnqMDM/gxUbFz5\n9i95g+V+SePM7J4az2sB8Au/N2d5bACLvE4kLSNd0yfJ9Ng1aDRpGN9Yf70X6bxfyZxLl+daxTrS\ne+CZ3BKnNnjkEEIIIYQdVFNTU4fG5qRJkyqW64kGVqdv8yuU6fSB1Mw2SDoSOAP4LjCWNHeoyw+v\nPtTsUq/nTDNbn9nWB/gj8D7wUB3nUPJ+hZgvAvYDjjazzUoPnditQvnNmdeb2Zp7kXq5Xmwgng4k\ntZAatPcAmNmLSg8SOROY7D1jkyvtuq11u2rzgwRMMbO7KmybSbrGQ0g9WtXKNzSvy8w+kPR/gONI\nw+dq2eenkh4EvkLqTRrtm7LX9yO2XtMP2dqzWvFhIlUIuMLM5jaw72vA32Zef8LXZf0SmCPpODP7\nTgN1hAbEt3/FidwWI/JajMhrcSK3xYi81q8n5mANl3S8L18IPF62fRFwsqRBkvoCFwCP+XC7vmb2\nW+BHwCifR/OqpK/Blqey7Z49mJnd7g8zGJVtXPm2zWZ2MOmb+7yHHPwBGJuZQzMwp1ypUbI38IY3\nrk4Fhlco05U5wJVbdpCOqmGfzsGkuVtfBK7KrBsKbDKz+4CbSA9ZgDQUsDTk8mlS/gf6/KSxVCDp\n8uw8qRzzSUNAS/OIBpR2999zgG+X5jBJGuZzkwDuB84HziU1tvLK71d2zKqU5skN8eVdSA2lZf76\nHEnXV9n/k2b2nJndCCwGDq9S5VrgGF8+t9Y4M+YA4z1WJB1afp/n8Xv+HUmlYZXjgH8vK3YN6SEY\n0bgKIYQQQuhmPdHAeh64XNIq0sT6O3y9wZYPhBNJc3/agcVm9gBp3kibpHbSwwEm+n7jgCslLScN\n3RrcQEwvAIMqbfAhfT8hNfLagdI8pLyeuOmkBxcsB74JrK5QptL+JdeRHgSyQukx3deWF1B6UMKd\nXZwPwNWkh1csVnqIQgtwBGlOVzvwY9KwS4C7gEckzff8TyLNHXscWNXpyMnhVH8wxATgVEkrSI3Y\nkb6+dK3nAvcBT3mZmfg8Lc/7AGCdmb3eRfkB2WNmSRrqPU3l9iDN31oGLAFeBf7Vt42g+sNEJkha\n6df4A+DhCmWy8VwL3CZpEak3K0/ePfEr0nVY6vfEHZT1NndxrgCXA3eT7vMXK8w3Gwi81EVcoQDx\nd0SKE7ktRuS1GJHX4kRuixF5rZ/SVJmdi8932tfMJlYtHACQNBsYY2ZdNRg+diTdA1xtZtUajzsE\n79VaAZxnZmtyyhgtPRpW72uBov8tjEnCxYncFiPyWozIa3Eit8WIvOaThJl1GlW1szawRgC/BjaW\n/S2sEHZYkg4jDcVcAVxsOW9+STvdPwqDDxzM+nXrqxcMIYQQQnDRwAoh1ERSXtsrhBBCCCG4vAZW\nT8zBCiGEnV6MYS9O5LYYkddiRF6LE7ktRuS1ftHACiGEEEIIIYRuEkMEQwgdxBDBEEIIIYTqYohg\nCCGEEEIIIRQsGlghhNADYgx7cSK3xYi8FiPyWpzIbTEir/WLBlYIIYQQQgghdJOYgxVC6GBn/DtY\nYccxePBw1q9/ubfDCCGEsBOIv4MVQqhJamDFvwvh40rE/2shhBB6QjzkIoQQelVbbwewA2vr7QB2\nSDHvohiR1+JEbosRea1foQ0sScMlrczZ1ippVJH155F0kKSlkh7OrFvbG7HkkXSKpKk1lKs7bklX\nSdotZ9vFkm7z5WZJ46oc62JJzVXKvFdvjNWUjun3WGsN5SdL+pOkd2s8fpf59+0P1B4xSOonaa7f\ne2MlnSTpWX99eN57JbN/1XOVtLukByWtlrRS0vWZbYd5fTPqiTuEEEIIIdSuJ3qwtsexGucAj5rZ\nlzPrtsc4a4mpkbgnAP0b2K/RGIrIreUs55kNHLsNdTSyvdwowMxslJnNBC4CrjezUcCmGo9XS5mb\nzOzTwNHASZLOIFX8gpl9BjhC0iF1xh62WVNvB7ADa+rtAHZITU1NvR3CDinyWpzIbTEir/XriQbW\nrpKmSVol6f5KPSeSLpC0wn9u8HV9JE31dcslXeXrR3gvwDJJzzT4QXEf4I2ydW9m4hnndbZL+o2v\nmyrpVkkLJL0kaYyv30PSPI9luaSzff1w70WYKmmNpOmSTvf910j6nJfrL+luSQslLZF0lofxAfBO\nDefyph9nkse7VNI6P2Z/781o9zyOlXQFMAxolTTf9/2Wx7QQODFz7I2kD/5d2eTlkHSApFl+bdol\nfb6U0kxur5G0yMs0+7opksZnyjRL+l5e+TIfAW9VS5KZLTKz16uVy9iSf++tKuV2iaQ9vMwASTP9\nOt+biX+tpEG+fIxSb+3+wL3AsX6cy4CvA9dl9/V9+ki6UdLTft6X1nquZrbJzB7z5Q+BpcAnyoq9\nTnoPhBBCCCGE7mZmhf0Aw4HNwOf99d3A93y5lfSN/lDgFWAQqcE3Hzjbtz2aOdZe/nshcLYv9wN2\nayCuScCEnG0jgeeBgf56H/89FZjhy58GXvTlvsCevrxvZv1w0of0kf76GeBuXz4bmOXLPwEu9OW9\ngTXA7mUxHQPcWeO57Q0sJ/VejAF+mdk2wH//MXN+QzL53wV4Aritwev9b8CVvqxMfe/679NL8fj2\nB4CTgKOAtsxxngMOzCvvr9+rUP9Q4MEqMb7bwHnNBk7w5f5+n54CvO11CngS+EImv4My1+4PvnwK\nMDtz3KnAmMz9ssKXLwV+mLnHFwPDGzjXfYD/BA4uWz8f+FwX+xk0Z35aDSx+tvkn8tgzucVC92ht\nbe3tEHZIkdfiRG6LEXndqrW11Zqbm7f8+P85lP/sQvH+ZGYLfXkacAXw88z2Y4FWM3sLQNJ04GRg\nMnCIpFuB3wOPStoTGGZms0ln9EG9wUgScKTHUslpwEwze9vr2JDZ9jtft1rSAaVDAlMknUxqTA7L\nbFtrZqt8+Tlgni+vBA725dHAWZJ+4K/7AQeRGlp4fUuAy2o8xWnAzWbWLmkj8DNJU4CHzOyJTMyl\nXqXj6Zj/GcChNdZV7jTg7z1mA8rnXo0GTpe01OvfAzjUzKZK2l/SEOAA4C0ze03ShErlSY3ATszs\nz8BXG4y9KwuAX/i9OctjA1jkdSJpGemaPkmmx65Bo0nD+Mb6671I5/1KqUC1c5XUF7gPuMXMXi7b\nvI70HngmP4SW+qMOIYQQQtiBNTU1dRgyOWnSpIrleqKBZVVeQ4UPpGa2QdKRwBnAd4GxpLlDXX54\n9aFml3o9Z5rZ+sy2PqTehfeBh+o4h5L3K8R8EbAfcLSZbVZ66MRuFcpvzrzezNbcCzjXzF5sIJ4O\nJLWQGrT3AJjZi0oPEjkTmCxpnplNrrTrttbtKl3b8nqmmNldFbbNJF3jIcCMGspXq6vbmNlPJT0I\nfAVYIGm0b8pe34/Yek0/ZOvw24oPE6lCwBVmNreReN2dwBoz+6cK234JzJF0nJl9ZxvqCHVp6u0A\ndmBNvR3ADinmXRQj8lqcyG0xIq/164k5WMMlHe/LFwKPl21fBJwsaZB/634B8JikfYG+ZvZb4EfA\nKDPbCLwq6Wuw5alsu2cPZma3m9nRlh4ksL5s22YzO5j0zf03cuL9AzA2M4dmYE65UqNkb+ANb1yd\nShrqVV6mK3OAK7fsIB1Vwz6dg0lzt74IXJVZNxTYZGb3ATeRhl0CvEvqFQF4mpT/gZJ2JTVyKh3/\n8uw8qRzzgfFevo+kAaXd/fcc4NulOUyShvncJID7gfOBc0mNrbzy+5Uds14d9pN0jjJP2qu4g/RJ\nM3vOzG4kDdc7vEoda0lDAyGdT73mAOMl7eL1H1p+n1eJdzJpSO3VOUWuAS6JxlUIIYQQQvfriQbW\n88DlklaR5oTc4esNwBtBE0l/yKQdWGxmD5Dm4LRJaic9HGCi7zcOuFLSctLQrcENxPQCac5RJz6k\n7yekRl47cHM23mxR/z2d9OCC5cA3gdUVylTav+Q60oNAVig9pvva8gL+oIQ7uzgfgKtJD69Y7A9R\naAGOABb5efyYNOwS4C7gEUnzPf+TSHPbHgdWdTpycjjwlyoxTABOlbSC1Igd6etL13ouadjaU15m\nJrCnb1sFDADWmT+MIqf8gOwxsyQN9Z6mTiT9VNKrwO5Kj2v/sW8aQfWHiUxQeuT5ctK8uocrlMnG\ncy1wm6RFpN6sPHn3xK9I12Gp3xN3UNbbnHeukg4EfgiMzDyY49tlxQYCL3URVyhEW28HsANr6+0A\ndkjxt2+KEXktTuS2GJHX+ilNldm5+Hynfc1sYtXCAQBJs0kPZOiqwfCxI+ke4Gozq9Z43CH4HMQV\nwHlmtianjPXgCMydSBsxlK0obWzNrdgZ/18rQltbWwwNKkDktTiR22JEXvNJwsw6jaraWRtYI4Bf\nAxut49/CCmGHJekw0lDMFcDFlvPmjwZW+HiLBlYIIYSeEQ2sEEJNUgMrhI+nwYOHs379y70dRggh\nhJ1AXgOrJ+ZghRA+Zir9TYf42baf1tbWXo9hR/3J5jYaV90n5l0UI/JanMhtMSKv9YsGVgghhBBC\nCCF0kxgiGELoQJLFvwshhBBCCF2LIYIhhBBCCCGEULBoYIUQQg+IMezFidwWI/JajMhrcSK3xYi8\n1i8aWCGEEEIIIYTQTWIOVgihg5iDFUIIIYRQXd4crF16I5gQwvZN6vRvRQg7ncEHDmb9uvW9HUYI\nIYSPmejBCiF0IMlo6e0odkBrgUN6O4gdVFG5bUl/E25n1dbWRlNTU2+HscOJvBYncluMyGu+eIpg\nCCGEEEIIIRSs0AaWpOGSVuZsa5U0qsj680g6SNJSSQ9n1q3tjVjySDpF0tQaytUdt6SrJO2Ws+1i\nSbf5crOkcVWOdbGk5ipl3qs3xmpKx/R7rLWG8pMl/UnSuzUev8v8+/YHao8YJPWTNNfvvbGSTpL0\nrL8+PO+9ktm/1nMdJWmFpBck3ZJZf5jXN6OeuEM3id6r4kRuCxHfWBcj8lqcyG0xIq/164kerO1x\nfMU5wKNm9uXMuu0xzlpiaiTuCUD/BvZrNIYicms5y3lmA8duQx2NbC83CjAzG2VmM4GLgOvNbBSw\nqcbj1VLmX4BLzOww4DBJZ5AqfsHMPgMcISk+koYQQgghFKAnGli7SpomaZWk+yv1nEi6wL9xXyHp\nBl/XR9JUX7dc0lW+foT3AiyT9EyDHxT3Ad4oW/dmJp5xXme7pN/4uqmSbpW0QNJLksb4+j0kzfNY\nlks629cPl7Ta91sjabqk033/NZI+5+X6S7pb0kJJSySd5WF8ALxTw7m86ceZ5PEulbTOj9lf0oO+\nfoX3mlwBDANaJc33fb/lMS0ETswceyPpg39XNnk5JB0gaZZfm3ZJny+lNJPbayQt8jLNvm6KpPGZ\nMs2SvpdXvsxHwFvVkmRmi8zs9WrlMrbk33urSrldImkPLzNA0ky/zvdm4l8raZAvH6PUW7s/cC9w\nrB/nMuDrwHXZfX2fPpJulPS0n/eltZ6rpCHAADNb7KvuIX2hkPU66T0QetJ21Ue+g4ncFiL+9k0x\nIq/FidwWI/Jav554iuCngG+Z2UJJdwPjgZ+XNkoaCtwAHA1sAOZ6I2UdcKCZfdbL7eW7TCd96z9b\nUj8aayT2BTZnV5jZ8V7PSOCHwAlm9rak7AfRIWZ2oqRPk3pEZgF/Bc4xs42S9gUW+jaAEcC5ZrZK\n0jPA+b7/2V7HGOAfgflmdomkvYFFkuaZ2VPAUx7TMcB3zOyy8hMpxW1mzUCzH+M/gH8GvgS8ZmZf\n9eMMMLP3JF0NNPn5DQFaSPl/F2gDlvoxb66WSDO7P/PyNqDNzMZIErBnqZjXfzpwqJkd59tnSzoJ\nmAHcAtzu5b8OjM4rb2ZP4I02M1sHnOfHHwrcVTrfbZHNP/B9YLyZPSWpP+maAxwFjATWAwskfcHM\nnqRzL5OZ2ZuS/gH4vpmVGuEnAA+Y2SxJwzPlLwE2mNnxfo8vkPSomb1Sw7keSHrvlKzzdVmbSe+B\nfNmBiAcTQ7BCCCGEsNNra2urqcHZEw2sP5nZQl+eBlxBpoFFGrbVamZvAUiaDpwMTAYOkXQr8Hvg\nUUl7AsPMbDaAmX1QbzD+Qf1Ij6WS04CZZva217Ehs+13vm61pANKhwSmSDqZ9MF1WGbbWjNb5cvP\nAfN8eSXpYyvAaOAsST/w1/2Ag4A1pUrNbAnQqXGVYxpws5m1S9oI/EzSFOAhb5iUYi71Kh1Px/zP\nAA6tsa5ypwF/7zEbUD73ajRwuqSlXv8epAbUVEn7e2PvAOAtM3tN0oRK5YEnqMDM/gxsc+OqggXA\nL/zenOWxASzyOpG0jHRNnyTTY9eg0aRhfGP99V6k836lVGAbz3Ud6T3wTG6JUxs8csgXjdTiRG4L\nEfMuihF5LU7kthiR162ampo65GPSpEkVy/VEA6vTt/kVynT6QGpmGyQdCZwBfBcYS5o71OWHVx9q\ndqnXc6aZrc9s6wP8EXgfeKiOcyh5v0LMFwH7AUeb2Walh07sVqH85szrzWzNvUi9XC82EE8HklpI\nDdp7AMzsRaUHiZwJTPaescmVdt3Wul21+UECppjZXRW2zSRd4yGkHq1q5XtszpyZ/VTSg8BXSL1J\no31T9vp+xNZr+iFbe1YrPkykCgFXmNncBvZ9DfjbzOtP+LqsXwJzJB1nZt9poI4QQgghhJCjJ+Zg\nDZd0vC9fCDxetn0RcLKkQZL6AhcAj/lwu75m9lvgR8AoM9sIvCrpa7DlqWy7Zw9mZreb2dH+IIH1\nZds2m9nBpG/uv5ET7x+AsZk5NANzypUaJXsDb3jj6lRgeIUyXZkDXLllB+moGvbpHEyau/VF4KrM\nuqHAJjO7D7iJ9JAFSEMBS0Munyblf6CkXUmNnErHvzw7TyrHfNIQ0NI8ogGl3f33HODbpTlMkob5\n3CSA+4HzgXNJja288vuVHbNeHfaTdI6k67vcQfqkmT1nZjcCi4HDq9SxFjjGl89tIMY5wHhJu3j9\nh5bf53n8nn9HUmlY5Tjg38uKXUN6CEY0rnpSzBMqTuS2EDHvohiR1+JEbosRea1fTzSwngcul7SK\nNLH+Dl9vsOUD4UTS3J92YLGZPUCaN9ImqZ30cICJvt844EpJy0lDtwY3ENMLwKBKG3xI309Ijbx2\noDQPKa8nbjrpwQXLgW8CqyuUqbR/yXWkB4GsUHpM97XlBfxBCXd2cT4AV5MeXrHYH6LQAhxBmtPV\nDvyYNOwS4C7gEUnzPf+TSHPHHgdWdTpycjjwlyoxTABOlbSC1Igd6etL13oucB/wlJeZic/T8rwP\nANaVHkaRU35A9phZkoZ6T1Mnkn4q6VVgd6XHtf/YN42g+sNEJkha6df4A+DhCmWy8VwL3CZpEak3\nK0/ePfEr0nVY6vfEHZT1Nnd1rsDlwN2k+/xFM3ukbPtA4KUu4gohhBBCCA3SzvhX6n2+075mNrFq\n4QCApNnAGDPrqsHwsSPpHuBqM6vWeNwheK/WCuA8M1uTU8Zo6dGwQtg+tcDO+H9kCCGE2kjCzDqN\nqtpZG1gjgF8DG8v+FlYIOyxJh5GGYq4ALracN7+kne8fhRAqGHzgYNavW1+9YAghhJ1SXgOrJ4YI\nbnfM7D/N7O+icRV2Jv6Hho8ys3F5jatM2fjp5p/W1tZej2FH/Skqtzt74yrmXRQj8lqcyG0xIq/1\n2ykbWCGEEEIIIYRQhJ1yiGAIIZ8ki38XQgghhBC6FkMEQwghhBBCCKFg0cAKIYQeEGPYixO5LUbk\ntRiR1+JEbosRea1fNLBCCCGEEEIIoZvEHKwQQgcxByuEEEIIobq8OVi79EYwIYTtW/p7xD1v8ODh\nrF//cq/UHUIIIYTQHWKIYAihAuuVn9dff6VHzq43xBj24kRuixF5LUbktTiR22JEXusXDawQQggh\nhBBC6Cbd0sCSNFzSypxtrZJGdUc99ZJ0kKSlkh7OrFvbG7HkkXSKpKk1lFvrv3NzXVZ+gKRXJd2W\nPYakQXXEVjVXfn0P6mL7xZL+qdY6a4zr4tJ5SWqWNK5K+WMltfvPcknfqKGOqZJOrrJ9TJ1xnyTp\nWb8n/0bSTZJWSvqpn8f3quxfy7l+UdIzfp6LJZ2a2fZ9Sc/Xcv6h+zU1NfV2CDusyG0xIq/FiLwW\nJ3JbjMhr/bpzDtb2OCv+HOBRM5uYWbc9xllLTJaznOc64LEG6tmW8kUfp1ErgWPMbLOkIcCzkv63\nmX3Uw3FcBFxvZvcBSLoUGGhmJqm5m+p4E/iqma2X9D+AOcAnAMzsZklPADcBM7qpvhBCCCGEkNGd\nQwR3lTRN0ipJ90varbyApAskrfCfG3xdH+8NWOHful/l60dImitpmX8jf0gDMe0DvFG27s1MPOO8\nznZJv/F1UyXdKmmBpJdKvRSS9pA0L9M7cLavHy5pte+3RtJ0Saf7/mskfc7L9Zd0t6SFkpZIOsvD\n+AB4p4ZzebN8haS7Mj0zb0j6X77+GOAA4NHyXYArvf7lkg7LnNu/+jVYJul/5tVZwV+Aj/w4X/Jj\nL5M0t0K8+0n635Ke9p8TlKyVtFem3AuS9q9UvkL9G4FNXQVoZn81s83+cnfgnRoaVxtI1wZJN3jP\n0zJJN2bKnFLhPjlF0gOZc/knv88u+X/s3X28XeOd///XWyZpSkhECNLmRvodlY6kSaiaKkfRjpmh\nqqXUXfvwQItSN50avRGlqlW+37T90dGaNCWtikGjmhE0J+0DKXIrEtFqOqhJUKIyo5Tz/v2xrs06\n+6x9d5yVHSef5+NxHvZe67PW+lyfvU/s61zXtTZwJHCRpGsl/QwYAiySdERVnXaRNDeNQC2ovE7A\nC020dZnttenxQ8BgSQNzIWuBoQ3aHkoQc9jLE7UtR9S1HFHX8kRtyxF1bV1fjmDtCnzK9kJJ1wCn\nAldUdkraCbgUmEz24fWO1El5Ahhle2KKq3zQnkX21/45kgbRu87gAKArv8H2Xuk6E4Dzgb1tPydp\nWC5sR9vvk7QbMAe4CfgLcJjtDZK2AxamfQDjgY/aXinpAeCodPyh6RqHA18E7rJ9oqShwH2S7rR9\nL3BvymkqcIrtk6sbUsm7attJ6bjRwFxghiQB3yIbLTmooCZP2Z4q6TPAucDJwJeB9bnXYGitaxbk\n8LF0zAjgamAf249V1bNiOnCF7XskvR243fYESbcAHwFmSnoP8AfbT0uaVR0PTKi6/uWVx5JOyTb5\n6uoLp/P+OzAO+EQT7TorHTec7HV/Z3q+TS6s6H0CBSN2tq+RtA9wq+2b0rn+bHtKepwfwbqa7H3w\naMr7KuAA2/nfp5ptzcV8DFhs+6+5zV009Xs/Lfe4I/2EEEIIIWy+Ojs7m+pw9mUH6zHbC9Pj64DP\nkutgAXsC820/C5A+PO8LXAyMkzQd+AUwT9IQYGfbcwBsv9xqMqmjMSnlUuQDwGzbz6VrrM/tuyVt\nWyVph8opga8rW5fTBeyc27fG9sr0+CHgzvT4QWBsevxB4BBJn0/PBwGjgdWVi9peRNYDx74QAAAg\nAElEQVThaaWdg4HZwOm2n5B0GnCb7SezElB9v+2b038XkXVqAA4EXluXY7uZEbVq7wUW2H4snWN9\nQcyBwG7ptQEYImlL4AbgK8BM4Chen75WK76Q7X+rs+8+4O8k7QrcLmm+7T830a7ngRcl/QC4Dfh5\nbl/R++QNkbQV8PfA7Fy7B1bH1WtrOs+7gK/Ts5P9DLC9pGE1XqNkWvNJh6bEHPbyRG3LEXUtR9S1\nPFHbckRdX9fR0dGtHhdeeGFhXJlrsIrW3fT4ch3b6yVNAj4EfBo4AvhcUWy3E0mnAiel6/xjZVpU\n2rcF8HvgJbIPxa16qSDnY4ARwOS0lmcNMLggviv3PD9aILJRrt/2Ip96rgJutD0/Pd8b2CfVZ2uy\nqZsv2D6/KtdX6fvvQWv05UkC9qoaUQG4V9mU0BFk6+a+Wi9eb+A7mmyvlvQo8H/IOpmN4l9No0gH\nkL03T0+Pofh98grdR1t7TJVtYAvgucrIVm9IehvZaNpxtv+Q32f7RUnXA7+X9HHbPaZyhhBCCCGE\n3uvLNVhjJFWmlH0C+HXV/vuAfSUNlzQAOBpYkKbbDbB9M/AlYIrtDcDjkj4MIGmQpLfmT2b7StuT\nbU/Jd67Svi7bY4EHyI3MVPklcESaAoakbWvEVT44DyWbXtel7M5sYwpi6rkdOOO1A6R3N3FMXWm0\naojtyyrbbB9re6ztXcimAP4o17mq5Q7gtNx5e0zvU7b+bKc651gIvF/SmBRfVM95wJm5c07K7buZ\nbMRzZW5kpV580ySNTe85Un7vAH6bns9UWidX49itgGG2/xM4G5hYKzT997+ACZIGpjoeUCM+f8xr\nbL8ArEnT+yo51LpmUb5DyUbZvpAbUc7vH0b2OzEqOlcbV8xhL0/UthxR13JEXcsTtS1H1LV1fdnB\nehg4TdJKsptLfC9tN0DqBJ0HdAJLgPtt3wqMAjolLQGuTTEAx5PdkGEZcDcwshc5PQIU3pY8Ten7\nGlknbwlQWctTayRuFrBnyudYYFVBTNHxFReRjSYtV3ab9a9WB0iaKqnmmpoC5wC7K7vJxWJJjaYX\n1srtYmC4sluGL6FqwU2aqjYeeLbmie1nyKY33pzOcX1B2JnAHspusLECOCW37wayUcLrm4zvQdIp\nNWqwD7BM0uJ0nZNz0wMnAk/WOe3WwM/T6/4r4Ky0vfB9YvuJdI0VqS2Lq2PqPK84FjhR2U01VgCH\nVgfUaevpZK/VV3LvixG5/UOBdbbr3iwjhBBCCCH0jux230G7PGm903ZVt2kPLUrreT5l+9x259KX\nJG0N/MD2ZvO9UGm643TbRXdkrMS4fXfWF/3536QQQggh9B+SsN1jRlJ/72CNB34IbLB9cJvTCaGt\nJJ1DNkp4me2f1Ilr2z8KI0eOYe3aP7Tr8iGEEEIITavVwerLKYKbHNuP2n5/dK5CyG5pn9Ys1uxc\n5WLb8tOfO1cxh708UdtyRF3LEXUtT9S2HFHX1vXrDlYIIYQQQgghbEz9eopgCKF1khz/LoQQQggh\n1LdZThEMIYQQQgghhI0pOlghhLARxBz28kRtyxF1LUfUtTxR23JEXVsXHawQQgghhBBC6COxBiuE\n0E2swQohhBBCaCzWYIUQmiapz392fNuO7W5WCCGEEELpYgQrhNCNJDOthBNPy75fa3PV2dlJR0dH\nu9Pol6K25Yi6liPqWp6obTmirrXFCFYIIYQQQgghlKxPOliSxkh6sMa++ZKm9MV1WiVptKTFkubm\ntq1pRy61SNpP0owm4tak/9asdVX81pIel/Tt/DkkDW8ht4a1Sq/v6Dr7T5D0nWav2WReJ1TaJekC\nScc3iN9T0pL0s0zSx5u4xgxJ+zbYf3iLee8jaUV6T75F0mWSHpT0jdSOsxsc37CtKe5fJf1W0ipJ\nH8xtP0fSw820P/S9+OtfeaK25Yi6liPqWp6obTmirq3ryxGsTXHuz2HAPNsH57Ztink2k5NrPK7l\nImBBL67zRuLLPk9vPQhMtT0Z+BDw/0ka0IY8jgEusT3F9kvAScBE21/oqwtI2g04EtgNOBi4UpIA\nbF8OnACc1lfXCyGEEEII3fVlB2ugpOskrZR0g6TB1QGSjpa0PP1cmrZtkUYDlqfRhTPT9vGS7pC0\nVNIDksb1IqdhwFNV257O5XN8uuYSSTPTthmSpku6W9LvKqMUkraSdGfKZZmkQ9P2MWmkYIak1ZJm\nSTooHb9a0h4pbktJ10haKGmRpENSGi8DzzfRlqerN0j6fm5k5ilJX07bpwI7APOqDwHOSNdfJulv\nc2379/QaLJX0kVrXLPAn4NV0nn9I514q6Y6CfEdIulHSb9LP3sqskbRNLu4RSdsXxRdcfwPwYr0E\nbf/Fdld6+lbgeduvNmjXerLXBkmXppGnpZK+mYvZr+B9sp+kW3Nt+U56n51I1vG5SNK1kn4GDAEW\nSTqiqk67SJor6X5JCyqvE/BCo7YCHwaut/2K7T8AvwXek9u/Fhja4ByhBPE9IuWJ2pYj6lqOqGt5\norbliLq27m/68Fy7Ap+yvVDSNcCpwBWVnZJ2Ai4FJpN9eL0jdVKeAEbZnpjiKh+0Z5H9tX+OpEH0\nrjM4AOjKb7C9V7rOBOB8YG/bz0kalgvb0fb70mjAHOAm4C/AYbY3SNoOWJj2AYwHPmp7paQHgKPS\n8YemaxwOfBG4y/aJkoYC90m60/a9wL0pp6nAKbZPrm5IJe+qbSel40YDc4EZabTiW2SjJQcV1OQp\n21MlfQY4FzgZ+DKwPvcaDK11zYIcPpaOGQFcDexj+7GqelZMB66wfY+ktwO3254g6RbgI8BMSe8B\n/mD7aUmzquOBCVXXv7zyWNIp2SZfXX3hdN5/B8YBn2iiXWel44aTve7vTM+3yYUVvU+gYMTO9jWS\n9gFutX1TOtefbU9Jjy/IhV9N9j54NOV9FXCA7fzvU622jiK9n5I/pm0VXTTzez8/93gsWdVCCCGE\nEDZjnZ2dTXU4+7KD9ZjthenxdcBnyXWwgD2B+bafBUgfnvcFLgbGSZoO/AKYJ2kIsLPtOQC2X241\nmdTRmJRyKfIBYLbt59I11uf23ZK2rZK0Q+WUwNeVrcvpAnbO7Vtje2V6/BBwZ3r8INnHU4APAodI\n+nx6PggYDayuXNT2IrIOTyvtHAzMBk63/YSk04DbbD+ZlYDqO5vcnP67iKxTA3Ag8Nq6HNvNjKhV\ney+wwPZj6RzrC2IOBHZLrw3AEElbAjcAXwFmAkcBP20QX8j2v9XZdx/wd5J2BW6XNN/2n5to1/PA\ni5J+ANwG/Dy3r+h98oZI2gr4e2B2rt0Dq+PqtbWBZ4DtJQ2r8Rpl9u/l2UNNMYe9PFHbckRdyxF1\nLU/UthxR19d1dHR0q8eFF15YGNeXHazqv9oXrbvpcRtD2+slTSJbG/Np4Ajgc0Wx3U4knUq2hsXA\nP9pem9u3BfB74CWyD8Wteqkg52OAEcBk213KbgAxuCC+K/c8P1ogslGu3/Yin3quAm60XRlz2BvY\nJ9Vna7Kpmy/YPr8q11fp29cfGrxmaf9etv9atf1eZVNCR5Ctm/tqvfjX+x2ts71a0qPA/yHrZDaK\nfzWNIh1A9t48PT2G4vfJK3Qfbe0xVbaBLYDnKiNbvfBH4O25529L2wCw/aKk64HfS/q47R5TOUMI\nIYQQQu/15RqsMZIqU8o+Afy6av99wL6Shiu7wcDRwII03W6A7ZuBLwFTbG8AHpf0YQBJgyS9NX8y\n21fanpxuGLC2al+X7bHAA+RGZqr8EjgiTQFD0rY14iofnIeSTa/rkrQ/MKYgpp7bgTNeO0B6dxPH\n1JVGq4bYvqyyzfaxtsfa3oVsCuCPcp2rWu4gd+ODoul9ytaf7VTnHAuB90sak+KL6jkPODN3zkm5\nfTeTjXiuzI2s1ItvmqSx6T1Hyu8dZGuTkDRTaZ1cjWO3AobZ/k/gbGBirdD03/8CJkgamOp4QI34\n/DGvsf0CsEbSx3I51LpmkTnAUel3ZhxZW+/LnWsY2e/EqOhcbVwxh708UdtyRF3LEXUtT9S2HFHX\n1vVlB+th4DRJK8luLvG9tN0AqRN0HtAJLAHut30r2fqQTklLgGtTDMDxZDdkWAbcDYzsRU6PAIW3\nJU9T+r5G1slbAlTW8tQaiZsF7JnyORZYVRBTdHzFRWSjScuV3Wb9q9UBkqZK6rF+qI5zgN2V3eRi\nsaRG0wtr5XYxMFzZLcOXAB1VeYlsndmzNU9sP0M2vfHmdI7rC8LOBPZQdoONFcApuX03kI0SXt9k\nfA+STqlRg32AZZIWp+ucnJseOBF4ss5ptwZ+nl73XwFnpe2F7xPbT6RrrEhtWVwdU+d5xbHAicpu\nqrECOLQ6oFZb0/v6BmAl2ZTbU939232HAutsN7pZRgghhBBC6AV1/+zVv6T1TtvZPq9hcKhJ0rvI\nbmBybrtz6UuStgZ+YHuz+V6oNN1xuu2iOzJWYsy0Ei4+DfrzvzchhBBC2LxIwnaPGUn9vYM1Hvgh\nsKHqu7BC2OxIOodslPAy2z+pE1fKPwojR41k7RNrGweGEEIIIbwJ1Opg9eUUwU2O7Udtvz86VyFk\nt7RPaxZrdq5ysX3+s7l3rmIOe3mituWIupYj6lqeqG05oq6t69cdrBBCCCGEEELYmPr1FMEQQusk\nOf5dCCGEEEKob7OcIhhCCCGEEEIIG1N0sEIIYSOIOezlidqWI+pajqhreaK25Yi6ti46WCGEEEII\nIYTQR2INVgihm1iDFUIIIYTQWK01WH/TjmRCCJs2qce/FWEjGTlyDGvX/qHdaYQQQgihl2KKYAih\ngOOnz3/mNxW3bt1/NfMChZxYH1COqGs5oq7lidqWI+rauuhghRBCCCGEEEIf6ZMOlqQxkh6ssW++\npCl9cZ1WSRotabGkublta9qRSy2S9pM0o4m4Nem/NWtdFb+1pMclfTt/DknDW8itYa3S6zu6zv4T\nJH2n2Ws2mdcJlXZJukDS8Q3ih0v6paQX8vVocMwMSfs22H94i3nvI2lFek++RdJlkh6U9I3UjrMb\nHN9MWw+U9ICkZZLul7R/bt85kh6W9PFW8g59paPdCfRbHR0d7U6hX4q6liPqWp6obTmirq3ryxGs\nTXFV/GHAPNsH57Ztink2k5NrPK7lImBBL67zRuLLPk9v/QX4EnBOm/M4BrjE9hTbLwEnARNtf6EP\nr/E08M+2JwGfBK6t7LB9OXACcFofXi+EEEIIIeT0ZQdroKTrJK2UdIOkwdUBko6WtDz9XJq2bZFG\nA5anv7qfmbaPl3SHpKXpL/LjepHTMOCpqm1P5/I5Pl1ziaSZadsMSdMl3S3pd5VRCklbSbozNzpw\naNo+RtKqdNxqSbMkHZSOXy1pjxS3paRrJC2UtEjSISmNl4Hnm2jL09UbJH0/5b5E0lOSvpy2TwV2\nAOZVHwKcka6/TNLf5tr27+k1WCrpI7WuWeBPwKvpPP+Qzr1U0h0F+Y6QdKOk36SfvZVZI2mbXNwj\nkrYvii+4/gbgxXoJ2v5f2/cALzXRnor1ZK8Nki5NI09LJX0zF7NfwftkP0m35trynfQ+OxE4ErhI\n0rWSfgYMARZJOqKqTrtImptGoBZUXifghSbausz22vT4IWCwpIG5kLXA0BbqEPpMZ7sT6LdifUA5\noq7liLqWJ2pbjqhr6/ryLoK7Ap+yvVDSNcCpwBWVnZJ2Ai4FJpN9eL0jdVKeAEbZnpjiKh+0Z5H9\ntX+OpEH0rjM4AOjKb7C9V7rOBOB8YG/bz0kalgvb0fb7JO0GzAFuIhsFOcz2BknbAQvTPoDxwEdt\nr5T0AHBUOv7QdI3DgS8Cd9k+UdJQ4D5Jd9q+F7g35TQVOMX2ydUNqeRdte2kdNxoYC4wQ5KAb5GN\nlhxUUJOnbE+V9BngXOBk4MvA+txrMLTWNQty+Fg6ZgRwNbCP7ceq6lkxHbjC9j2S3g7cbnuCpFuA\njwAzJb0H+IPtpyXNqo4HJlRd//LKY0mnZJt8daO8m2jXWemcw8le93em59vkworeJ1AwYmf7Gkn7\nALfavimd68+2p6THF+TCryZ7Hzya6nEVcIDt/O9Tw7ZK+hiw2PZfc5u7aOr3flrucQcxvS2EEEII\nm7vOzs6mOpx92cF6zPbC9Pg64LPkOljAnsB8288CpA/P+wIXA+MkTQd+AcyTNATY2fYcANsvt5pM\n6mhMSrkU+QAw2/Zz6Rrrc/tuSdtWSdqhckrg68rW5XQBO+f2rbG9Mj1+CLgzPX4QGJsefxA4RNLn\n0/NBwGhgdeWitheRdXhaaedgYDZwuu0nJJ0G3Gb7yawEVN9v++b030VknRqAA4HX1uXYbmZErdp7\ngQW2H0vnWF8QcyCwW3ptAIZI2hK4AfgKMBM4Cvhpg/hCtv+tF3k38jzwoqQfALcBP8/tK3qfvCGS\ntgL+Hpida/fA6rhGbZX0LuDr9OxkPwNsL2lYjdcomdZ80qFJHe1OoN+K9QHliLqWI+panqhtOaKu\nr+vo6OhWjwsvvLAwri87WNV/tS9ad9Pjy3Vsr5c0CfgQ8GngCOBzRbHdTiSdSraGxcA/VqZFpX1b\nAL8nmxJ2WwttqMhPJavkcQwwAphsu0vZDSAGF8R35Z7nRwtENsr1217kU89VwI2256fnewP7pPps\nTTZ18wXb51fl+ip9/z1ojb48ScBeVSMqAPcqmxI6gmzd3FfrxWsjfkeT7VfTKNIBZO/N09NjKH6f\nvEL30dYeU2Ub2AJ4rjKy1RuS3kY2mnac7T/k99l+UdL1wO8lfdx2j6mcIYQQQgih9/pyDdYYSZUp\nZZ8Afl21/z5gX2V3dBsAHA0sSNPtBti+mexGBFNsbwAel/RhAEmDJL01fzLbV9qenG4YsLZqX5ft\nscAD5EZmqvwSOCJNAUPStjXiKh+ch5JNr+tSdme2MQUx9dwOnPHaAdK7mzimrjRaNcT2ZZVtto+1\nPdb2LmRTAH+U61zVcge5Gx8UTe9Ttv5spzrnWAi8X9KYFF9Uz3nAmblzTsrtu5lsxHNlbmSlXnxv\ndXutJM1UWidXGJyNKA2z/Z/A2cDEBuf9L2CCpIGpjgfUiO+RC4DtF4A1aXpfJYda1yzKdyjZKNsX\nciPK+f3DyH4nRkXnamPrbHcC/VasDyhH1LUcUdfyRG3LEXVtXV92sB4GTpO0kuzmEt9L2w2QOkHn\nkX3KWALcb/tWYBTQKWkJ2R3PzkvHHU92Q4ZlwN3AyF7k9AhQeFvyNKXva2SdvCVAZS1PrZG4WcCe\nKZ9jgVUFMUXHV1xENpq0XNlt1r9aHSBpqqRW1g+dA+yu7CYXiyU1ml5YK7eLgeHKbhm+hKq5TGmq\n2njg2Zontp8hm954czrH9QVhZwJ7KLvBxgrglNy+G8hGCa9vMr4HSafUqkEacbwcOEHSY5LemXZN\nBJ6sc9qtgZ+n1/1XwFlpe+H7xPYTqS0rUlsWV8fUeV5xLHCisptqrAAOLWhPrbaeTvZafSX3vhiR\n2z8UWGe77s0yQgghhBBC78hu9x20y5PWO21n+7yGwaGmtJ7nU7bPbXcufUnS1sAPbG823wuVpjtO\nt110R8ZKjNt/Z/3NmejP/y6HEEII/YUkbPeYkdTfO1jjgR8CG6q+CyuEzY6kc8hGCS+z/ZM6cdHB\naqvoYIUQQghvBrU6WH05RXCTY/tR2++PzlUI2S3t05rFmp2r1yl+2vQzcmR+eWdoRqwPKEfUtRxR\n1/JEbcsRdW1dX99FLoTQD8QISt/r7OyMW92GEEIIm4F+PUUwhNA6SY5/F0IIIYQQ6tsspwiGEEII\nIYQQwsYUHawQQtgIYg57eaK25Yi6liPqWp6obTmirq2LDlYIIYQQQggh9JFYgxVC6CbWYIUQQggh\nNBZrsEIIIYQQQgihZNHBCiH0ICl+evGz49t2rFnTmMNenqhtOaKu5Yi6lidqW46oa+vie7BCCD1N\na3cCb07rpq1rdwohhBBCaLNYgxVC6EaSo4PVS9PiS5pDCCGEzUWpa7AkjZH0YI198yVN6YvrtErS\naEmLJc3NbVvTjlxqkbSfpBlNxK1J/61Z66r4rSU9Lunb+XNIGt5Cbg1rlV7f0XX2nyDpO81es8m8\nTqi0S9IFko5vED9c0i8lvZCvR4NjZkjat8H+w1vMex9JK9J78i2SLpP0oKRvpHac3eD4hm1Ncf8q\n6beSVkn6YG77OZIelvTxVvIOIYQQQgjN68s1WJvin20PA+bZPji3bVPMs5mcXONxLRcBC3pxnTcS\nX/Z5eusvwJeAc9qcxzHAJban2H4JOAmYaPsLfXUBSbsBRwK7AQcDV0oSgO3LgROA0/rqeqF5MYe9\nPFHbckRdyxF1LU/UthxR19b1ZQdroKTrJK2UdIOkwdUBko6WtDz9XJq2bZFGA5ZLWibpzLR9vKQ7\nJC2V9ICkcb3IaRjwVNW2p3P5HJ+uuUTSzLRthqTpku6W9LvKKIWkrSTdmXJZJunQtH1MGimYIWm1\npFmSDkrHr5a0R4rbUtI1khZKWiTpkJTGy8DzTbTl6eoNkr6fcl8i6SlJX07bpwI7APOqDwHOSNdf\nJulvc2379/QaLJX0kVrXLPAn4NV0nn9I514q6Y6CfEdIulHSb9LP3sqskbRNLu4RSdsXxRdcfwPw\nYr0Ebf+v7XuAl5poT8V6stcGSZemkaelkr6Zi9mv4H2yn6Rbc235TnqfnUjW8blI0rWSfgYMARZJ\nOqKqTrtImivpfkkLKq8T8EKjtgIfBq63/YrtPwC/Bd6T278WGNpCHUIIIYQQQgv68iYXuwKfsr1Q\n0jXAqcAVlZ2SdgIuBSaTfXi9I3VSngBG2Z6Y4ioftGeR/bV/jqRB9K4zOADoym+wvVe6zgTgfGBv\n289JGpYL29H2+9JowBzgJrJRkMNsb5C0HbAw7QMYD3zU9kpJDwBHpeMPTdc4HPgicJftEyUNBe6T\ndKfte4F7U05TgVNsn1zdkEreVdtOSseNBuYCM9JoxbfIRksOKqjJU7anSvoMcC5wMvBlYH3uNRha\n65oFOXwsHTMCuBrYx/ZjVfWsmA5cYfseSW8Hbrc9QdItwEeAmZLeA/zB9tOSZlXHAxOqrn955bGk\nU7JNvrpR3k2066x0zuFkr/s70/NtcmFF7xMoGLGzfY2kfYBbbd+UzvVn21PS4wty4VeTvQ8eTfW4\nCjjAdv73qVZbR5HeT8kf07aKLpr5vZ+fezwW6M2fN0I3HR0d7U6h34raliPqWo6oa3mituWIur6u\ns7OzqRG9vuxgPWZ7YXp8HfBZch0sYE9gvu1nAdKH532Bi4FxkqYDvwDmSRoC7Gx7DoDtl1tNJnU0\nJqVcinwAmG37uXSN9bl9t6RtqyTtUDkl8HVl63K6gJ1z+9bYXpkePwTcmR4/SPbxFOCDwCGSPp+e\nDwJGA6srF7W9iKzD00o7BwOzgdNtPyHpNOA2209mJaB64d3N6b+LyDo1AAcCr63Lsd3MiFq19wIL\nbD+WzrG+IOZAYLf02gAMkbQlcAPwFWAmcBTw0wbxhWz/Wy/ybuR54EVJPwBuA36e21f0PnlDJG0F\n/D0wO9fugdVxb6CtzwDbSxpW4zXK7N/Ls4cQQggh9FMdHR3dOpwXXnhhYVyZa7CK1t30uMtG+pA3\nCegEPg18v1ZstxNJp6apcYsl7Vi1bwtgDdk6lNuayr67/FSySh7HACOAybYnk009HFwQ35V7nh8t\nENko1+T0M872at64q4AbbVfGHPYGTpf0e7KRrOMkXVLQtlfp+9v0133N0v69cjUYnabv3QuMT6Ng\nhwH/US++j3Ouy/arZFPsbgT+GfjP3O6i98krdP+96jFVtoEtgOfSOq1Ku/+uheP/CLw99/xtaRsA\ntl8Ergd+L6lohDOUJOawlydqW46oazmiruWJ2pYj6tq6vuxgjZFUmVL2CeDXVfvvA/ZVdke3AcDR\nwII03W6A7ZvJbkQwxfYG4HFJHwaQNEjSW/Mns31l+vA5xfbaqn1dtscCD5AbmanyS+CINAUMSdvW\niKt8cB5KNr2uS9L+wJiCmHpuB8547QDp3U0cU1carRpi+7LKNtvH2h5rexeyKYA/sn1+g1PdQe7G\nB0XT+5StP9upzjkWAu+XNCbFF9VzHnBm7pyTcvtuJhvxXJkbWakX31vdXitJM5XWyRUGZyNKw2z/\nJ3A2MLHBef8LmCBpYKrjAc3mAmD7BWCNpI/lcqh1zSJzgKPS78w44B1kv3uVcw0j+50YZbvHOrkQ\nQgghhPDG9GUH62HgNEkryW4u8b203QCpE3Qe2UjVEuB+27eSrQ/plLQEuDbFABxPdkOGZcDdwMhe\n5PQIUHhb8jSl72tknbwlQGUtT62RuFnAnimfY4FVBTFFx1dcRHYjkOXKbrP+1eoASVMltbJ+6Bxg\n99xIXqPphbVyuxgYruyW4UuAjqq8RLbO7NmaJ7afIZveeHM6x/UFYWcCeyi7wcYK4JTcvhvIRgmv\nbzK+B0mn1KqBslvOXw6cIOkxSe9MuyYCT9Y57dbAz9Pr/ivgrLS98H1i+4nUlhWpLYurY+o8rzgW\nOFHZTTVWAIcWtKewrel9fQOwkmzK7anu/sVMQ4F1aSQrbEQxh708UdtyRF3LEXUtT9S2HFHX1vXr\nLxpO6522s31ew+BQk6R3kd3A5Nx259KXJG0N/MD2ZvO9UOmmGdNtF92RsRITXzTcW9Pii4ZDCCGE\nzYVqfNFwf+9gjQd+CGyo+i6sEDY7ks4hGyW8zPZP6sT1338USjZy1EjWPrG2cF9nZ2f8FbAkUdty\nRF3LEXUtT9S2HFHX2mp1sPr6JgebFNuPAu9vdx4hbArSLe0vbxhIjMKEEEIIIfRWvx7BCiG0TpLj\n34UQQgghhPpqjWD15U0uQgghhBBCCGGzFh2sEELYCOJ7RMoTtS1H1LUcUdfyRG3LEXVtXXSwQggh\nhBBCCKGPxBqsEEI3sQYrhBBCCKGxWIMVQgghhBBCCCWLDlYIoQdJ8dPPfnbcccPzrOIAACAASURB\nVGy731alifUB5Yi6liPqWp6obTmirq3r19+DFULorZgi2Pc6gY62XX3duh4zGEIIIYRQgliDFULo\nRpKjg9UfKb5AOoQQQuhDm9QaLEljJD1YY998SVM2dk7p2qMlLZY0N7dtTTtyqUXSfpJmNBG3Jv23\nZq2r4reW9Likb+e2zZc0usFxMyTt2yDfWxtdvxX5c0o6QdIFTRzzDUkPSlou6cgm4i+QdHyD/We3\nmPeukpZIWiRpnKQzJK2UdG1qx3caHN+wrZImSbontXVpvq2Sjpb0sKSzWsk7hBBCCCE0r51rsDbF\nP6UeBsyzfXBu26aYZzM5ucbjWi4CFvQunZZyKeOcdc8v6R+BdwMTgfcC50oaUkJOjRwGzLY91fYa\n4DPAgbaPS/tbfV2L/A9wnO3dgYOB/ydpGwDbPwH2A6KD1Rad7U6g34r1AeWIupYj6lqeqG05oq6t\na2cHa6Ck69Jf8G+QNLg6IP3FfXn6uTRt2yKNmiyXtEzSmWn7eEl3pL/aPyBpXC9yGgY8VbXt6Vw+\nx6drLpE0M22bIWm6pLsl/U7S4Wn7VpLuTLksk3Ro2j5G0qp03GpJsyQdlI5fLWmPFLelpGskLUwj\nHoekNF4Gnm+iLU9Xb5D0/ZT7EklPSfpy2j4V2AGYV3XIn4BXG1xnfcoJSXumdixNeW9Vdf3CNkm6\nV9Juubj5kqbUqUHei8CGBjlOAH7lzP8Cy4F/aHDMC+ncpJGmh1K7fpyLeVfK9XeSPptiu40YSjon\njXYdDHwO+IykuyRdBewCzK28h3PHjJB0o6TfpJ+9m22r7d/ZfjQ9/m+y9/P2uf3rgKEN2h5CCCGE\nEHqpnTe52BX4lO2Fkq4BTgWuqOyUtBNwKTCZ7EP8HamT8gQwyvbEFLdNOmQWcIntOZIG0bvO4wCg\nK7/B9l7pOhOA84G9bT8naVgubEfb70udhDnATcBfgMNsb5C0HbAw7QMYD3zU9kpJDwBHpeMPTdc4\nHPgicJftEyUNBe6TdKfte4F7U05TgVNsn1zdkEreVdtOSseNBuYCMyQJ+BZwDHBQVfzHGhXM9lnp\nnAOB64EjbC9OI0QvVoUXtikd93FgmqQdUz0XS/pajfj89W+oPE4dsKm2p1VddxnwFUlXAFsB+wMP\nNWjXFbmnXwDG2v5r7v0G2Xu4g6zDslrSlZXDe57OcyV9D3ihcm5JHwI60vvphFz8dOAK2/dIejtw\nOzChybaSi3kPMLDS4cpp4ncjf9oO2nlzhv6jo90J9FsdHR3tTqFfirqWI+panqhtOaKur+vs7Gxq\nRK+dHazHbC9Mj68DPkuugwXsCcy3/SyApFnAvsDFwDhJ04FfAPPSh/mdbc8BsP1yq8mkjsaklEuR\nD5BN73ouXWN9bt8tadsqSTtUTgl8Xdn6pC5g59y+NbZXpscPAZVOw4PA2PT4g8Ahkj6fng8CRgOr\nKxe1vQjo0blq0M7BwGzgdNtPSDoNuM32k1kJ6O2txnYFnrS9OOW2IV0vH1OrTbPJRs+mAUcCNzaI\nL2T7VqDHei/bd0jaE7iHbETnHhqPzOUtA34s6RbSa53cZvsV4E+S1gEjWzgnZLUuqveBwG56vXhD\nJG2ZRt+A2m197cTZHyh+BBxXsPtZSeMLOl450xomH0IIIYSwOeno6OjW4bzwwgsL4zalNVhFa0t6\nfPhMHZtJZAsaPg18v1ZstxNJp6apcYvTKEl+3xbAGmA34Lamsu/upYKcjwFGAJNtTyb7YD+4IL4r\n97yL1zu9Ihvlmpx+xtlezRt3FXCj7fnp+d7A6ZJ+TzaSdZykS3p57kads8I22X4SeEbS7mQjWT/N\nHdMnNbB9STrHh8je94+0cPg/Ad8FpgD3p/cL9Hwd/wZ4hWwktKLH1NcmCNgr1+7R+c5Vw4OlrYGf\nA/9q+/6CkOnAUkmf7EVuodc6251AvxXrA8oRdS1H1LU8UdtyRF1b184O1hhJlWlsnwB+XbX/PmBf\nScMlDQCOBhak6XYDbN8MfAmYkkZLHpf0YQBJgyS9NX8y21emD6tTbK+t2tdleyzwANkH/CK/BI6Q\nNDxdY9sacZVOxlDgKdtdkvYHxhTE1HM7cMZrB0jvbuKYutJo1RDbl1W22T7W9ljbuwDnAj+yfX7B\nsTOV1ofVsBrYMU1bRNKQ9Lrl1WvTT4F/AbaxvaKJ+KYpW7dXed0mAruT1ptJuqTyvqlxrIDRthcA\n5wHbAPVukLEO2F7StpLeAvxzL1KeB7y2LkvSpGYPTFM1bwFmpt+RIucD77D9w17kFkIIIYQQ6mhn\nB+th4DRJK8luLvG9tN0AqRN0HtmffZcA96dpUaOATklLgGtTDMDxwBmSlgF30/p0LchGNYYX7UhT\n+r5G1slbAlyezzcfmv47C9gz5XMssKogpuj4iovIbgSyPN004avVAZKmSrq6TnuqnQPsnhvJa2V6\n4UTgyVo7bf+VrHP6XUlLyToJb6kKq9em/6Dn6NXFdeJ7kHSIpGkFuwYCv5a0gux9dqztylq73YG1\nBcdUDACuS6/jImC67T8XxFXet6+kPO8n6yCuKojtdkyBM4E9lN0cZQVwSnVAnbYeCewDfDL3Ok+s\nihmUbnYRNqqOdifQb8X6gHJEXcsRdS1P1LYcUdfWxRcN56S1PtvZPq9h8GYkTTn7ge1ao3tvWpLm\nVt2Wv19L6wCX2d6pTkx80XC/FF80HEIIIfQlbUpfNLwJuwl4n3JfNBzA9gv9sXMFsJl1ro4mG1n8\nZhPR8dPPfkaOzM9S7l9ifUA5oq7liLqWJ2pbjqhr69p5F8FNTrqr2vvbnUcIZUhfNPyTJmNLzmbz\n09nZGdMsQgghhM1ATBEMIXQjyfHvQgghhBBCfTFFMIQQQgghhBBKFh2sEELYCGIOe3mituWIupYj\n6lqeqG05oq6tiw5WCCGEEEIIIfSRWIMVQugm1mCFEEIIITQWa7BCCCGEEEIIoWTRwQoh9CApfuKn\n9J8d37Zjn7xfY31AOaKu5Yi6lidqW46oa+vie7BCCD1Na3cC/dAaYFy7k9i0rJu2rt0phBBCCH0u\n1mCFELqR5OhghY1iWnypdQghhDcvaRNagyVpjKQHa+ybL2nKxs4pXXu0pMWS5ua2rWlHLrVI2k/S\njCbi1qT/1qx1VfzWkh6X9O3ctvmSRjc4boakfRvke2uj67cif05JJ0i6oIljviHpQUnLJR3ZRPwF\nko5vsP/sFvPeVdISSYskjZN0hqSVkq5N7fhOg+ObbesJkh6RtDrfBklHS3pY0lmt5B1CCCGEEJrX\nzjVYm+KfLQ8D5tk+OLdtU8yzmZxc43EtFwELepdOS7mUcc6655f0j8C7gYnAe4FzJQ0pIadGDgNm\n255qew3wGeBA28el/a2+rj1I2hb4CrAnsBdwgaShALZ/AuwHRAerHTapP9X0L7E+oBxR13JEXcsT\ntS1H1LV17exgDZR0XfoL/g2SBlcHpL+4L08/l6ZtW6RRk+WSlkk6M20fL+kOSUslPSCpN6sdhgFP\nVW17OpfP8emaSyTNTNtmSJou6W5Jv5N0eNq+laQ7Uy7LJB2ato+RtCodt1rSLEkHpeNXS9ojxW0p\n6RpJC9OIxyEpjZeB55toy9PVGyR9P+W+RNJTkr6ctk8FdgDmVR3yJ+DVBtdZn3JC0p6pHUtT3ltV\nXb+wTZLulbRbLm6+pCl1apD3IrChQY4TgF8587/AcuAfGhzzQjo3aaTpodSuH+di3pVy/Z2kz6bY\nbiOGks5Jo10HA58DPiPpLklXAbsAcyvv4dwxIyTdKOk36WfvFtr6IbI/Ejxvez3Za/paW22vA4Y2\nOEcIIYQQQuildt7kYlfgU7YXSroGOBW4orJT0k7ApcBksg/xd6ROyhPAKNsTU9w26ZBZwCW250ga\nRO86jwOArvwG23ul60wAzgf2tv2cpGG5sB1tvy91EuYANwF/AQ6zvUHSdsDCtA9gPPBR2yslPQAc\nlY4/NF3jcOCLwF22T0wjEPdJutP2vcC9KaepwCm2T65uSCXvqm0npeNGA3OBGZIEfAs4BjioKv5j\njQpm+6x0zoHA9cARthenEaIXq8IL25SO+zgwTdKOqZ6LJX2tRnz++jdUHqcO2FTb06quuwz4iqQr\ngK2A/YGHGrTritzTLwBjbf81936D7D3cQdZhWS3pysrhPU/nuZK+B7xQObekDwEd6f10Qi5+OnCF\n7XskvR24HZjQZFtHAY/nnv8xbctr/LsxP/d4LHFzhr4QNSxNR0dHu1Pol6Ku5Yi6lidqW46o6+s6\nOzubGtFrZwfrMdsL0+PrgM+S62CRTXGab/tZAEmzgH2Bi4FxkqYDvwDmpQ/zO9ueA2D75VaTSR2N\nSSmXIh8gm971XLrG+ty+W9K2VZJ2qJwS+Lqy9UldwM65fWtsr0yPHwIqnYYHyT7OAnwQOETS59Pz\nQcBoYHXlorYXAT06Vw3aORiYDZxu+wlJpwG32X4yKwE9Fuo1aVfgSduLU24b0vXyMbXaNJtspGUa\ncCRwY4P4QrZvBXqs97J9h6Q9gXvIRijvofHIXN4y4MeSbiG91slttl8B/iRpHTCyhXNCVuuieh8I\n7KbXizdE0pZp9A2o3dYmPStpvO1Ha0bs38szhxBCCCH0Ux0dHd06nBdeeGFh3Ka0BqtobUmPD5+p\nYzMJ6AQ+DXy/Vmy3E0mnpqlxi9MoSX7fFmQrJHYDbmsq++5eKsj5GGAEMNn2ZLIP9oML4rtyz7t4\nvdMrslGuyelnnO3VvHFXATfaroxR7A2cLun3ZCNZx0m6pJfnbtQ5K2yT7SeBZyTtTjaS9dPcMX1S\nA9uXpHN8iOx9/0gLh/8T8F1gCnB/er9Az9fxb4BXyEZCK3pMfW2CgL1y7R6d71w18Ee6d0Lflrbl\nTQeWSvpkL3ILvRVrsEoT6wPKEXUtR9S1PFHbckRdW9fODtYYSZVpbJ8Afl21/z5gX0nDJQ0AjgYW\npOl2A2zfDHwJmJJGSx6X9GEASYMkvTV/MttXpg+rU2yvrdrXZXss8ADZB/wivwSOkDQ8XWPbGnGV\nTsZQ4CnbXZL2B8YUxNRzO3DGawdI727imLrSaNUQ25dVttk+1vZY27sA5wI/sn1+wbEzldaH1bAa\n2DFNW0TSkPS65dVr00+BfwG2sb2iifimKVu3V3ndJgK7k9abSbqk8r6pcayA0bYXAOcB2wD1bpCx\nDthe0raS3gL8cy9Snge8ti5L0qQWjr0dOEjS0PQePShtyzsfeIftH/YitxBCCCGEUEc7O1gPA6dJ\nWkl2c4nvpe0GSJ2g88hGqpYA96dpUaOATklLgGtTDMDxwBmSlgF30/p0LchGNYYX7UhT+r5G1slb\nAlyezzcfmv47C9gz5XMssKogpuj4iovIbgSyPN004avVAZKmSrq6TnuqnQPsnhvJa2V64UTgyVo7\nbf+VrHP6XUlLyToJb6kKq9em/6Dn6NXFdeJ7kHSIpGkFuwYCv5a0gux9dqztylq73YG1BcdUDACu\nS6/jImC67T8XxFXet6+kPO8n69isKojtdkyBM4E9lN0cZQVwSnVArbamKawXkf2x4DfAhVXTWQEG\npZtdhI0p1mCVJtYHlCPqWo6oa3mituWIurYuvmg4J6312c72eQ2DNyOStgZ+YLvW6N6blqS5Vbfl\n79fSOsBltneqExNfNBw2jmnxRcMhhBDevFTji4ajg5UjaTzwQ2DD5vShO2weJB1NdkfEmbb/b524\n+EchbBQjR41k7RP1BpCb09nZGX9hLUHUtRxR1/JEbcsRda2tVgernXcR3OSku6q9v915hFCG9EXD\nP2kytuRsNj/xP6gQQghh8xAjWCGEbiQ5/l0IIYQQQqiv1ghWO29yEUIIIYQQQgj9SnSwQghhI4jv\nESlP1LYcUddyRF3LE7UtR9S1ddHBCiGEEEIIIYQ+EmuwQgjdxBqsEEIIIYTGYg1WCCGEEEIIIZQs\nbtMeQuhB6vHHmBBCCG8CI0eOYe3aP7Q7jbaIr8MoR9S1ddHBCiEUiCmCfa8T6GhzDv1VJ1HbMnQS\ndS1DJ2XWdd26+ANZCO0Wa7BCCN1IcnSwQgjhzUrxZfEhbCSb1BosSWMkPVhj33xJUzZ2TunaoyUt\nljQ3t21NO3KpRdJ+kmY0Ebcm/bdmravit5b0uKRv57bNlzS6wXEzJO3bIN9bG12/FflzSjpB0gVN\nHPNqem2XSLqlifgLJB3fYP/ZLea9a7r+IknjJJ0haaWka1M7vtPg+IZtlTRJ0j2SHpS0VNKRuX1H\nS3pY0lmt5B1CCCGEEJrXzptcbIp/XjkMmGf74Ny2TTHPZnJyjce1XAQs6F06LeVSxjmbOf//2J5i\ne7Ltw0rIpxmHAbNtT7W9BvgMcKDt49L+Vl/XIv8DHGd7d+Bg4P9J2gbA9k+A/YDoYLVFZ7sT6Mc6\n251AP9XZ7gT6qc52J9Bvxfc1lSPq2rp2drAGSrou/QX/BkmDqwPSX9yXp59L07Yt0qjJcknLJJ2Z\nto+XdEf6q/0Dksb1IqdhwFNV257O5XN8uuYSSTPTthmSpku6W9LvJB2etm8l6c6UyzJJh6btYySt\nSsetljRL0kHp+NWS9khxW0q6RtLCNOJxSErjZeD5JtrydPUGSd9PuS+R9JSkL6ftU4EdgHlVh/wJ\neLXBddannJC0Z2rH0pT3VlXXL2yTpHsl7ZaLmy9pSp0a5L0IbGiQI0Crk9JfSOcmjTQ9lNr141zM\nu1Kuv5P02RTbbcRQ0jlptOtg4HPAZyTdJekqYBdgbuU9nDtmhKQbJf0m/ezdbFtt/872o+nxf5O9\nn7fP7V8HDG2xFiGEEEIIoUntvMnFrsCnbC+UdA1wKnBFZaeknYBLgclkH+LvSJ2UJ4BRtiemuG3S\nIbOAS2zPkTSI3nUeBwBd+Q2290rXmQCcD+xt+zlJw3JhO9p+X+okzAFuAv4CHGZ7g6TtgIVpH8B4\n4KO2V0p6ADgqHX9ousbhwBeBu2yfKGkocJ+kO23fC9ybcpoKnGL75OqGVPKu2nZSOm40MBeYIUnA\nt4BjgIOq4j/WqGC2z0rnHAhcDxxhe7GkIaQOSk5hm9JxHwemSdox1XOxpK/ViM9f/4bK49QBm2p7\nWkGqb0m1fhn4hu2fNWjXFbmnXwDG2v5r7v0G2Xu4g6zDslrSlZXDe57OcyV9D3ihcm5JHwI60vvp\nhFz8dOAK2/dIejtwOzChhbZWYt4DDKx0uHKa+N3In7aDWOjeFzranUA/1tHuBPqpjnYn0E91tDuB\nfivudFeOqOvrOjs7mxrRa2cH6zHbC9Pj64DPkutgAXsC820/CyBpFrAvcDEwTtJ04BfAvPRhfmfb\ncwBsv9xqMqmjMSnlUuQDZNO7nkvXWJ/bd0vatkrSDpVTAl9Xtj6pC9g5t2+N7ZXp8UNApdPwIDA2\nPf4gcIikz6fng4DRwOrKRW0vAnp0rhq0czAwGzjd9hOSTgNus/1kVoKWR3oqdgWetL045bYhXS8f\nU6tNs8lGz6YBRwI3NogvZPtWoNZ6rzG2/zuNbP5S0vI0Ta8Zy4AfK1u7lV+/dZvtV4A/SVoHjGzy\nfBWiuN4HArvp9eINkbSl7f+tBDRoa+UPFD8CjivY/ayk8QUdr5xpDZMPIYQQQticdHR0dOtwXnjh\nhYVxm9IarKK1JT0+fKaOzSSyScyfBr5fK7bbiaRT09S4xWmUJL9vC2ANsBtwW1PZd/dSQc7HACOA\nybYnk03VGlwQ35V73sXrnV6RjXJNTj/jbK/mjbsKuNH2/PR8b+B0Sb8nG8k6TtIlvTx3o85ZYZts\nPwk8I2l3spGsn+aO6ZMapOlypE5VJ9nIaLP+CfguMAW4P71foOfr+DfAK2QjoRU9pr42QcBeuXaP\nzneuGh4sbQ38HPhX2/cXhEwHlkr6ZC9yC73W2e4E+rHOdifQT3W2O4F+qrPdCfRbsVaoHFHX1rWz\ngzVGUmUa2yeAX1ftvw/YV9JwSQOAo4EFabrdANs3A18CpqTRksclfRhA0iBJb82fzPaV6cPqFNtr\nq/Z12R4LPED2Ab/IL4EjJA1P19i2RlylkzEUeMp2l6T9gTEFMfXcDpzx2gHSu5s4pq40WjXE9mWV\nbbaPtT3W9i7AucCPbJ9fcOxMpfVhNawGdkzTFpE0JL1uefXa9FPgX4BtbK9oIr5pkoalaaNIGgG8\nD1iZnl9Sed/UOFbAaNsLgPOAbYAhdS63Dthe0raS3gL8cy9Snge8ti5L0qRmD0xTNW8BZqbfkSLn\nA++w/cNe5BZCCCGEEOpoZwfrYeA0SSvJbi7xvbTdAKkTdB7Zn3qWAPenaVGjgE5JS4BrUwzA8cAZ\nkpYBd9P6dC2AR4DhRTvSlL6vkXXylgCX5/PNh6b/zgL2TPkcC6wqiCk6vuIishuBLE83TfhqdYCk\nqZKurtOeaucAu+dG8lqZXjgReLLWTtt/JeucflfSUrJOwluqwuq16T/oOXp1cZ34HiQdImlawa7d\ngAfS63YX2Vq9h9O+3YG1BcdUDACuS6/jImC67T8XxFXet6+kPO8n6yCuKojtdkyBM4E9lN0cZQVw\nSnVAnbYeCewDfDL3Ok+sihmUbnYRNqqOdifQj3W0O4F+qqPdCfRTHe1OoN+KtULliLq2Lr5oOCet\n9dnO9nkNgzcjacrZD2zXGt1705I0t+q2/P1aWge4zPZOdWLii4ZDCOFNK75oOISNRZvSFw1vwm4C\n3qfcFw0HsP1Cf+xcAWxmnaujyUYWv9nuXDZPne1OoB/rbHcC/VRnuxPopzrbnUC/FWuFyhF1bV07\n7yK4yUl3VXt/u/MIoQzpi4Z/0lx0b28mGUIIoZ1GjhzTOCiEUKqYIhhC6EaS49+FEEIIIYT6Yopg\nCCGEEEIIIZQsOlghhLARxBz28kRtyxF1LUfUtTxR23JEXVsXHawQQgghhBBC6COxBiuE0E2swQoh\nhBBCaCzWYIUQQgghhBBCyaKDFUIIG0HMYS9P1LYcUddyRF3LE7UtR9S1dfE9WCGEHqT4HqwQQghh\nYxo5aiRrn1jb7jRCH4g1WCGEbiSZae3OIoQQQtjMTIP4XP7mEmuwQgghhBBCCKFkbelgSRoj6cEa\n++ZLmrKxc0rXHi1psaS5uW1r2pFLLZL2kzSjibg16b81a10Vv7WkxyV9O7dtvqTRDY6bIWnfBvne\n2uj6rcifU9IJki5o4phX02u7RNItTcRfIOn4BvvPbjHvXdP1F0kaJ+kMSSslXZva8Z0Gxzfb1hMk\nPSJpdb4Nko6W9LCks1rJO/SRTepfkn4maluOqGs5oq7lidqWItZgta6da7A2xTHQw4B5ts/LbdsU\n82wmJ9d4XMtFwILepdNSLmWcs5nz/4/ttnTccw4DZtu+BEDSZ4ADbD8p6QRaf117kLQt8BVgCiBg\nkaSf2f8/e2cfblVV7f/PFxJRUURNULqgcXu8Wr6B/MyrF4/mS2q+XJOMMK3bY5amll7NtKuH1NRK\nf3kzNZMfcgVNLVSUSEQ5ZCKCcDioIGlhvl3UUgrN9zN+f8yxYZ199uvpLDYcxud59nPWmnPMOccc\na7FZY40x57a/mtmtkh4E5gP/9x+ZSBAEQRAEQVCaRqYIbiRpkr/Bv11S32IBf+O+2D+Xe1kvj5os\nltQm6UwvHybpfkmLJD0maccu6LQl8EpR2asZfU70MVslTfSyCZKulvSwpGckHevlm0ma6bq0STrK\ny4dKWurtlkmaLOlgb79M0l4ut6mk8ZLmesTjSFfjXeCvNczl1eICST933VslvSLpv7x8BLAtMKOo\nyV+AD6qMs9J1QtJIn8ci13uzovFLzknSI5J2zsjNkjS8gg2yvAW8UUVHSM5GPazyvvFI05M+r1sy\nMh93XZ+RdLrLdogYSjrbo12HAd8Evi7pAUnXAR8Fphfu4UybbST9UtKj/tmnjrkeSnpJ8FczW0m6\npp8uVJrZy0D/Om0RdAdd+UYKaiNsmw9h13wIu+ZH2DYXmpqaGq3CekcjI1g7AV82s7mSxgOnAlcV\nKiVtB1wO7El6iL/fnZQXgMFmtpvLbeFNJgPfN7OpkvrQNeexN9CeLTCzvX2cXYDzgX3M7HVJW2bE\nBpnZvu4kTAWmAG8Dx5jZG5K2BuZ6HcAw4LNmtkTSY8Dnvf1RPsaxwAXAA2b2FUn9gXmSZprZI8Aj\nrtMI4BQz+2rxRAp6F5Wd7O2GANOBCZIE/AgYCxxcJH9cNYOZ2be8z42AXwCjzWyhpH64g5Kh5Jy8\n3fFAs6RBbs+Fki4tI58d//bCsTtgI8ysuYSqG7ut3wWuMLO7q8zrqszpt4EdzOy9zP0G6R5uIjks\nyyRdW2jeuTubLul6YFWhb0mHAk1+P52Ukb8auMrM5kj6J+A+YJca5zoYeD5z/qKXZan+b2NW5ngH\n4j+tIAiCIAg2eFpaWmpKmWykg/Wcmc3140nA6WQcLGAkMMvMXgOQNBkYBVwC7CjpauDXwAx/mN/e\nzKYCmNm79SrjjsburkspDiSld73uY6zM1N3lZUslbVvoErhMaX1SO7B9pm65mS3x4yeBgtPwOOlx\nFuAQ4EhJ5/h5H2AIsKwwqJktADo5V1Xm2Re4A/iGmb0g6TRgmqepFfTuCjsBL5nZQtftDR8vK1Nu\nTneQIi3NwOeAX1aRL4mZ3QOUW+811Mz+1yObD0pabGa1Zmu3Abcord3Krt+aZmbvA3+R9DIwsMb+\nCojS9j4I2FlrjNdP0qZm9veCQJW5VuM1ScPM7A9lJQ7oYs9BeZYTjmpehG3zIeyaD2HX/Ajb5kJL\nS0tEsZympqYOthg3blxJuXVpDVaptSWdHj7NbKWk3UmpUF8DRpNSryo6BpJOBU72cQ43sxWZul7A\nH4F3gGl1zKHAOyV0HgtsA+xpZu1Km070LSHfnjlvZ801ESnK9XQX9KnEdcAvzawQo9gH2M/tszkp\ndXOVmZ3fhb6rOWdl5yTpz5J2JUWyTslUdZL3KFddmNn/+t/lklpIkdFaif3A3AAAIABJREFUHawj\nSM79UcAFkj7h5cXX8UPA+6RIaIFOqa81IGBvM3uvC21fJEXVCnyEjvEoSBGyRZJON7ObujBGEARB\nEARBUIZGrsEaKqmQxvYF4KGi+nnAKElbSeoNjAFme7pdbzO7E/guMNyjJc9LOhpAUh9Jm2Q7M7Nr\nzWxPMxueda68rt3MdgAeIz3gl+JBYLSkrXyMAWXkCk5Gf+AVd64OAIaWkKnEfcAZqxtIe9TQpiIe\nrepnZj8slJnZCWa2g5l9FPhP4H9KOVeSJsrXh5VhGTDI0xaR1M+vW5ZKc7oNOBfYwsyeqEG+ZiRt\n6WmjSNoG2BdY4uffL9w3ZdoKGGJms4HzgC2AfhWGexn4sKQBkjYGPtMFlWcAq9dl+QuFWrkPOFhS\nf79HD/ayLOcD/xzO1Vom3qrmR9g2H8Ku+RB2zY+wbS5E9Kp+GulgPQWcJmkJaXOJ673cANwJOg9o\nAVqB+Z4WNRhokdQK3OwyACcCZ0hqAx6m/nQtgN8DW5Wq8JS+S0lOXitwZVbfrKj/nQyMdH1OAJaW\nkCnVvsDFpGjSYt804XvFApJGSLqhwnyKORvYVWmTi4WS6kkv3A14qVylR1uOB66RtIjkJGxcJFZp\nTr/y9rdlyi6pIN8JSUdKai5RtTPwmF+3B0hr9Z7yul2BSj+b3huY5NdxAXC1mf2thFzhvn3f9ZxP\ncmyWlpDt0KYEZwJ7KW2O8gQdI3pA+bl6CuvFpJcFjwLjitJZAfr4ZhdBEARBEARBN6P4xeg1+Fqf\nrYu2ad/gkbQ5cKOZlYvurbdImm5mhzVaj7WFrwNsM7PtKsgYzWtPpw2GWBuQH2HbfAi75kPYNT/W\nd9s2w7r4XB5rsMojCTPrlJnWyDVY6yJTgJs2tIfuapjZKsqnTq7XbEjXWdIY0o6IP6gq3Jy3NkEQ\nBEEQZBk4uCvJV8G6SESwgiDogCSL74UgCIIgCILKlItgNXINVhAEQRAEQRAEQY8iHKwgCIK1QC0/\nTBh0jbBtPoRd8yHsmh9h23wIu9ZPOFhBEARBEARBEATdRKzBCoKgA7EGKwiCIAiCoDqxBisIgiAI\ngiAIgiBnwsEKgiBYC0QOe36EbfMh7JoPYdf8CNvmQ9i1fuJ3sIIg6ITUKdodBEEQ9FAGDhzKihXP\nNlqNIOgxxBqsIAg6IMkgvheCIAg2HEQ8DwZB/cQarCAIgiAIgiAIgpxpiIMlaaikx8vUzZI0fG3r\n5GMPkbRQ0vRM2fJG6FIOSftLmlCD3HL/W9bWRfKbS3pe0n9nymZJGlKl3QRJo6roe0+18esh26ek\nkyRdVEOb6ZJelzS1xjEuknRilfqzatcaJO0kqVXSAkk7SjpD0hJJN/s8flKlfdW5Stpd0hxJj0ta\nJOlzmboxkp6S9K169A66i5ZGK9CDaWm0Aj2UlkYr0ENpabQCPZZYK5QPYdf6aWQEa12MRR8DzDCz\nwzJl66KetehkZY7LcTEwu2vq1KVLHn3W0v8PgBNy0KMejgHuMLMRZrYc+DpwkJl90evrva6leBP4\nopntChwG/FjSFgBmdiuwPxAOVhAEQRAEQU400sHaSNIkf4N/u6S+xQL+xn2xfy73sl4eNVksqU3S\nmV4+TNL9/tb+MUk7dkGnLYFXispezehzoo/ZKmmil02QdLWkhyU9I+lYL99M0kzXpU3SUV4+VNJS\nb7dM0mRJB3v7ZZL2crlNJY2XNNcjHke6Gu8Cf61hLq8WF0j6ueveKukVSf/l5SOAbYEZRU3+AnxQ\nZZyVrhOSRvo8FrnemxWNX3JOkh6RtHNGbpak4RVskOUt4I0qOmJms2qRy7DK+8YjTU/6vG7JyHzc\ndX1G0uku2yFiKOlsj3YdBnwT+LqkByRdB3wUmF64hzNttpH0S0mP+mefWudqZs+Y2R/8+H9J9/OH\nM/UvA/3rsEPQbTQ1WoEeTFOjFeihNDVagR5KU6MV6LE0NTU1WoUeSdi1fhq5i+BOwJfNbK6k8cCp\nwFWFSknbAZcDe5Ie4u93J+UFYLCZ7eZyW3iTycD3zWyqpD50zXnsDbRnC8xsbx9nF+B8YB8ze13S\nlhmxQWa2rzsJU4EpwNvAMWb2hqStgbleBzAM+KyZLZH0GPB5b3+Uj3EscAHwgJl9RVJ/YJ6kmWb2\nCPCI6zQCOMXMvlo8kYLeRWUne7shwHRggiQBPwLGAgcXyR9XzWBm9i3vcyPgF8BoM1soqR/uoGQo\nOSdvdzzQLGmQ23OhpEvLyGfHv71w7A7YCDNrrqZ3DfO6KnP6bWAHM3svc79BuoebSA7LMknXFpp3\n7s6mS7oeWFXoW9KhQJPfTydl5K8GrjKzOZL+CbgP2KXeuUr6P8BGBYcrQw3/NrLdNhEPBEEQBEEQ\nbOi0tLTUlDLZSAfrOTOb68eTgNPJOFjASGCWmb0GIGkyMAq4BNhR0tXAr4EZ/jC/vZlNBTCzd+tV\nxh2N3V2XUhxISu963cdYmam7y8uWStq20CVwmdL6pHZg+0zdcjNb4sdPAgWn4XFgBz8+BDhS0jl+\n3gcYAiwrDGpmC4BOzlWVefYF7gC+YWYvSDoNmGZmLyUT0NX9uXcCXjKzha7bGz5eVqbcnO4gRc+a\ngc8Bv6wiXxIzuwfo1vVeThtwi6S78GvtTDOz94G/SHoZGFhnv6K0vQ8CdtYa4/WTtKmZ/b0gUG2u\n/oLif4Avlqh+TdKwEo5Xhuaqygf10kI4qnnRQtg2D1oIu+ZBC2HXfGhpaYloSw6EXdfQ1NTUwRbj\nxo0rKddIB6vTW/4SMp0ePs1spaTdgUOBrwGjSalXFR0DSacCJ/s4h5vZikxdL+CPwDvAtDrmUOCd\nEjqPBbYB9jSzdqVNJ/qWkG/PnLez5pqIFOV6ugv6VOI64JeeMgewD7Cf22dzUurmKjM7vwt9V3PO\nys5J0p8l7UqKZJ2Sqeok71GutckRJOf+KOACSZ/w8uLr+CHgfVIktECn1NcaELC3mb3XhbZI2hy4\nF/iOmc0vIXI1sEjS6WZ2U1fGCIIgCIIgCErTyDVYQyUV0ti+ADxUVD8PGCVpK0m9gTHAbE+3621m\ndwLfBYZ7tOR5SUcDSOojaZNsZ2Z2rZntaWbDs86V17Wb2Q7AY6QH/FI8CIyWtJWPMaCMXMHJ6A+8\n4s7VAcDQEjKVuA84Y3UDaY8a2lTEo1X9zOyHhTIzO8HMdjCzjwL/CfxPKedK0kT5+rAyLAMGedoi\nkvr5dctSaU63AecCW5jZEzXId4VOESNJ3y/cNyUbpCjSEDObDZwHbAH0qzDGy8CHJQ2QtDHwmS7o\nOQNYvS7LXyjUhKdq3gVM9H8jpTgf+OdwrtY2TY1WoAfT1GgFeihNjVagh9LUaAV6LBFlyYewa/00\n0sF6CjhN0hLS5hLXe7kBuBN0HimW3grM97SowUCLpFbgZpcBOBE4Q1Ib8DD1p2sB/B7YqlSFp/Rd\nSnLyWoErs/pmRf3vZGCk63MCsLSETKn2BS4mRZMW+6YJ3ysWkDRC0g0V5lPM2cCuSptcLJRUT3rh\nbsBL5So92nI8cI2kRSQnYeMisUpz+pW3vy1TdkkF+U5IOlJSc5m633rfB0p6TlJhvdmuwIpSbZze\nwCS/jguAq83sbyXkCvft+67nfJKDuLSEbIc2JTgT2Etpc5Qn6BjRK8yn3Fw/B+wHfClznXcrkunj\nm10EQRAEQRAE3Yzil7vX4Gt9tjaz86oKb0B4ytmNZlYuurfeIml60bb8PRpfB9hmZttVkLF189cJ\n1ndaiDfXedFC2DYPWgi75kEL655dRU94Hoy1QvkQdi2PJMysU2ZaI9dgrYtMAW7a0B66q2Fmqyif\nOrlesyFdZ0ljSDsi/qAG6bzVCYIgCNYRBg4cWl0oCIKaiQhWEAQdkGTxvRAEQRAEQVCZchGsRq7B\nCoIgCIIgCIIg6FGEgxUEQbAWqOWHCYOuEbbNh7BrPoRd8yNsmw9h1/oJBysIgiAIgiAIgqCbiDVY\nQRB0INZgBUEQBEEQVCfWYAVBEARBEARBEORMOFhBEARrgchhz4+wbT6EXfMh7JofYdt8CLvWT/wO\nVhAEnZDid7CCIAiCoLsZOHggK15Y0Wg1gpyJNVhBEHRAktHcaC2CIAiCoAfSDPHs3XOINVhBEARB\nEARBEAQ50xAHS9JQSY+XqZslafja1snHHiJpoaTpmbLljdClHJL2lzShBrnl/resrYvkN5f0vKT/\nzpTNkjSkSrsJkkZV0feeauPXQ7ZPSSdJuqiGNtMlvS5pao1jXCTpxCr1Z9WuNUjaSVKrpAWSdpR0\nhqQlkm72efykSvta53qSpN9LWpadg6Qxkp6S9K169A66iXXqm6SHEbbNh7BrPoRd8yNsmwuxBqt+\nGhnBWhfjo8cAM8zssEzZuqhnLTpZmeNyXAzM7po6demSR5+19P8D4IQc9KiHY4A7zGyEmS0Hvg4c\nZGZf9Pp6r2snJA0ALgRGAnsDF0nqD2BmtwL7A+FgBUEQBEEQ5EQjHayNJE3yN/i3S+pbLOBv3Bf7\n53Iv6+VRk8WS2iSd6eXDJN0vaZGkxyTt2AWdtgReKSp7NaPPiT5mq6SJXjZB0tWSHpb0jKRjvXwz\nSTNdlzZJR3n5UElLvd0ySZMlHeztl0nay+U2lTRe0lyPeBzparwL/LWGubxaXCDp5657q6RXJP2X\nl48AtgVmFDX5C/BBlXFWuk5IGunzWOR6b1Y0fsk5SXpE0s4ZuVmShlewQZa3gDeq6IiZzapFLsMq\n7xuPND3p87olI/Nx1/UZSae7bIeIoaSzPdp1GPBN4OuSHpB0HfBRYHrhHs602UbSLyU96p996pjr\noaSXBH81s5Wka/rpjB1eBvrXYYegu+jKN1JQG2HbfAi75kPYNT/CtrnQ1NTUaBXWOxq5i+BOwJfN\nbK6k8cCpwFWFSknbAZcDe5Ie4u93J+UFYLCZ7eZyW3iTycD3zWyqpD50zXnsDbRnC8xsbx9nF+B8\nYB8ze13SlhmxQWa2rzsJU4EpwNvAMWb2hqStgbleBzAM+KyZLZH0GPB5b3+Uj3EscAHwgJl9xSMQ\n8yTNNLNHgEdcpxHAKWb21eKJFPQuKjvZ2w0BpgMTJAn4ETAWOLhI/rhqBjOzb3mfGwG/AEab2UJJ\n/XAHJUPJOXm744FmSYPcngslXVpGPjv+7YVjd8BGmFlzNb1rmNdVmdNvAzuY2XuZ+w3SPdxEcliW\nSbq20LxzdzZd0vXAqkLfkg4Fmvx+OikjfzVwlZnNkfRPwH3ALjXOdTDwfOb8RS/LUv3fxqzM8Q7E\nf1pBEARBEGzwtLS01JQy2UgH6zkzm+vHk4DTyThYpBSnWWb2GoCkycAo4BJgR0lXA78GZvjD/PZm\nNhXAzN6tVxl3NHZ3XUpxICm963UfY2Wm7i4vWypp20KXwGVK65Page0zdcvNbIkfPwkUnIbHSY+z\nAIcAR0o6x8/7AEOAZYVBzWwB0Mm5qjLPvsAdwDfM7AVJpwHTzOylZAK6uj/3TsBLZrbQdXvDx8vK\nlJvTHaRISzPwOeCXVeRLYmb3AN263stpA26RdBd+rZ1pZvY+8BdJLwMD6+xXlLb3QcDOWmO8fpI2\nNbO/FwT+wbm+JmmYmf2hrMQBXew5KM9ywlHNi7BtPoRd8yHsmh9h21xoaWmJKJbT1NTUwRbjxo0r\nKddIB6vTW/4SMp0ePs1spaTdSalQXwNGk1KvKjoGkk4FTvZxDjezFZm6XsAfgXeAaXXMocA7JXQe\nC2wD7Glm7UqbTvQtId+eOW9nzTURKcr1dBf0qcR1wC89ZQ5gH2A/t8/mpNTNVWZ2fhf6ruaclZ2T\npD9L2pUUyTolU9VJ3qNca5MjSM79UcAFkj7h5cXX8UPA+6RIaIFOqa81IGBvM3uvC21fJEXVCnyE\njvEoSBGyRZJON7ObujBGEARBEARBUIZGrsEaKqmQxvYF4KGi+nnAKElbSeoNjAFme7pdbzO7E/gu\nMNyjJc9LOhpAUh9Jm2Q7M7NrzWxPMxueda68rt3MdgAeIz3gl+JBYLSkrXyMAWXkCk5Gf+AVd64O\nAIaWkKnEfcAZqxtIe9TQpiIerepnZj8slJnZCWa2g5l9FPhP4H9KOVeSJsrXh5VhGTDI0xaR1M+v\nW5ZKc7oNOBfYwsyeqEG+K3SKGEn6fuG+KdkgRZGGmNls4DxgC6BfhTFeBj4saYCkjYHPdEHPGcDq\ndVn+QqFW7gMOltTf79GDvSzL+cA/h3O1lom3qvkRts2HsGs+hF3zI2ybCxG9qp9GOlhPAadJWkLa\nXOJ6LzcAd4LOA1qAVmC+p0UNBloktQI3uwzAicAZktqAh6k/XQvg98BWpSo8pe9SkpPXClyZ1Tcr\n6n8nAyNdnxOApSVkSrUvcDEpmrTYN034XrGApBGSbqgwn2LOBnZV2uRioaR60gt3A14qV+nRluOB\nayQtIjkJGxeJVZrTr7z9bZmySyrId0LSkZKay9T91vs+UNJzkgrrzXYFKv2kem9gkl/HBcDVZva3\nEnKF+/Z913M+ybFZWkK2Q5sSnAnspbQ5yhN0jOgV5lNyrp7CejHpZcGjwLiidFaAPr7ZRRAEQRAE\nQdDNKH5Neg2+1mdrMzuvqvAGhKTNgRvNrFx0b71F0vSibfl7NL4OsM3MtqsgYzSvPZ02GGJtQH6E\nbfMh7JoPYdf8WB9s2wzr27N3rMEqjyTMrFNmWiPXYK2LTAFu2tAeuqthZqsonzq5XrMhXWdJY0g7\nIv6gqnBz3toEQRAEwYbHwMFdSbAK1jcighUEQQckWXwvBEEQBEEQVKZcBKuRa7CCIAiCIAiCIAh6\nFOFgBUEQrAVq+WHCoGuEbfMh7JoPYdf8CNvmQ9i1fsLBCoIgCIIgCIIg6CZiDVYQBB2INVhBEARB\nEATViTVYQRAEQRAEQRAEORMOVhAEwVogctjzI2ybD2HXfAi75kfYNh/CrvUTv4MVBEEnpE7R7qCb\nGDhwKCtWPNtoNYIgCIIgyIlYgxUEQQckGcT3Qn6I+N4NgiAIgvWfWIMVBEEQBEEQBEGQMzU5WJKG\nSnq8TN0sScO7V63akDRE0kJJ0zNlyxuhSzkk7S9pQg1yyzPy95STkbRVN+pVcpxMfUW9/b6YVUWm\n2++PbJ+1XG9Ju0maI6lN0t2S+tXQpmK/klbVrvHqNj+U9LikKyRtI2mupAWS9qvl2tY41x9IWipp\nkaRfSdoiU/dbSfMkbVuv7kF30NJoBXossT4gH8Ku+RB2zY+wbT6EXeunngjWupjTcgwww8wOy5St\ni3rWopOVOa63n3qo1l+9ejeCWsa/ETjXzHYH7gTO7YZ+uzLvk4HdzOzbwEHAYjMbYWa/q7G/WmRm\nAB83sz2Ap4HvrG5sNgpYABxRt+ZBEARBEARBTdTjYG0kaZKkJZJul9S3WEDSGEmL/XO5l/WSNMHL\n2iSd6eXDJN3vb9ofk7RjF/TfEnilqOzVjD4n+pitkiZ62QRJV0t6WNIzko718s0kzXRd2iQd5eVD\nPSIwQdIySZMlHeztl0nay+U2lTQ+E5U40tV4F/hrDXN5NXPcX9K9kp6SdG2mfHWOp6SzPBqyOGPT\nTb1dq5eP9vKRru8i12+z7MCSpnkksFXSSklfrFHvD4DXvI9emQjNIkmnFQu73ea4jW9zfQ+VdHtG\nZnVkTdIhxfJV7FaOj7kTAzAT+GwNbV51HQZJmu32WSxp3zWq6hKf6xxJH/bCCYV7ys9X+d+7gX7A\nAknnAlcAx3i/fel4bcdKetTrrpNW7zhRda5mNtPM2v10LvCRIpEVpH83wVqnqdEK9FiampoarUKP\nJOyaD2HX/Ajb5kPYtX7q2UVwJ+DLZjZX0njgVOCqQqWk7YDLgT2BlcD97qS8AAw2s91crpCyNBn4\nvplNldSHrq0H6w20ZwvMbG8fZxfgfGAfM3tdUvahcpCZ7StpZ2AqMAV4GzjGzN6QtDXp4XSqyw8D\nPmtmSyQ9Bnze2x/lYxwLXAA8YGZfkdQfmCdpppk9AjziOo0ATjGzrxZPpKC3MxLYGXgOuE/SsWY2\npVCplB53ksv1Bh6V1OJ6vmhmn3G5zSVtBPwCGG1mC5XS494qGvuITL//D7jLzFYV9C6Hmb0AHOen\nXwWGkiI0VmRv3KbfBT5lZm+5k3EWcBnwM0mbmNlbwPHALS5/QQn5S8rZTdI04CtmtqJI1SclHWVm\nU4HP0dnpKDW3Qr9fAH5jZpe5o1Nw8jYD5pjZdyVdQYpOfb9UV97f0ZL+ZmaF1MaXgRFmdoafF+bw\nL26DfzWzDyT9FBgLTKpxrln+g3Tts7ST7pkqNGeOmwjnIAiCIAiCDZ2WlpaaUibrcWqeM7O5fjwJ\n2K+ofiQwy8xe8zfok4FRwB+BHT1qdCiwyh/yt/cHXszsXTN7uw5d8Ifd3UkOXCkOBO4ws9d9jJWZ\nuru8bClQWI8i4DJJbaQox/Zas1ZluZkt8eMnvR7gcWAHPz4EOE9SK2mxRR9gSFYhM1tQyrkqwTwz\n+5OlrcZupbOt9wPuNLO3zexNkoP4b67PwZIuk7SfO0k7AS+Z2ULX4Y1MhGM1krYBbgbGeLt6OQj4\nmetcbG+ATwK7AA+7jU4EhpjZB8BvgCMl9Salr00tJ19JATM7oozD8R/AaZLmkxyjd+uY13zgy5Iu\nJDmPb3r5O2b2az9ewJr7oJha9zsvpP99ChgOzPd5Hwh8tJNw+bmmQaULgPfM7JaiqheB3aqr05z5\nNFUXD2qgpdEK9FhifUA+hF3zIeyaH2HbfAi7rqGpqYnm5ubVn3LUE8EqXv9Raj1Ip4dJM1spaXfg\nUOBrwGjgm6VkO3QknUqKChhwePZhUlIvkuP2DjCtjjkUeKeEzmOBbYA9zaxdaUOBviXk2zPn7ayx\noUhRrqe7oE8xtdi6cyOzpz0KdThwsaQHSM5kNVv3Ijlyze505oFI6+XGlqi7DfgG8Dow38zedAe6\nnHxdmNnvSfcfkj5GHWuQzOwhSaO8zU2SrjSzScB7GbEPWHMfvI+/uPA5bFSnugImmtkFdbZb04H0\nJdI9cGCJ6inAhZKWmNkuXR0jCIIgCIIgKE09EayhkrJpUw8V1c8DRknayiMRY4DZnurV28zuJKWI\nDTezN4DnJR0NIKmPpE2ynZnZtWa2p5kNL35Tb2btZrYD8BgpnaoUDwKj5TuzSRpQRq7gfPQHXnHn\n6gBSuluxTCXuA85Y3UDao4Y25dhbae1XL9L8im39EGn9Tl+l9VT/DjzkaZpvedTiR6RIyDJgkKcn\nIqmfX58sVwBtZnZHKWWU1nBNrKLz/cAphb5L2HsusK+kYV6/qTs7ALNd15NZk9JWSb4uMuujepHu\nwev9fHtJM6u0HUK6L8aTNsso7IhY7p54FtjLj4+mo4NV6T4q1D0AHJfReYDrUBOSPg2cAxxlZu+U\nEDkRmB7OVSNoarQCPZZYH5APYdd8CLvmR9g2H8Ku9VOPg/UUKc1qCWmR/PVeXkgJWwGcR8qDaSVF\nIu4BBgMtnu50s8tAetA7w1PyHgYGdkH/3wMlt7b2lL5LSU5eK3BlVt+sqP+dDIx0fU4AlpaQKdW+\nwMWkjUAWK21p/71iAUkjJN1QYT4F5gHXkNIR/2Bmd2XHNrNW4CZS+tojwA1m1gbsSlr71QpcCFxi\nZu+RnLRrJC0i7TK3cdF4ZwOHKG1ysVDSZ4rqhwB/r6LzjcDzwGIff0yRzn8GvgTc6jaeQ0pfxFMW\n7wU+7X8rylPmGiht1jGoRNUYScuAJaQ1ajd5+XZ0jESVoglok7SQtH7rx5V0AH4O7O82+CTwZqau\nUiSyYKelJCdwhs97BtBpThXm+hPSZhr3+7W8tqh+AGl3wSAIgiAIgiAH5Etm1ksknQNsbWbnVRUO\nuoxv4nCzmT3RaF26E6WdDv9kZvc2Wpe1hW+asdjMflZBxhq/+35PpIXkr4v1+Xt3XaSlpSXesOZA\n2DUfwq75EbbNh7BreSRhZp0ylOpZg7UuMoW0LmZ60W9hBd2I/25Tj8PMftpoHdYmkmaT1g2W2u0w\nCIIgCIIg6AbW6whWEATdT4pgBXkxcOBQVqx4ttFqBEEQBEHwD9JTI1hBEORAvHgJgiAIgiDoGl35\ncd8gCIKgTuJ3RPIjbJsPYdd8CLvmR9g2H8Ku9RMOVhAEQRAEQRAEQTcRa7CCIOiAJIvvhSAIgiAI\ngsqUW4MVEawgCIIgCIIgCIJuIhysIAiCtUDksOdH2DYfwq75EHbNj7BtPoRd6yccrCAIgiAIgiAI\ngm4i1mAFQdCB+B2s9ZOBgwey4oUVjVYjCIIgCDYYyq3BCgcrCIIOSDKaG61FUDfN8ftlQRAEQbA2\niU0ugiAIGsnyRivQc4n1AfkQds2HsGt+hG3zIexaPzU5WJKGSnq8TN0sScO7V63akDRE0kJJ0zNl\n69RjjKT9JU2oQW55Rv6ecjKStupGvUqOk6mvqLffF7OqyHT7/ZHts5brLWk3SXMktUm6W1K/GtpU\n7FfSqto1Xt3mh5Iel3SFpG0kzZW0QNJ+tVzbGuc6QNIMScsk3Sepf6but5LmSdq2Xt2DIAiCIAiC\n2qgngrUu5p4cA8wws8MyZeuinrXoZGWO6+2nHqr1V6/ejaCW8W8EzjWz3YE7gXO7od+uzPtkYDcz\n+zZwELDYzEaY2e9q7K8WmfOAmWa2E/Ag8J3Vjc1GAQuAI+rWPPjH2bHRCvRcmpqaGq1CjyTsmg9h\n1/wI2+ZD2LV+6nGwNpI0SdISSbdL6lssIGmMpMX+udzLekma4GVtks708mGS7pe0SNJjkrry+LEl\n8EpR2asZfU70MVslTfSyCZKulvSwpGckHevlm0ma6bq0STrKy4dKWurtlkmaLOlgb79M0l4ut6mk\n8ZmoxJGuxrvAX2uYy6uZ4/6S7pX0lKRrM+WrczwlneXRkMUZm27q7Vq9fLSXj3R9F7l+m2UHljTN\nI4GtklZK+mKNen8AvOZ99MpEaBZJOq1Y2O02x218m+t7qKTbMzIqbMp2AAAgAElEQVSrI2uSDimW\nr2K3cnzMnRiAmcBna2jzquswSNJst89iSfuuUVWX+FznSPqwF04o3FN+vsr/3g30AxZIOhe4AjjG\n++1Lx2s7VtKjXnedpEJdLXM9GpjoxxNJLyGyrCD9uwmCIAiCIAhyoB4HayfgGjPbBVgFnJqtlLQd\ncDnQBOwBjHQnZQ9gsJnt5hGEQtrZZOAnZrYH8K/A/3ZB/95Ae7bAzPZ2fXYBzgeazGxP4MyM2CAz\n2xc4kvSgC/A2cIyZ7QUcCFyZkR8G/NCjAjsBn/f25/gYABcAD5jZJ739jyRtYmaPmNm3XKcRkm4o\nNZGC3s5I4DRgZ+Cfsw/s3s9w4CSX2wc4WdLuwKeBF81sTzPbDfiNpI2AXwCnu60PAt4qGvsIMxsO\nfAV4Frgrq3c5zOwFMzvOT78KDCVFaPYgXd+szlsD3wU+5TZeAJxFcnj+j6RNXPR44BaXv6CEfFm7\nuaM4qISqTxYcZuBzwEcqzauo3y8Av3H77A4s8vLNgDk+14dI0amSXXl/RwN/N7PhZvYD4ELgF37+\ndmYO/+I2+Fcfsx0YW8dctzWzl11+BVCcDthO+ndTmVmZzzqVdLseE3bMjVgfkA9h13wIu+ZH2DYf\nwq5raGlpobm5efWnHB+qo8/nzGyuH08CTgeuytSPBGaZWSGiMRkYBVwC7CjpauDXwAylNTDbm9lU\nADN7tw498P5FeuCdVEbkQOAOM3vdx1iZqbvLy5ZqzXoUAZdJGkV6CN0+U7fczJb48ZMkpwDgcWAH\nPz4EOFLSOX7eBxgCLCsMamYLSI5INeaZ2Z98nrcC+wFTMvX7AXcWHswlTQH+DbiP5NhdBkwzs99J\n+gTwkpktdB3e8DYdBpS0DXAzcJyZ1b2+iOS4XWe+jVmRvQE+CewCPOzXbiOSg/KBpN+QbPcrUvra\nOSRHvZN8JQXMrFzq238AP5H0X8BUUnSuVuYD491RvdvM2rz8HTP7tR8vIM2/FJ12lilDIf3vU8Bw\nYL7Puy/wcifh8nMt12+BF0m2rcwBNfYeBEEQBEGwgdDU1NQhZXLcuHEl5epxsIof1EqtB+n0MGlm\nKz26cijwNWA08M1Ssh06kk4lRQUMONzfxhfqegF/BN4BptUxhwLvlNB5LLANsKeZtSttKNC3hHx7\n5rydNTYU8Fkze7oL+hRTi607NzJ72qNbhwMXS3qA5ExWs3Uv4Fag2cyWdkHfWhBpvdzYEnW3Ad8A\nXgfmm9mb7lyUk68LM/s96f5D0seoYw2SmT3kTvcRwE2SrjSzScB7GbEPWHMfvI9HhjOOYT0ImGhm\nF9TZrsDLkgaa2cse4SpOoZ0CXChpiUejg7VFrMHKjVgfkA9h13wIu+ZH2DYfwq71U0+K4FBJ2bSp\nh4rq5wGjJG0lqTcwBpjtqV69zexOUorYcI+iPC/paABJfTIpYgCY2bWe6jY861x5XbuZ7QA8Rkqn\nKsWDwGj5zmySBpSRKzgf/YFX3Lk6gJTuVixTifuAM1Y3kPaooU059lZa+9WLNL9iWz9EWr/T19dT\n/TvwkKdpvmVmtwA/IkVClgGDJI1wvfr59clyBdBmZneUUkZpDdfEUnUZ7gdOKfRdwt5zgX0lDfP6\nTd3ZAZjtup5MSmesJl8XmfVRvUj34PV+vr2kmVXaDiHdF+NJm2UUdkQsd088C+zlx0fT0cGqdB8V\n6h4AjsvoPMB1qJWpwJf8+CTg7qL6E4Hp4VwFQRAEQRDkQz0O1lPAaZKWkBbJX+/lhZSwFaQdzFqA\nVlIk4h5gMNAiqZWUgnaetzsROENSG/AwMLAL+v8eKLm1taf0XUpy8lpZs6aqXHRoMmndWBtwArC0\nhEyp9gUuJm0EslhpS/vvFQtUWoNVxDzgGlI64h/M7K7s2GbWCtxESl97BLjBU9d2Beb5fC8ELjGz\n90hO2jWSFgEzgI2LxjsbOERpk4uFkj5TVD8E+HsVnW8EngcW+/hjinT+M+nB/1a38RzSejbMrB24\nl7SG7N5q8pS5BhXWJY2RtAxYQlqjdpOXb0fHSFQpmoA2SQtJ67d+XEkH4OfA/m6DTwJvZuoqRSIL\ndlpKcgJn+LxnAJ3mVGGuVwAH+3w/RVoXmWUA0B1R1qBeYg1WbsT6gHwIu+ZD2DU/wrb5EHatH/mS\nmfUSX++0tZmdV1U46DKSrgBuNrMnGq1Ld6K00+GfzOzeRuuytpD0U9L28D+rIGM0rz2dNhiWk2+a\nYDOsz9/n/wgtLS2RwpIDYdd8CLvmR9g2H8Ku5ZGEmXXKUFrfHaxhpEjOG0W/hRUEQRGSZpPWDZ5g\nZi9WkFt/vxQ2YAYOHsiKF1ZUFwyCIAiCoFvokQ5WEATdjySL74UgCIIgCILKlHOw6lmDFQRBEHSR\nyGHPj7BtPoRd8yHsmh9h23wIu9ZPOFhBEARBEARBEATdRKQIBkHQgUgRDIIgCIIgqE6kCAZBEARB\nEARBEORMOFhBEARrgchhz4+wbT6EXfMh7JofYdt8CLvWTzhYQRAEQRAEQRAE3USswQqCoAPxO1hB\nEAQwcOBQVqx4ttFqBEGwDhO/gxUEQU0kByu+F4Ig2NAR8YwUBEElYpOLIAiChtLSaAV6MC2NVqCH\n0tJoBXoksZ4lP8K2+RB2rZ+aHCxJQyU9XqZulqTh3atWbUgaImmhpOmZsuWN0KUckvaXNKEGueUZ\n+XvKyUjaqhv1KjlOpr6i3n5fzKoi0+33R7bPWq63pIskveD3ykJJn66hTcV+Ja2qXePVbX4o6XFJ\nV0jaRtJcSQsk7VfLta1xrj+QtFTSIkm/krRFpu63kuZJ2rZe3YMgCIIgCILaqCeCtS7GyY8BZpjZ\nYZmydVHPWnSyMsf19lMP1fqrV+9GUOv4V5nZcP/8phv67cq8TwZ2M7NvAwcBi81shJn9rsb+apGZ\nAXzczPYAnga+s7qx2ShgAXBE3ZoH3UBToxXowTQ1WoEeSlOjFeiRNDU1NVqFHkvYNh/CrvVTj4O1\nkaRJkpZIul1S32IBSWMkLfbP5V7WS9IEL2uTdKaXD5N0v79pf0zSjl3Qf0vglaKyVzP6nOhjtkqa\n6GUTJF0t6WFJz0g61ss3kzTTdWmTdJSXD/WIwARJyyRNlnSwt18maS+X21TS+ExU4khX413grzXM\n5dXMcX9J90p6StK1mfLVOZ6SzvJoyOKMTTf1dq1ePtrLR7q+i1y/zbIDS5rmkZ1WSSslfbFGvT8A\nXvM+emUiNIsknVYs7Hab4za+zfU9VNLtGZnVkTVJhxTLV7FbJTrlx1bhVddhkKTZbp/FkvZdo6ou\n8bnOkfRhL5xQuKf8fJX/vRvoByyQdC5wBXCM99uXjtd2rKRHve46SYW6qnM1s5lm1u6nc4GPFIms\nIP27CYIgCIIgCPLAzKp+gKFAO/BJPx8PnOXHs4DhwHbAn4CtSI7bA8BRXjcj09cW/ncucJQf9wH6\n1qJLkV7jgG+WqdsFeAoY4Odb+t8JwG1+vDPwtB/3Bvr58daZ8qEkZ2MXP38MGO/HRwFT/PhS4At+\n3B9YBmxSpNMI4IYqc9of+LuPK1JE4livW+72HQ60AX2BzYAngN2BY4GfZfraHNgI+AMw3Mv6+fXZ\nH5haNPZwYBGweReuxdeA21mzcUrB3oX7Y2tgdsEmwLnAd93uz2bKrwXGlJPP9llCh2nAoBLlF7nt\nFgE3Av3rmNdZwHf8WMBmftwOHO7HVwDnZ+6vYzPt/1bm+CTgvzPnhWv7L8BUoLeX/xQ4oda5FslM\nLdyTmbL/Av6zSjuDizKfWQYWn3/4E3YM265vnw3drlgezJo1K5d+g7BtXoRd1zBr1iy76KKLVn/8\ne4Liz4eonefMbK4fTwJOB67K1I8EZplZIaIxGRgFXALsKOlq4NfADEn9gO3NbCpJs3fr0APvXySn\nYlIZkQOBO8zsdR9jZabuLi9bmlmPIuAySaNID8/bZ+qWm9kSP34SmOnHjwM7+PEhwJGSzvHzPsAQ\nkqOFj7cA+GoN05tnZn/yed4K7AdMydTvB9xpZm+7zBTg34D7gB9JugyYZma/k/QJ4CUzW+g6vOFt\nOgwoaRvgZuA4M6t7fREp5e06MzMfZ2VR/SdJTu/Dfu02AuaY2QeSfkOy3a9I6WvnkHJTOslXUsDM\nyqW+XQt8z8xM0iWk+/YrNc5rPjBe0kbA3WbW5uXvmNmv/XgBaf6lqDVyZv73UySHdL7Puy/wcifh\n8nNNg0oXAO+Z2S1FVS9SU95Pc3WRIAiCIAiCDYimpqYOKZPjxo0rKVePg2VVzqHEw6SZrZS0O3Ao\nKcoxGvhmKdkOHUmnktasGClSsCJT1wv4I/AO6U1+vbxTQuexwDbAnmbWrrShQN8S8u2Z83bW2FDA\nZ83s6S7oU0wttu7cyOxppc0fDgculvQAyZmsZutewK1As5kt7YK+tSBSJHNsibrbgG8ArwPzzexN\ndy7KydeFmWVT634OlN3co0Tbh9zpPgK4SdKVZjYJeC8j9gFr7oP38dTbjGNYDwImmtkFdbZb04H0\nJdI9cGCJ6inAhZKWmNkuXR0j6ApNjVagB9PUaAV6KE2NVqBHEutZ8iNsmw9h1/qpZw3WUEl7+/EX\ngIeK6ucBoyRtJak3Kc1rtqStSelOd5JSwoZ7FOV5SUcDSOojaZNsZ2Z2rZntaWlTghVFde1mtgMp\nXe/4Mvo+CIyW78wmaUAZuYLz0R94xZ2rA0gpesUylbgPOGN1A2mPGtqUY2+ltV+9SPMrtvVDpPU7\nfX091b8DD0naDnjLoxY/IkVClgGDJI1wvfr59clyBdBmZneUUsbXcE2sovP9wCmFvkvYey6wr6Rh\nXr+ppI953WzX9WTgFzXI14WkQZnTY0kplUjaXtLM0q1Wtx1Cui/Gk9ILCzsilrsnngX28uOj6ehg\nVbqPCnUPAMdl1nQNcB1qQmmHxHNI6bfvlBA5EZgezlUQBEEQBEE+1ONgPQWcJmkJaZH89V5eSAlb\nAZxH+uGMVlIk4h5gMNAiqZWUgnaetzsROENSG/AwMLAL+v+etG6lE57SdynJyWsFrszqmxX1v5OB\nka7PCcDSEjKl2he4mLQRyGKlLe2/VywgaYSkGyrMp8A84BpSOuIfzOyu7Nhm1grcREpfe4S0rqsN\n2BWY5/O9ELjEzN4jOWnXSFpEWtO1cdF4ZwOHKG1ysVDSZ4rqh5DWhVXiRuB5YLGPP6ZI5z8DXwJu\ndRvPAXbyunbgXuDT/reiPGWugdJmHYNKVP3Ar8si0tqzb3n5dnSMRJWiCWiTtBD4HPDjSjqQImT7\nuw0+CbyZqasUiSzYaSnpRcQMn/cMoNOcKsz1J6R1dvf7tby2qH4AaXfBYK3T0mgFejAtjVagh9LS\naAV6JPGbQvkRts2HsGv9FDYkWC/x9U5bm9l5VYWDLiPpCuBmM3ui0bp0J0o7Hf7JzO5ttC5rC0k/\nJW0P/7MKMlZjVmpQFy1EylVetBC2zYMWNmy7ijyekVpaWiLlKifCtvkQdi2PJMysU4bS+u5gDSNF\nct6wjr+FFQRBEZJmk9YNnmBmL1aQW3+/FIIgCLqJgQOHsmLFs41WIwiCdZge6WAFQdD9SLL4XgiC\nIAiCIKhMOQernjVYQRAEQReJHPb8CNvmQ9g1H8Ku+RG2zYewa/2EgxUEQRAEQRAEQdBNRIpgEAQd\niBTBIAiCIAiC6kSKYBAEQRAEQRAEQc6EgxUEQbAWiBz2/Ajb5kPYNR/CrvkRts2HsGv9hIMVBEEQ\nBEEQBEHQTcQarCAIOhC/gxUEQbDuMXDwQFa8sKLRagRBkCF+BysIgpqQZDQ3WosgCIKgA80Qz2xB\nsG4Rm1wEQRA0kuWNVqAHE7bNh7BrPoRdcyPWCuVD2LV+anKwJA2V9HiZulmShnevWrUhaYikhZKm\nZ8rWqa8uSftLmlCD3PKM/D3lZCRt1Y16lRwnU19Rb78vZlWR6fb7I9tnLddb0kWSXvB7ZaGkT9fQ\npmK/klbVrvHqNj+U9LikKyRtI2mupAWS9qvl2tY41wGSZkhaJuk+Sf0zdb+VNE/StvXqHgRBEARB\nENRGPRGsdTEufQwww8wOy5Sti3rWopOVOa63n3qo1l+9ejeCWse/ysyG++c33dBvV+Z9MrCbmX0b\nOAhYbGYjzOx3NfZXi8x5wEwz2wl4EPjO6sZmo4AFwBF1ax784+zYaAV6MGHbfAi75kPYNTeampoa\nrUKPJOxaP/U4WBtJmiRpiaTbJfUtFpA0RtJi/1zuZb0kTfCyNklnevkwSfdLWiTpMUld+crZEnil\nqOzVjD4n+pitkiZ62QRJV0t6WNIzko718s0kzXRd2iQd5eVDJS31dsskTZZ0sLdfJmkvl9tU0vhM\nVOJIV+Nd4K81zOXVzHF/SfdKekrStZny1Tmeks7yaMjijE039XatXj7ay0e6votcv82yA0ua5pGd\nVkkrJX2xRr0/AF7zPnplIjSLJJ1WLOx2m+M2vs31PVTS7RmZ1ZE1SYcUy1exWyU65cdW4VXXYZCk\n2W6fxZL2XaOqLvG5zpH0YS+cULin/HyV/70b6AcskHQucAVwjPfbl47XdqykR73uOkmFulrmejQw\n0Y8nkl5CZFlB+ncTBEEQBEEQ5MCH6pDdCfiymc2VNB44FbiqUClpO+ByYE9gJXC/OykvAIPNbDeX\n28KbTAa+b2ZTJfWha+vBegPt2QIz29vH2QU4H9jHzF6XlH2oHGRm+0raGZgKTAHeBo4xszckbQ3M\n9TqAYcBnzWyJpMeAz3v7o3yMY4ELgAfM7CueljVP0kwzewR4xHUaAZxiZl8tnkhBb2cksDPwHHCf\npGPNbEqhUik97iSX6w08KqnF9XzRzD7jcptL2gj4BTDazBZK6ge8VTT2EZl+/x9wl5mtKuhdDjN7\nATjOT78KDCVFaKzI3rhNvwt8yszecifjLOAy4GeSNjGzt4DjgVtc/oIS8peUs5ukacBXzKzUNkvf\ncMfxMeBsM6voPGb6/QLwGzO7zB2dgpO3GTDHzL4r6QpSdOr7pbry/o6W9DczK6Q2vgyMMLMz/Lww\nh39xG/yrmX0g6afAWGBSjXPd1sxe9jFXqHM6YDvpnqlMNvFzB+KNa3ewnLBjXoRt8yHsmg9h19xo\naWmJaEsOhF3X0NLSUtOatHocrOfMbK4fTwJOJ+NgkR72Z5lZIaIxGRhFeiDeUdLVwK+BGf6Qv72Z\nTQUws3fr0APvX8DurkspDgTuMLPXfYyVmbq7vGxp5gFUwGWSRpEeQrfP1C03syV+/CQw048fJz1+\nAhwCHCnpHD/vAwwBlhUGNbMFJEekGvPM7E8+z1uB/UhOYIH9gDvN7G2XmQL8G3Af8CNJlwHTzOx3\nkj4BvGRmC12HN7xNhwElbQPcDBznzlW9HARcZ77FUZG9AT4J7AI87NduI5KD8oGk35Bs9ytS+to5\nQFMp+UoKFBzFElwLfM8dv0tI9+1XapzXfGC8O6p3m1mbl79jZr/24wWk+Zei1shZIf3vU8BwYL7P\nuy/wcifh8nMt12+BF0m2rcwBNfYeBEEQBEGwgdDU1NTB2Rw3blxJuXocrOIHtVLrQTo9TJrZSkm7\nA4cCXwNGA98sJduhI+lUUlTAgMOzb+ol9QL+CLwDTKtjDgXeKaHzWGAbYE8za1faUKBvCfn2zHk7\na2woUpTr6S7oU0wttu7cyOxpj0IdDlws6QGSM1nN1r2AW4FmM1vaBX1rQaT1cmNL1N0GfAN4HZhv\nZm+6c1FOvi7MLJta93Og7OYeJdo+5E73EcBNkq40s0nAexmxD1hzH7yPR2MzjmE9CJhoZhfU2a7A\ny5IGmtnLkgbROYV2CnChpCVmtksXxwi6Qryxzo+wbT6EXfMh7JobEWXJh7Br/dSTljdUUjZt6qGi\n+nnAKElbSeoNjAFme6pXbzO7k5QiNtyjKM9LOhpAUh9Jm2Q7M7NrzWxP35RgRVFdu5ntQEr3Or6M\nvg8Co+U7s0kaUEau4Hz0B15x5+oAUrpbsUwl7gPOWN1A2qOGNuXYW2ntVy/S/Ipt/RBp/U5fpfVU\n/w485Gmab5nZLcCPSJGQZcAgT09EUj+/PlmuANrM7I5Syiit4ZpYqi7D/cAphb5L2HsusK+kYV6/\nqaSPed1s1/VkUjpjNfm6cEejwLHAE16+vaSZpVutbjuEdF+MB250PaH8PfEssJcfH01HB6vSfVSo\newA4TmvWdA1wHWplKvAlPz4JuLuo/kRgejhXQRAEQRAE+VCPg/UUcJqkJaRF8td7eSElbAVpB7MW\noJUUibgHGAy0SGolpaCd5+1OBM6Q1AY8DAzsgv6/B0pube0pfZeSnLxW4MqsvllR/zsZGOn6nAAs\nLSFTqn2Bi0kbgSxW2tL+e8UCkkZIuqHCfArMA64hpSP+wczuyo5tZq3ATaT0tUeAGzx1bVfS2q9W\n4ELgEjN7j+SkXSNpETAD2LhovLOBQ5Q2uVgo6TNF9UOAv1fR+UbgeWCxjz+mSOc/kx78b3UbzyGt\n68PM2oF7gU/734rylLkGSpt1DCpR9QO/LouA/YFvefl2dIxElaIJaJO0EPgc8ONKOpAiZPu7DT4J\nvJmpqxSJLNhpKelFxAyf9wyg05wqzPUK4GBJy0jphpcX1Q8AuiPKGtTLOvUDEj2MsG0+hF3zIeya\nG/F7TfkQdq0frc+/Cu7rnbY2s/OqCgddxjdxuNnMnmi0Lt2J0k6HfzKzexuty9rCN81YbGY/qyBj\nNK89nTYYYmF7foRt8yHsmg9dtWszrM/PbGuD2IwhH8Ku5ZGEmXXKUFrfHaxhpEjOG0W/hRUEQRGS\nZpPWDZ5gZi9WkFt/vxSCIAh6KAMHD2TFC6U2yQ2CoFH0SAcrCILuR5LF90IQBEEQBEFlyjlYXfnt\nqSAIgqBOIoc9P8K2+RB2zYewa36EbfMh7Fo/4WAFQRAEQRAEQRB0E5EiGARBByJFMAiCIAiCoDqR\nIhgE/5+9O4+zo6rzPv75JoBsQ4CoCSAJTEZ5RIUQQFEwacSdVQw7A8OAy0tHVFyGEZUEQUAFjSAq\nYyagiQjMRHYkLOkMBENCCAmQEEHC6hPgUeIERxHTv+ePc25SffuuTVe603zfr9d9dd1Tp6p+9atK\n554+59Q1MzMzMyuZG1hmZuuBx7CXx7kth/NaDue1PM5tOZzX9rmBZWZmZmZm1kc8B8vMuvH3YJmZ\nlW/EiNGsXPl4f4dhZq+AvwfLzFqSGlj+vWBmVi7hz2BmGzY/5MLMrF919ncAg1hnfwcwSHX2dwCD\nVGd/BzBoea5QOZzX9rXUwJI0WtIDddbNljSub8NqjaRRku6TdHOhbEV/xFKPpAmSprVQb0Wh/vX1\n6kjatg/jqnmcwvqGcef7YnaTOn1+fxT32cr1ljRR0oOS1rQaS7P9SlrdWrTdtvm2pAcknS/ptZLm\nSVooab9Wrm2L5/otScsk3S/pvyRtVVj335LmS3p9u7GbmZmZWWva6cEaiP3YhwGzIuJDhbKBGGcr\nMUWd5Xb3045m+2s37v7QyvEfAD4CzOnD/fbmvD8G7BYR/wq8F1gSEXtGxF0t7q+VOrOAt0TEWOAR\n4N/WbhwxHlgIHNh25NYHOvo7gEGso78DGKQ6+juAQaqjvwMYtDo6Ovo7hEHJeW1fOw2sjSVNl7RU\n0lWSNq2uIOkYSUvy67xcNkTStFy2WNJnc/kYSbfmv7TfK2nnXsS/NfBcVdnzhXhOyMdcJOnyXDZN\n0hRJcyU9KunwXL6FpNtyLIslHZLLR+cegWmSlkuaIel9efvlkvbK9TaXNLXQK3FwDuOvwB9bOJfn\nC8vDJN0g6WFJlxTK147xlHRa7g1ZUsjp5nm7Rbn8iFy+d473/hzfFsUDS7ox9wQukrRK0j+2GPca\n4A95H0MKPTT3S/p0deWct7tzjq/M8X5A0lWFOmt71iS9v7p+k7zVFBHLI+KRYv5a8HyOYaSkOTk/\nSyTtuy5UnZ3P9W5Jr8uF0yr3VH6/Ov+8FtgSWCjpy8D5wGF5v5vS/doeJ+mevO6HkirrWjnX2yKi\nK7+dB7yhqspK0r8bMzMzMyvBRm3U3QU4KSLmSZoKfAq4sLJS0nbAecAewCrg1txIeRrYISJ2y/Uq\nQ5ZmAN+MiOskbULv5oMNBbqKBRHxjnycXYGvAO+MiBckFT9UjoyIfSW9GbgOmAn8BTgsIl6UNJz0\n4fS6XH8M8NGIWCrpXuDovP0h+RiHA2cAt0fEyZKGAfMl3RYRvwZ+nWPaE/hERHy8+kQqcWd7A28G\nngRukXR4RMysrFQa5nZirjcUuEdSZ47zmYg4KNf7O0kbA78AjoiI+yRtCfy56tgHFvb7H8A1EbG6\nEnc9EfE0MDG//TgwmtRDE1X5Juf0q8ABEfHn3Mg4DTgX+LGkzSLiz8BRwM9z/TNq1D+7Xt4k3Qic\nHBErG8XdisJ+jwV+FRHn5oZOpZG3BXB3RHxV0vmk3qlv1tpV3t+hkv4nIipDG58F9oyIU/P7yjn8\nn5yDd0XEGkk/AI4DpvfiXP+ZdO2Lukj3TBOTCssd+C+ufaET57EsnTi3ZejEeS1DJ85rOTo7O93b\nUgLndZ3Ozs6W5qS108B6MiLm5eXpwGcoNLBIH/ZnR0SlR2MGMJ70gXhnSVOAm4BZ+UP+9hFxHUBE\n/LWNOMj7F7B7jqWW9wBXR8QL+RirCuuuyWXLtG4+ioBzJY0nfQjdvrBuRUQszcsPAbfl5QeAnfLy\n+4GDJX0pv98EGAUsrxw0IhaSGiLNzI+IJ/J5XgHsR2oEVuwH/DIi/pLrzATeDdwCfEfSucCNEXGX\npLcCv4uI+3IML+Ztuh1Q0muBnwETc+OqXe8Ffhj5kUhV+QbYB9gVmJuv3cakBsoaSb8i5e6/SMPX\nvkT636dH/UYBVBqKfWwBMDU3VK+NiMW5/KWIuCkvLySdfy2t9ppVhv8dAIwDFuTz3hR4tkflJucq\n6Qzg5Yj4edWqZ2jpf/ZJzauYmZmZvYp0dHR0a2xOnjy5ZuJYhU4AACAASURBVL12GljV8z9qzQfp\n8WEyIlZJ2h34APBJ4Ajgc7XqdtuR9ClSr0AAHy7+pV7SEOAx4CXgxjbOoeKlGjEfB7wW2CMiupQe\nKLBpjfpdhfddrMuhSL1cj/Qinmqt5LrnRhGP5F6oDwPfkHQ7qTHZLNdDgCuASRGxrBfxtkKk+XLH\n1Vh3JfAvwAvAgoj4U25c1Ku/3kTEnbnRfSBwmaQLImI68HKh2hrW3Qd/I/fGFhqG7RBweUSc0duY\nJf0T6R54T43VM4GvS1oaEbv29hjWGx39HcAg1tHfAQxSHf0dwCDV0d8BDFruZSmH89q+dobljZZU\nHDZ1Z9X6+cB4SdtKGgocA8zJQ72GRsQvSUPExuVelKckHQogaRNJmxV3FhGXRMQeETGuehhURHRF\nxE7AvaThVLXcARyh/GQ2SdvUqVdpfAwDnsuNq/1Jw92q6zRyC3Dq2g2ksS1sU887lOZ+DSGdX3Wu\n7yTN39lUaT7VR4A78zDNP+dei++QekKWAyPz8EQkbZmvT9H5wOKIuLpWMEpzuC5vEvOtwCcq+66R\n73nAvpLG5PWbS3pjXjcnx/ox1g1pa1T/lSjOddpe0m0NK0ujSPfFVOAnOc5u+6nyOLBXXj6U7g2s\nRvdRZd3twMTCnK5tcgwtkfRBUg/gIRHxUo0qJwA3u3FlZmZmVo52GlgPA5+WtJQ0Sf5HubwyJGwl\ncDppcPEiUk/E9cAOQKekRaQhaKfn7U4ATpW0GJgLjOhF/L8Baj7aOg/pO4fUyFsEXFCMt1g1/5wB\n7J3jOR5YVqNOre0rvkF6EMgSpUfan1VdQdKeki5tcD4V84GLScMRfxsR1xSPHRGLgMtIw9d+DVya\nh669jTT3axHwdeDsiHiZ1Ei7WNL9pKfMvabqeF8A3q/0kIv7JB1UtX4U8L9NYv4J8BSwJB//mKqY\n/x/wT8AVOcd3k+b1kR/KcAPwwfyzYX3qXAOlh3WMrFF+mKSnSMMUb9C6x/pvR/eeqFo6gMWS7gOO\nBL7XKAbg34EJOQf7AH8qrGvUE1nJ0zLSHyJm5fOeBdQ6p5rnClxEepjGrflaXlK1fhvS0wVtvevs\n7wAGsc7+DmCQ6uzvAAapzv4OYNDy9zWVw3ltnzbkbxHP852GR8TpTStbr+WHOPwsIh7s71j6ktKT\nDp+IiBv6O5b1JT80Y0lE/LhBnej/p+8PRp14aFBZOnFuy9CJ81qGTlJexYb8GWwg8sMYyuG81ieJ\niOgxQmlDb2CNIfXkvFj1XVhmVkXSHNK8weMj4pkG9dzAMjMrnRtYZhu6QdnAMrO+lxpYZmZWphEj\nRrNy5eP9HYaZvQL1Gli9+e4pMxvkIsKvPn7Nnj2732MYrC/n1nndkF6VvLpx1fc8V6gczmv73MAy\nMzMzMzPrIx4iaGbdSAr/XjAzMzNrzEMEzczMzMzMSuYGlpnZeuAx7OVxbsvhvJbDeS2Pc1sO57V9\nbmCZmZmZmZn1Ec/BMrNuPAfLzMzMrLl6c7A26o9gzGxgk3r8rjAzs0FixA4jWPn0yv4Ow2zQcg+W\nmXUjKZjU31EMQiuAnfs7iEHKuS2H81qOgZDXSen7Dgebzs5OOjo6+juMQcd5rc9PETQzMzMzMytZ\nSw0sSaMlPVBn3WxJ4/o2rNZIGiXpPkk3F8pW9Ecs9UiaIGlaC/VWFOpfX6+OpG37MK6axymsbxh3\nvi9mN6nT5/dHcZ+tXG9JEyU9KGlNq7E026+k1a1F222bb0t6QNL5kl4raZ6khZL2a+Xatniu20ia\nJWm5pFskDSus+29J8yW9vt3YrQ/091+sBzPnthzOazmc19K4l6Uczmv72unBGoh9yYcBsyLiQ4Wy\ngRhnKzFFneV299OOZvtrN+7+0MrxHwA+Aszpw/325rw/BuwWEf8KvBdYEhF7RsRdLe6vlTqnA7dF\nxC7AHcC/rd04YjywEDiw7cjNzMzMrCXtNLA2ljRd0lJJV0natLqCpGMkLcmv83LZEEnTctliSZ/N\n5WMk3Srpfkn3SurN33S2Bp6rKnu+EM8J+ZiLJF2ey6ZJmiJprqRHJR2ey7eQdFuOZbGkQ3L5aEnL\n8nbLJc2Q9L68/XJJe+V6m0uaWuiVODiH8Vfgjy2cy/OF5WGSbpD0sKRLCuVrx3hKOi33hiwp5HTz\nvN2iXH5ELt87x3t/jm+L4oEl3Zh7AhdJWiXpH1uMew3wh7yPIYUemvslfbq6cs7b3TnHV+Z4PyDp\nqkKdtT1rkt5fXb9J3mqKiOUR8Ugxfy14PscwUtKcnJ8lkvZdF6rOzud6t6TX5cJplXsqv1+df14L\nbAkslPRl4HzgsLzfTel+bY+TdE9e90Np7RMnmp4rcChweV6+nPRHiKKVpH83tr4NqL71Qca5LYfz\nWg7ntTT+vqZyOK/ta+cpgrsAJ0XEPElTgU8BF1ZWStoOOA/YA1gF3JobKU8DO0TEbrneVnmTGcA3\nI+I6SZvQu/lgQ4GuYkFEvCMfZ1fgK8A7I+IFScUPlSMjYl9JbwauA2YCfwEOi4gXJQ0H5uV1AGOA\nj0bEUkn3Akfn7Q/JxzgcOAO4PSJOzsOy5ku6LSJ+Dfw6x7Qn8ImI+Hj1iVTizvYG3gw8Cdwi6fCI\nmFlZqTTM7cRcbyhwj6TOHOczEXFQrvd3kjYGfgEcERH3SdoS+HPVsQ8s7Pc/gGsiYnUl7noi4mlg\nYn77cWA0qYcmqvJNzulXgQMi4s+5kXEacC7wY0mbRcSfgaOAn+f6Z9Sof3a9vEm6ETg5Il7xo5EK\n+z0W+FVEnJsbOpVG3hbA3RHxVUnnk3qnvllrV3l/h0r6n4ioDG18FtgzIk7N7yvn8H9yDt4VEWsk\n/QA4Dpje4rm+PiKezcdcqZ7DAbtI90xjxYGfO+EhLWZmZvaq19nZ2VKDs50G1pMRMS8vTwc+Q6GB\nRfqwPzsiKj0aM4DxpA/EO0uaAtwEzMof8rePiOsAIuKvbcRB3r+A3XMstbwHuDoiXsjHWFVYd00u\nW1b4ACrgXEnjSR9Cty+sWxERS/PyQ8BtefkB0sdPgPcDB0v6Un6/CTAKWF45aEQsJDVEmpkfEU/k\n87wC2I/UCKzYD/hlRPwl15kJvBu4BfiOpHOBGyPiLklvBX4XEfflGF7M23Q7oKTXAj8DJubGVbve\nC/yw8gVKVfkG2AfYFZibr93GpAbKGkm/IuXuv0jD174EdNSq3yiASkOxjy0ApuaG6rURsTiXvxQR\nN+XlhaTzr6XVXrPK8L8DgHHAgnzemwLP9qjc+rlWDyt8hpTbxvZvce/WOjdSy+PclsN5LYfzWhrP\nFSqH87pOR0dHt3xMnjy5Zr12GljVH9RqzQfp8WEyIlZJ2h34APBJ4Ajgc7XqdtuR9ClSr0AAHy7+\npV7SEOAx4CXgxjbOoeKlGjEfB7wW2CMiupQeKLBpjfpdhfddrMuhSL1cj/Qinmqt5LrnRhGP5F6o\nDwPfkHQ7qTHZLNdDgCuASRGxrBfxtkKk+XLH1Vh3JfAvwAvAgoj4U25c1Ku/3kTEnbnRfSBwmaQL\nImI68HKh2hrW3Qd/I/fGFhqG7RBweUSc0cuQn5U0IiKelTSSnkNoZwJfl7Q0Inbt5THMzMzMrI52\nhuWNllQcNnVn1fr5wHhJ20oaChwDzMlDvYZGxC9JQ8TG5V6UpyQdCiBpE0mbFXcWEZdExB4RMa56\nGFREdEXETsC9pOFUtdwBHKH8ZDZJ29SpV2l8DAOey42r/UnD3arrNHILcOraDaSxLWxTzzuU5n4N\nIZ1fda7vJM3f2VRpPtVHgDvzMM0/R8TPge+QekKWAyPz8EQkbZmvT9H5wOKIuLpWMEpzuC6vta7g\nVuATlX3XyPc8YF9JY/L6zSW9Ma+bk2P9GGk4Y7P6r0RxrtP2km5rWFkaRbovpgI/yXF220+Vx4G9\n8vKhdG9gNbqPKutuByYW5nRtk2No1XXAP+XlE4Frq9afANzsxlU/8LyL8ji35XBey+G8lsZzhcrh\nvLavnQbWw8CnJS0lTZL/US6vDAlbSXqCWSewiNQTcT2wA9ApaRFpCNrpebsTgFMlLQbmAiN6Ef9v\ngJqPts5D+s4hNfIWARcU4y1WzT9nAHvneI4HltWoU2v7im+QHgSyROmR9mdVV5C0p6RLG5xPxXzg\nYtJwxN9GxDXFY0fEIuAy0vC1XwOX5qFrbyPN/VoEfB04OyJeJjXSLpZ0PzALeE3V8b4AvF/pIRf3\nSTqoav0o4H+bxPwT4ClgST7+MVUx/z/SB/8rco7vJs3rIyK6gBuAD+afDetT5xooPaxjZI3ywyQ9\nRRqmeIPWPdZ/O7r3RNXSASyWdB9wJPC9RjEA/w5MyDnYB/hTYV2jnshKnpaR/hAxK5/3LKDWOdU8\nV1Jj+X2SlpOGG55XtX4boC96Wc3MzMysBm3I3+Sd5zsNj4jTm1a2XssPcfhZRDzY37H0JaUnHT4R\nETf0dyzrS35oxpKI+HGDOsGk9ReTmZmtZ5NgQ/78ZzZQSCIieoxQ2tAbWGNIPTkvVn0XlplVkTSH\nNG/w+Ih4pkG9DfeXgpmZNTVihxGsfPoVP3DX7FVvUDawzKzvSQr/Xuh7nZ2dfhJTSZzbcjiv5XBe\ny+PclsN5ra9eA6s33z1lZmZmZmZmNbgHy8y6cQ+WmZmZWXPuwTIzMzMzMyuZG1hmZuuBv0ekPM5t\nOZzXcjiv5XFuy+G8ts8NLDMzMzMzsz7iOVhm1o3nYJmZmZk1V28O1kb9EYyZDWxSj98V9gqNGDGa\nlSsf7+8wzMzMrGQeImhmNYRfffx69tkn2rsE1jLPDyiH81oO57U8zm05nNf2NWxgSRot6YE662ZL\nGldOWI1JGiXpPkk3F8pW9Ecs9UiaIGlaC/UGVNxFrcTWrI6kMyWd1ndRdd+npGmSxjepv7WkmZIW\nS5onadcWjjFb0qgm69u6/yVNlLRU0u35/RWS7pf02XwehzfZvpVzPTaf52JJd0narbDuAkkPSZrQ\nTtxmZmZm1rpWerAG4mSMw4BZEfGhQtlAjLOVmAZi3BUbevwVXwEWRcTuwInA9/spjpOBUyLiAEkj\ngb0iYmxETOnDYzwGjM/nejZwaWVFRHwBOAv45z48nlm/6+jo6O8QBiXntRzOa3mc23I4r+1rpYG1\nsaTp+S/vV0natLqCpGMkLcmv83LZkPwX9yX5r+mfzeVjJN2a/3J/r6SdexH31sBzVWXPF+I5IR9z\nkaTLc9k0SVMkzZX0aKW3QNIWkm7LsSyWdEguHy1pWd5uuaQZkt6Xt18uaa9cb3NJU3PPyEJJB+cw\n/gr8sYVzeT7vZ6SkOblnbomkfXP5akln53zdLel1ufygwjFnFcrPlPTTXHe5pFNy+YS8/xskPSzp\nEiUnSfpuIXenSLqgOqfN4q+X9yJJfy/pZkkLcixvkrSVpMcLdTaX9KSkobXq1zj+KlKuG9kVuAMg\nIpYDO1Xy1cDvgTX17uPsSEn35HxWrteJki4qnM/1ksZL+hqwHzBV0reAW4Ad8vXerypP4yR15vO+\nWdKIVs81IuZFROW+mwfsUFVlJenfj5mZmZmVISLqvoDRQBewT34/FTgtL88GxgHbAU8A25IabLcD\nh+R1swr72ir/nAcckpc3ATZtFEOduCYDn6uzblfgYWCb/H7r/HMacGVefjPwSF4eCmyZl4cXykeT\nPszumt/fC0zNy4cAM/PyOcCxeXkYsBzYrCqmPYFLm5zTacC/5WUBW+TlLuDDefl84CuVYxW2PRn4\ndl4+E1iUczsceBIYCUwA/jefl4BZwOHAFsCjwNC8/VzgLb24JvXyfmbhnrkNGJOX3w7cnpd/CUzI\ny0dWctWg/tp91rgvDqpRfg5wQWE/fwX2aPG86t3Hsws5/xBwa14+Efh+of71pB6lyjZ7FO6vJYV6\n0/L12Chfg+GFfExt9Vyr6nyx+r4D3g3c0GS7gPCrz1+ElWP27Nn9HcKg5LyWw3ktj3NbDue1vvx/\ne4/PUq08RfDJiJiXl6cDnwEuLKzfG5gdEX8AkDQDGE8anrSzpCnATcAsSVsC20fEdaSImvU89CBJ\nwO45llreA1wdES/kY6wqrLsmly2T9PrKLoFzlea2dAHbF9atiIilefkh0gd+gAeAnfLy+4GDJX0p\nv98EGEVqaJGPtxD4eJNTW0Dq3dgYuDYiFufylyLipry8EHhvXt5R0lWkBu7GwIrCvq7Nuf29pDtI\njYo/AvMj4glI83+A/SJiptKcoIMkPQxsFBEPNYm1lkZ5R9IWwLuAq/M1JMcNcBVwFDAHOBr4QZP6\nNUXEmXVWnQdMkXQf6dotAta0eF6PUXUfF9bNzD8XkhpMrWj2eL5dgLcCt+bzHgL8rrpSg3NNB5H2\nB04i9ZoVPQO8SdJrIuKl+nuYVFjuyC8zMzOzV6/Ozs6WHvrRSgMrmryHGh8aI2KVpN2BDwCfBI4A\nPlerbrcdSZ8CPpaP8+GIWFlYN4T0gfcl4MYWYq9W/EBZieM44LWknoUupYc2bFqjflfhfRfrcifg\noxHxSC/iWSsi7syNvAOByyRdEBHTgZcL1dYUjnsR8J2IuFHpoQXFD9zFayRqX7NivamkeUoPk3pS\nyjAEeCEiaj0Y4jrgHEnbkHqM7gC2bFC/LRGxmsK8o3yNH2tx21r38Sl5deV+KF6Xv9F96G2PIbVN\nCHgwIvZtc7t1O0gPtrgU+GClwVsREY9JWgY8IemA+o3pSb09vNl65/kB5XBey+G8lse5LYfzuk5H\nR0e3fEyePLlmvVbmYI2W9I68fCxwZ9X6+cB4SdtKGgocA8yRNJw07OyXwFeBcRHxIvCUpEMBJG0i\nabPiziLikojYIyLGFRtXeV1XROxEGq53VJ147wCOkLRtPsY2depVGljDgOdy42p/uvdEtPJlQLcA\np67dQBrbwjY9g0lPrHsuIqYCPyE1NBrFsBXrejZOrFp3aM7tcNLQwAW5fG+luWVDSPm7CyAi5gM7\nkq7dFXXiW9bkFBrmPTdyVkiaWNjnbnndn0jXdApp+Fo0qt8uScNyzyCSPgbMyfciSvPvtmuwbY/7\nuF7V/PNxYGye37Yjqfew7u5rlC0HXidpn3z8jdTCUw8L8Y4C/gv4x4j4bY31uwE7k3qSe9NTaWZm\nZmYNtNLAehj4tKSlpMnxP8rlabJGagSdDnSShl4tiIjrSZPrOyUtAn6W6wCcAJwqaTFprkllAn87\nfkOa89VDHtJ3DqmRtwioPLChXk/cDFLDYzFwPLCsRp1a21d8g/QgkCVKj7Q/q7qCpD0lXdpz0246\ngMV5GNuRwPeaHHcy8J+SFtDzYRRLSNfjbuCsQkP1XuBi0nDH3+ZGQ8VVwNxY94CEYvzDm8TeKO9F\nxwMnKz2w40HSXLaKK0m9ib8olB3XoH4PkiZLOqjGqjcDD+ZG4geAygNXBIwB/tBgt/Xu45r3U0TM\nJTWyHiJdw4XVdeq8r2z/MjAROF/S/aR/U+9s41y/Rvq3cYnSw0bmV63fBng8IrpqbGu2QfJ3tJTD\neS2H81oe57Yczmv7lOZnbVjyfKfhEXF608qvMpLOBFZHxIVV5ROAL0REzUaKpOuBCyNido11BwI7\nR8TFZcTcXyS9BTgpIr7Y37GsL5KOBD4SEcc0qBP12/XWe2JD/H27Iejs7PQQlhI4r+VwXsvj3JbD\nea1PEhHRY0TShtrAGgNcBrwY3b8L61Wv3QaWpGGkYZ6LIuLo9ReprW9Kj99/N+lplbc3qOcGVinc\nwDIzMxtMBlUDy8zKkxpY1tdGjBjNypWP93cYZmZm1kfqNbBamYNlZq8ytb7Twa9X9vrFLy7r78s6\naHl+QDmc13I4r+VxbsvhvLbPDSwzMzMzM7M+4iGCZtaNpPDvBTMzM7PGPETQzMzMzMysZG5gmZmt\nBx7DXh7nthzOazmc1/I4t+VwXtvnBpaZmZmZmVkf8RwsM+vGc7DMzMzMmqs3B2uj/gjGzAY2qcfv\nChugRuwwgpVPr+zvMMzMzCxzD5aZdSMpmNTfUQxCK4CdS9jvpPS9Za9mnZ2ddHR09HcYg47zWg7n\ntTzObTmc1/p69RRBSaMlPVBn3WxJ4/oqwHZIGiXpPkk3F8pW9Ecs9UiaIGlaC/UGVNxFrcTWrI6k\nMyWd1ndRdd+npGmSxjepv7WkmZIWS5onadcWjjFb0qgm69u6/yVNlLRU0u35/RWS7pf02XwehzfZ\nvum55nrfl/RI3vfYQvkFkh6SNKGduM3MzMysda085GIg/mn0MGBWRHyoUDYQ42wlpoEYd8WGHn/F\nV4BFEbE7cCLw/X6K42TglIg4QNJIYK+IGBsRU/rqAJI+BIyJiDcCnwB+VFkXEV8AzgL+ua+OZ20o\no/fKAPyX1ZI4r+VwXsvj3JbDeW1fKw2sjSVNz395v0rSptUVJB0jaUl+nZfLhuS/uC/JPQefzeVj\nJN2a/7p+r6TefOzYGniuquz5Qjwn5GMuknR5LpsmaYqkuZIerfQWSNpC0m05lsWSDsnloyUty9st\nlzRD0vvy9ssl7ZXrbS5pau4ZWSjp4BzGX4E/tnAuz+f9jJQ0J/fMLZG0by5fLensnK+7Jb0ulx9U\nOOasQvmZkn6a6y6XdEoun5D3f4OkhyVdouQkSd8t5O4USRdU57RZ/PXyXiTp7yXdLGlBjuVNkraS\n9HihzuaSnpQ0tFb9GsdfRcp1I7sCdwBExHJgp0q+Gvg9sKbefZwdKemenM/K9TpR0kWF87le0nhJ\nXwP2A6ZK+hZwC7BDvt77VeVpnKTOfN43SxrRxrkeCvw0n+s9wLDC9gArSf9+zMzMzKwErTSwdgEu\njohdgdXAp4orJW0HnAd0AGOBvXMjZSywQ0TslnsOKsPlZgAXRcRY4F3A/+1F3EOBrmJBRLwjx7Mr\nqceiIyL2AIofiEdGxL7AwcD5uewvwGERsRfwHuCCQv0xwLcjYpech6Pz9l/KxwA4A7g9IvbJ239H\n0mYR8euI+HyOaU9Jl9Y6kUrcwLHAryJiHLA7cH8u3wK4O+frTuBjufzOiNgnIvYErgS+XNjt20jX\n413A15V6SwD2Bj4NvBn4B+AjwFXAwZKG5jonAf9RFVtdLea94lLgXyJib1IOfxgR/wMs0rphawfl\nPKypVb/G8T8fEfNyDJMlHVTjuIuBSoP67cAo4A1NzmtiRDxD/fsYYGg+/89Dt1lLPXr1IuIbwL3A\nsRHxZeAQ4NGIGBcRd1XqSdoIuAj4aD7vacA32zjXHYCnCu+fyWUVXaR/P7a+DdjBwBs+f0dLOZzX\ncjiv5XFuy+G8tq+Vpwg+WflQB0wHPgNcWFi/NzA7Iv4AIGkGMB44G9hZ0hTgJmCWpC2B7SPiOoCI\naPbX+B4kidQAmV6nynuAqyPihXyMVYV11+SyZZJeX9klcK7S3JYuYPvCuhURsTQvPwTclpcfAHbK\ny+8nNVC+lN9vQvoAv7xy0IhYCHy8yaktIPVubAxcGxGLc/lLEXFTXl4IvDcv7yjpKmA7YGO6f3y7\nNuf295LuAN5O6k2bHxFPQJr/A+wXETOV5gQdJOlhYKOIeKhJrLU0yjuStiA1+K7O15AcN6RG3lHA\nHOBo4AdN6tcUEWfWWXUeMEXSfaRrtwhY0+J5PUbVfVxYNzP/XAiMbnF/zR7PtwvwVuDWfN5DgN9V\nV2pwrs08A7xJ0msi4qW6tWYXlnfCw9vMzMzsVa+zs7OlBmcrDazqv8bXmnPT40NjRKyStDvwAeCT\nwBHA52rV7bYj6VOkXpoAPhwRKwvrhpA+8L4E3NhC7NWKHygrcRwHvBbYIyK6lB7asGmN+l2F912s\ny51IvQ2P9CKetSLiztzIOxC4TNIFETEdeLlQbU3huBcB34mIG3PvT/EDd/EaifrzpCrlU0m9Tw/T\nvYemLw0BXsg9dNWuA86RtA0wjjScb8sG9dsSEaspzDvK1/ixFretdR+fkldX7ofidfkb3XuGewyp\nbULAg7mntDeeAXYsvH9DLgMgIh6TtAx4QtIBdRvT+/fy6FafG6ml8fyAcjiv5XBey+PclsN5Xaej\no6NbPiZPnlyzXitDBEdLKg5ju7Nq/XxgvKRt8zCzY4A5koaThlD9EvgqMC4iXgSeknQogKRNJG1W\n3FlEXBIRe+ShUyur1nVFxE6koVZH1Yn3DuAISdvmY2xTp16lgTUMeC43rvane09EK18GdAtw6toN\nCk9ta4fSE+uei4ipwE9IDY1GMWzFup6NE6vWHZpzOxyYQOodgzR8c3RuqB4F3AUQEfNJH8qPAa6o\nE9+yJqfQMO+5kbNC0sTCPnfL6/5EuqZTgBsiqVu/XZKG5Z5BJH0MmJPvRZTm323XYNse93G9qvnn\n48BYJTuSeg/r7r5G2XLgdZL2ycffSC089bDgOuCEvO0+wKqIeLZwPruRPupv38ueSjMzMzNroJUG\n1sPApyUtJU2OrzyVLAByI+h0oJM09GpBRFxPmvfRKWkR8LNcB9KHv1MlLQbmAsUJ+K36DbBtrRV5\nSN85pEbeItbNqarXEzeD1PBYDBwPLKtRp9b2Fd8gPQhkidIj7c+qrtBoDlZBB7A4D2M7Evhek+NO\nBv5T0gJ6PoxiCel63A2cVWio3gtcTBru+NvcaKi4CpgbET0ezJEbGQ01yHvR8cDJSg/seJA0D6ni\nSlJv4i8KZcc1qN9Dg3lJbwYezI3ED5Dnh+UheGOAPzTYbb37uOb9FBFzSY2sh0jXcGF1nTrvK9u/\nDEwEzpd0P+nf1DtbPdc8nHSFpEeBH1M1ZxLYBng8Irqqt7WSeQ5WaTw/oBzOazmc1/I4t+VwXtu3\nQX7RcJ7vNDwiTm9a+VVG0pnA6oi4sKp8AvCFiKjZSJF0PXBhRMyuse5AYOeIuLiMmPuLpLcAJ0XE\nF/s7lvVF0pHARyLimAZ1/EXDZfAXDZfGX4JZDue1HM5reZzbcjiv9anOFw1vqA2sMcBlwItV34X1\nqtduA0vSMNIwz0URcfT6i9TWN6XH778b+LeIuL1BvGGqlgAAIABJREFUPTewNiST3MAyMzPrD4Oq\ngWVm5ZHkXwobkBE7jGDl0yubVzQzM7M+Va+B1cocLDN7lYkIv/r4NXv27FL268aV5weUxXkth/Na\nHue2HM5r+9zAMjMzMzMz6yMeImhm3UgK/14wMzMza8xDBM3MzMzMzErmBpaZ2XrgMezlcW7L4byW\nw3ktj3NbDue1fW5gmZmZmZmZ9RHPwTKzbjwHy8zMzKy5enOwNuqPYMxsYJN6/K4wszpGjBjNypWP\n93cYZmY2QHiIoJnVEH71+Wv2AIhhsL76N7fPPvsEg5HnXZTDeS2Pc1sO57V9DRtYkkZLeqDOutmS\nxpUTVmOSRkm6T9LNhbIV/RFLPZImSJrWQr0BFXdRK7E1qyPpTEmn9V1U3fcpaZqk8S1s831Jj0i6\nX9LYFurPljSqyfq27n9JEyUtlXR7fn9Fjuez+TwOb7J903OVdKykxfl1l6TdCusukPSQpAntxG1m\nZmZmrWtliGCUHkX7DgNmRcTphbKBGGcrMQ3EuCs29PgBkPQhYExEvFHSO4AfAfv0QygnA6dExN2S\nRgJ7RcQbc4xNG+MtegwYHxF/lPRB4FLyuUbEFyTNB/4ZmNNHx7OWdfR3AINYR38HMCh1dHT0dwiD\nkvNaHue2HM5r+1oZIrixpOn5L+9XSdq0uoKkYyQtya/zctmQ/Bf3Jfmv6Z/N5WMk3Zr/cn+vpJ17\nEffWwHNVZc8X4jkhH3ORpMtz2TRJUyTNlfRopbdA0haSbsuxLJZ0SC4fLWlZ3m65pBmS3pe3Xy5p\nr1xvc0lTJc2TtFDSwTmMvwJ/bOFcns/7GSlpTu6ZWyJp31y+WtLZOV93S3pdLj+ocMxZhfIzJf00\n110u6ZRcPiHv/wZJD0u6RMlJkr5byN0pki6ozmmz+OvlvUjS30u6WdKCHMubJG0l6fFCnc0lPSlp\naK36NY6/ipTrRg4FfgoQEfcAwySNaLLN74E19e7j7EhJ9+R8Vq7XiZIuKpzP9ZLGS/oasB8wVdK3\ngFuAHfL13q8qT+MkdebzvrkQa9NzjYh5EVG57+YBO1RVWUn692NmZmZmZYiIui9gNNAF7JPfTwVO\ny8uzgXHAdsATwLakBtvtwCF53azCvrbKP+cBh+TlTYBNG8VQJ67JwOfqrNsVeBjYJr/fOv+cBlyZ\nl98MPJKXhwJb5uXhhfLRpA+zu+b39wJT8/IhwMy8fA5wbF4eBiwHNquKaU/g0ibndBrwb3lZwBZ5\nuQv4cF4+H/hK5ViFbU8Gvp2XzwQW5dwOB54ERgITgP/N5yVgFnA4sAXwKDA0bz8XeEsvrkm9vJ9Z\nuGduI/UkAbwduD0v/xKYkJePrOSqQf21+6xxXxxUo/x64F2F97cB41o8r3r38exCzj8E3JqXTwS+\nX3Xs8YVt9ijcX0sK9abl67FRvgbDC/mY2uq5VtX5YvV9B7wbuKHJdgHhV5+/Zg+AGAbrq79zSwxG\ns2fP7u8QBiXntTzObTmc1/ry73+qX60MEXwyIubl5enAZ4ALC+v3BmZHxB8AJM0AxgNnAztLmgLc\nBMyStCWwfURcR4qoWc9DD5IE7J5jqeU9wNUR8UI+xqrCumty2TJJr6/sEjhXaW5LF7B9Yd2KiFia\nlx8ifTAHeADYKS+/HzhY0pfy+02AUaSGFvl4C4GPNzm1BaTejY2BayNicS5/KSJuyssLgffm5R0l\nXUVq4G4MrCjs69qc299LuoPUOPkjMD8inoA0/wfYLyJmKs0JOkjSw8BGEfFQk1hraZR3JG0BvAu4\nOl9DctwAVwFHkYatHQ38oEn9miLizF7E3cxjVN3HhXUz88+FpAZTK5o9nm8X4K3Arfm8hwC/q67U\n7Fwl7Q+cROo1K3oGeJOk10TES/X3MKmw3IGHYJmZmdmrXWdnZ0sP/ejNHKzq91DjQ2NErJK0O/AB\n4JPAEcDnatXttiPpU8DH8nE+HBErC+uGkD7wvgTc2ELs1YofKCtxHAe8ltSz0KX00IZNa9TvKrzv\nYl3uBHw0Ih7pRTxrRcSduZF3IHCZpAsiYjrwcqHamsJxLwK+ExE3Kj20oPiBu3iNRO1rVqw3FfgK\nqQdq2is5jwaGAC9ERK0HQ1wHnCNpG1KP0R3Alg3qt+sZYMfC+zfksqbq3Men5NWV+6F4Xf5G96G3\nPYbUNiHgwYjYt83t1u0gPdjiUuCDlQZvRUQ8JmkZ8ISkA+o3pif19vBWV0d/BzCIdfR3AIOS512U\nw3ktj3NbDud1nY6Ojm75mDx5cs16rczBGq30YACAY4E7q9bPB8ZL2lbSUOAYYI6k4aRhZ78Evkoa\nkvUi8JSkQwEkbSJps+LOIuKSiNgjIsYVG1d5XVdE7EQarndUnXjvAI6QtG0+xjZ16lUaWMOA53Lj\nan+690S08mVAtwCnrt2ghSfU1QwmPbHuuYiYCvyE1NBoFMNWrOvZOLFq3aE5t8NJQwMX5PK9leaW\nDSHl7y6AiJhPaoAcA1xRJ75lTU6hYd4jYjWwQtLEwj53y+v+RLqmU0jD16JR/V64Djgh72MfYFVE\nPJvf3yZpu3ob1rqP61XNPx8Hxub5bTuSeg/r7r5G2XLgdTlOJG0kadcG+6iOdxTwX8A/RsRva6zf\nDdiZ1JPcm55KMzMzM2uglQbWw8CnJS0lTY7/US4PgNwIOh3oJM39WRAR15Mm13dKWgT8LNeB9EH3\nVEmLSXNNmj1soJbfkOZ89ZCH9J1DauQtAioPbKjXEzeD1PBYDBwPLKtRp9b2Fd8gPQhkidIj7c+q\nriBpT0mXNjgfSH+CXSzpPtK8m+81Oe5k4D8lLaDnwyiWkK7H3cBZhYbqvcDFpOGOv82NhoqrgLmx\n7gEJxfiHN4m9Ud6LjgdOVnpgx4OkuWwVV5J6E39RKDuuQf0eJE2WdFCN2G4iNdYeBX4MfCrXFzAG\n+EOD3da7j2veTxExl9TIeoh0DRdW16nzvrL9y8BE4HxJ95P+Tb2z1XMFvkb6t3GJ0sNG5let3wZ4\nPCK6amxrpers7wAGsc7+DmBQ8nfflMN5LY9zWw7ntX1K87M2LHm+0/Do/ph2Iz1FEFgdERdWlU8A\nvhARNRspkq4HLoyI2TXWHQjsHBEXlxFzf5H0FuCkiPhif8eyvkg6EvhIRBzToE7Ub9db73XioWxl\n6aR/cys2xP9Lm+ns7PTQoBI4r+VxbsvhvNYniYjoMSJpQ21gjQEuA16MiA/1czgDSrsNLEnDSMM8\nF0XE0esvUlvflB6//27S0ypvb1DPDSyztgzOBpaZmTU2qBpYZlYeN7DM2uUGlpnZq1G9BlYrc7DM\n7FVHfvnlV4uvESNa/ZaGDYvnXZTDeS2Pc1sO57V9rTym3cxeZfzX+L7nMezlcW7NzGwg8RBBM+tG\nUvj3gpmZmVljHiJoZmZmZmZWMjewzMzWA49hL49zWw7ntRzOa3mc23I4r+1zA8vMzMzMzKyPeA6W\nmXXjOVhmZmZmzXkOlpmZmZmZWcn8mHYz60Hq8ccYMzMzsw3SiB1GsPLplevteB4iaGbdSAom9XcU\ng9AKYOf+DmKQcm7L4byWw3ktj3NbjsGQ10nlfMdnr4YIShot6YE662ZLGtdXAbZD0ihJ90m6uVC2\noj9iqUfSBEnTWqg3oOIuaiW2ZnUknSnptL6Lqvs+JU2TNL6Fbb4v6RFJ90sa20L92ZJGNVnf1v0v\naaKkpZJuz++vyPF8Np/H4U22f0XnKukCSQ9JmtBO3NZHNvT/nAYy57Yczms5nNfyOLflcF7b1soc\nrIHYxXUYMCsiPlQoG4hxthLTQIy7YkOPHwBJHwLGRMQbgU8AP+qnUE4GTomIAySNBPaKiLERMaWv\nDtDoXCPiC8BZwD/31fHMzMzMrLtWGlgbS5qe//J+laRNqytIOkbSkvw6L5cNyX9xXyJpsaTP5vIx\nkm7Nf12/V1Jv2sVbA89VlT1fiOeEfMxFki7PZdMkTZE0V9Kjld4CSVtIui3HsljSIbl8tKRlebvl\nkmZIel/efrmkvXK9zSVNlTRP0kJJB+cw/gr8sYVzeT7vZ6SkOblnbomkfXP5akln53zdLel1ufyg\nwjFnFcrPlPTTXHe5pFNy+YS8/xskPSzpEiUnSfpuIXenSLqgOqfN4q+X9yJJfy/pZkkLcixvkrSV\npMcLdTaX9KSkobXq1zj+KlKuGzkU+ClARNwDDJM0osk2vwfW1LuPsyMl3ZPzWbleJ0q6qHA+10sa\nL+lrwH7AVEnfAm4BdsjXe7+qPI2T1JnP++ZCrH1xritJ/35sfRuwfdWDgHNbDue1HM5reZzbcjiv\nbWulgbULcHFE7AqsBj5VXClpO+A8oAMYC+ydGyljgR0iYreI2B2oDJebAVwUEWOBdwH/txdxDwW6\nigUR8Y4cz67AV4COiNgDKH4gHhkR+wIHA+fnsr8Ah0XEXsB7gAsK9ccA346IXXIejs7bfykfA+AM\n4PaI2Cdv/x1Jm0XEryPi8zmmPSVdWutEKnEDxwK/iohxwO7A/bl8C+DunK87gY/l8jsjYp+I2BO4\nEvhyYbdvI12PdwFfV+otAdgb+DTwZuAfgI8AVwEHSxqa65wE/EdVbHW1mPeKS4F/iYi9STn8YUT8\nD7BI64atHZTzsKZW/RrH/3xEzMsxTJZ0UI3j7gA8VXj/TC5rdF4TI+IZ6t/HAEPz+X8eus1a6tGr\nFxHfAO4Fjo2ILwOHAI9GxLiIuKtST9JGwEXAR/N5TwO+2Yfn2kX692NmZmZmJWjlKYJPVj7UAdOB\nzwAXFtbvDcyOiD8ASJoBjAfOBnaWNAW4CZglaUtg+4i4DiAimv01vgdJIjVAptep8h7g6oh4IR9j\nVWHdNblsmaTXV3YJnKs0t6UL2L6wbkVELM3LDwG35eUHgJ3y8vtJDZQv5febAKOA5ZWDRsRC4ONN\nTm0BqXdjY+DaiFicy1+KiJvy8kLgvXl5R0lXAdsBG9P97wvX5tz+XtIdwNtJvWnzI+IJSPN/gP0i\nYqbSnKCDJD0MbBQRDzWJtZZGeUfSFqQG39X5GpLjhtTIOwqYAxwN/KBJ/Zoi4sxexN3MY1Tdx4V1\nM/PPhcDoFvfX7PF8uwBvBW7N5z0E+F11pVdwrs8Ab5L0moh4qW6t2YXlnfD4677gHJbHuS2H81oO\n57U8zm05nNe1Ojs76ezsbFqvlQZW9V/ja8256fGhMSJWSdod+ADwSeAI4HO16nbbkfQpUi9NAB+O\niJWFdUNIH3hfAm5sIfZqxQ+UlTiOA14L7BERXUoPbdi0Rv2uwvsu1uVOpN6GR3oRz1oRcWdu5B0I\nXCbpgoiYDrxcqLamcNyLgO9ExI2596f4gbt4jUT9eVKV8qmk3qeH6d5D05eGAC/kHrpq1wHnSNoG\nGAfcAWzZoH67ngF2LLx/Qy5rqs59fEpeXbkfitflb3TvGe4xpLYJAQ/mntLeaHiuEfGYpGXAE5IO\nqNuY3r+XRzczMzMbpDo6Oujo6Fj7fvLkyTXrtTJEcLSk4jC2O6vWzwfGS9o2DzM7BpgjaThpCNUv\nga8C4yLiReApSYcCSNpE0mbFnUXEJRGxRx46tbJqXVdE7EQaanVUnXjvAI6QtG0+xjZ16lUaWMOA\n53Ljan+690S08mVAtwCnrt2ghSfU1QwmPbHuuYiYCvyE1NBoFMNWrOvZOLFq3aE5t8OBCaTeMUjD\nN0fnhupRwF0AETGf9KH8GOCKOvEta3IKDfMeEauBFZImFva5W173J9I1nQLcEEnd+r1wHXBC3sc+\nwKqIeDa/vy0Pc62p1n1cr2r++TgwVsmOpN7DuruvUbYceF2OE0kb5eGXrap7rrlsN9LforbvZU+l\n9ZbHsJfHuS2H81oO57U8zm05nNe2tdLAehj4tKSlpMnxlaeSBUBuBJ0OdAKLgAURcT1p3kenpEXA\nz3IdSB/+TpW0GJgLNHvYQC2/AbattSIP6TuH1MhbxLo5VfV64maQGh6LgeOBZTXq1Nq+4hukB4Es\nUXqk/VnVFRrNwSroABZLug84Evhek+NOBv5T0gJ6PoxiCel63A2cVWio3gtcTBru+NvcaKi4Cpgb\nET0ezJEbGQ01yHvR8cDJSg/seJA0D6niSlJv4i8KZcc1qN9DvXlJeYjlCkmPAj8mzyPMQ/DGAH9o\nsNt693HN+yki5pIaWQ+RruHC6jp13le2fxmYCJwv6X7Sv6l3vtJzLdgGeDwiuqq3NTMzM7NXboP8\nouE832l4RJzetPKrjKQzgdURcWFV+QTgCxFRs5Ei6XrgwoiYXWPdgcDOEXFxGTH3F0lvAU6KiC/2\ndyzri6QjgY9ExDEN6viLhs3MzGzwmDSAvmh4AJsJ7KvCFw1b70gaJmk58KdajSuAiLhxsDWuACLi\noVdZ4+oC4IukIahmZmZmVoINsgfLzMojyb8UzMzMbNAYscMIVj69snnFNtXrwWrlKYJm9irjP7z0\nvc7Ozm5PHrK+49yWw3kth/NaHue2HM5r+9yDZWbdSAr/XjAzMzNrbLDNwTIzMzMzMxtw3MAyM1sP\nWvnmd+sd57Yczms5nNfyOLflcF7b5waWmZmZmZlZH/EcLDPrxnOwzMzMzJrzHCwzMzMzM7OSuYFl\nZj1I6vYa+YaR/R3SBs9j2Mvj3JbDeS2H81oe57Yczmv7/D1YZtbTpO5vn530bL+EYWZmZrah8Rws\n63eSVkfE3/V3HACSxgGXA/Mj4uRctiIidu6HWHYHto+Im/P7E4GdImJyk+1uBvYB7oyIQwrlxwBn\nAj+OiO822D6qG1hM8pcPm5mZmRV5DpYNZAPpk/vxwA8qjausrfgk9dW/q7HAh6vKWonlW6Tz6L5h\nxBXABODzrzw0MzMzM6vFDSwbMCRNlrRI0n2SnpY0VdJoScskTZO0XNIMSe+TNDe/3ytvu7ekuyUt\nlHSXpDf2Moytgeeqyp7Px5ggaY6kGyQ9LOmSQuyrJX1H0iJgH0njJHVKWiDpZkkjcr1TJT0k6X5J\nP89lm+dznZfjP1jSxsBZwJE5H0cA/wu82OwEImJ2vXoR8SwwrO2s2CvmMezlcW7L4byWw3ktj3Nb\nDue1fZ6DZQNGRJwJnClpGPDfwEV51RjgoxGxVNK9wNERsa+kQ4AzgI8Ay4D9IqJL0gHAucDEXoQx\nFOiqiusdhbd7A28GngRukXR4RMwEtgB+HRFflLQRMAc4JCJ+L+lI4JvAycC/kob5vSxpq7zPM4Db\nI+LkfO7zgduArwN7RsSp1UFKOjivm9SLc/QfVszMzMxK4gaWDUTTgQsi4n5Jo4EVEbE0r3uI1PgA\neAAYnZe3Bn6ae66CXtzbuWH0FtY17GqZHxFP5PpXAPsBM4E1+SfALsBbgVslidSg+V1etxj4uaRr\ngGty2fuBgyV9Kb/fBBjVKNaIuB64vvWz6+YPksZExG/r1phdWN6pl0exbjo6Ovo7hEHLuS2H81oO\n57U8zm05nNd1Ojs7W+rRcwPLBhRJk4AnI+KnheKXCstdhfddrLuHvwHcERGH50ZZsYlQ2ffZwIFA\nRMS4qnVvIPUcPRoR9zYIsXoOVOX9nwvfzivgwYjYt8b2BwLjgUOAMyS9Ldf/aEQ8UhXTPg3ieCWm\nAPdL+kxEXFazxv4lHdnMzMxsA9XR0dGtwTl5cu3njnmokA0EgrXD3t4LfLbW+iaGAc/k5ZNqVYiI\nr0bEHtWNq7zuaWCHFIY6Ghzn7Xle2BDgKODOGjEuB15XaSBJ2kjSrnndqIiYA5wObEUaWngLsHYY\noKSxeXF1rtMbon7evgL8Q93GlZXCY9jL49yWw3kth/NaHue2HM5r+9zAsoGg0vPzeWB7YEF+sMOk\nqvXVy0XfAs6TtJBe3te5B+pRYNsG1e4FLiYNVfxtRFSG+a2NKyJeJs3/Ol/S/cAi4J15COJ0SYuB\nhcCUiPgfUu/bxpKWSHqA9HALSL1wuxYecrFWfhDGpFoBSvpv4ErgPZKelPS+qiqb5IddmJmZmVkf\n8/dgmRVI+gHwQET8qMa6CcAXit8ttaGR9HpgcURs16COvwfLzMzMrAl/D5ZZa34KnCRpan8H0tfy\nFw3PIvX2mZmZmVkJ3MAyK4iIeyLiHVVfNFxZN2dD7r2KiCsiYmxEfLdp5UndXyN2GFFmaK8KHsNe\nHue2HM5rOZzX8ji35XBe2+enCJpZDx4OaGZmZtY7noNlZt1ICv9eMDMzM2vMc7DMzMzMzMxK5gaW\nmdl64DHs5XFuy+G8lsN5LY9zWw7ntX1uYJmZmZmZmfURz8Eys248B8vMzMysOc/BMjMzMzMzK5kb\nWGbWg6Rur5FvGNnfIW3wPIa9PM5tOZzXcjiv5XFuy+G8ts/fg2VmPU3q/vbZSc/2SxhmZmZmGxrP\nwTLrJ5JGAzdExNtarP8t4GDgJeD/t3fnQZaV9RnHvw8gIgjExDiYGVmMIooiM+CgAWJjAi4ERIyC\nQRHUbEyEimK5JTBdRaKE0kgBEhcyIehIobggZRREhgQVWYZNFiWyCMYZF1DAKBH45Y97Gk/3dPfM\n7XvP9CzfT1VXn/Oec+5971M9XfPr933P+R5wdFXd38f7PQtYAiwA3lNVH5zivJpYYLHYhw9LkiS1\nuQZLWjf1U7VcBOxaVbsDtwHv7vO9fgq8FTilz+skSZK0hiywpNn1uCSfSHJzkvOSbJFkjyTXJlme\n5IYkjwBU1Ver6tHmuiuAef28UVX9pKquAR4e8mfQGnAOe3fMthvm2g1z7Y7ZdsNc+2eBJc2uZwGn\nV9VzgAeAY6rqmqqaX1ULgC8z+YjTm4D/WIv9lCRJ0hpwDZY0S5o1WJdV1Y7N/n7AW6vq0Gb/MOAt\nwAHtB1MleS+woKpePcP3PRF4YNo1WC9uNewInO0aLEmStHFbtmzZuBG90dHRSddgeRdBaXZNrFoK\nIMlzgROAfScUV0cBrwBeMtmLJTkJOBCoZgRsZvab8ZWSJEkbpJGREUZGRh7bHx0dnfQ8pwhKs2uH\nJHs1238GXJ5kW2ApcGRV3Tt2YpKXAe8ADq6qhyZ7sar6u9b0wums8tcWdcs57N0x226YazfMtTtm\n2w1z7Z8jWNLsuhVYlGQJ8G3gTOC1wPbAx5KE34xGnQZsDlzca+aKqjpmTd8oyRzgamBr4NEkxwHP\nqaoHh/mBJEmSNmauwZI0js/BkiRJWj2fgyVJkiRJHbPAkrSqxeO/5sydM4ud2TA4h707ZtsNc+2G\nuXbHbLthrv1zDZakVTgdUJIkaWZcgyVpnCTl7wVJkqTpuQZLkiRJkjpmgSVJa4Fz2Ltjtt0w126Y\na3fMthvm2j8LLEmSJEkaEtdgSRrHNViSJEmr5xosSZIkSeqYBZakVSQZ6Gu7edvN9kdY5ziHvTtm\n2w1z7Ya5dsdsu2Gu/fM5WJJWtXiwy1cuXjmUbkiSJK1vXIMlrWOS3AHsUVX3Jrm8qvZJ8mLg+Ko6\naIDXPQv4E2BlVe02zXk1aIHFYh9WLEmSNmyuwZLWH49VJlW1z2TtM7QEeOmAryFJkqRpWGBJsyTJ\nXya5NsnyJLcnuWTsUOucB1qXbJvkwiS3Jvlwv+9XVZcD9w3Ybc2Qc9i7Y7bdMNdumGt3zLYb5to/\nCyxpllTVR6pqPrAQuBv4wGSntbZfACwCng08I8mh3fdSkiRJ/XANljTLmtGolVU12uy312DdX1Xb\nNGuwRqtqpDnnaOB5VfW2Pt9rB+CLq12D9eJWw47ATn19JNdgSZKkDc6yZcvGjeiNjo5OugbLuwhK\nsyjJUcDTquqYNTh9YsUybj/JQuAjTfsJVXXhjDu234yvlCRJ2iCNjIwwMjLy2P7o6Oik5zlFUJol\nSfYA3g68frrTWtt7JdkhySbAYcDl7ROr6sqqml9VC6YprjLhNbWWOIe9O2bbDXPthrl2x2y7Ya79\ns8CSZs8i4EnApc2NLj7atLdHptrbVwKnAzcB36uqz/XzZkmWAt8Adk7y/WaaoSRJkobINViSxvE5\nWJIkSavnc7AkSZIkqWMWWJK0FjiHvTtm2w1z7Ya5dsdsu2Gu/fMugpJWtXiwy+fMnTOUbkiSJK1v\nXIMlaZwk5e8FSZKk6bkGS5IkSZI6ZoElSWuBc9i7Y7bdMNdumGt3zLYb5to/CyxJkiRJGhLXYEka\nxzVYkiRJq+caLEmSJEnqmAWWJK0FzmHvjtl2w1y7Ya7dMdtumGv/fA6WpFUkq4x2S5IkbTTmzJ3D\nintWzOha12BJGidJDfqgYUmSpPXaYlhdneQaLGkNJHk0ySmt/bcnOWGW+rJD059FrbbTkhw5G/2R\nJEnS6llgSeM9BBya5LdnuyONHwHHJXE67/rujtnuwAbMbLthrt0w1+6YbTfMtW8WWNJ4DwMfBd42\n8UAzonRJkuuSXJxkXtO+JMmpSb6e5L+THNq65vgkVzbXnDiD/vwYuAQ4apL+7J7km81rn59k26b9\n0iTvT/KtJLcm2btp3yTJPzXt1yX58xn0R5IkSdOwwJLGK+AM4IgkW084dhqwpKp2B5Y2+2O2q6q9\ngYOAkwGS7A88s6oWAvOBPZPsM4P+nAwcn1XvPHE28I6mP98G2gXcplW1F/C38NiKqjcDP2vaFwJ/\nkWSHPvujmdpptjuwATPbbphrN8y1O2bbDXPtm9OOpAmq6sEkZwPHAb9sHXoR8Kpm+xyaQqrx+eba\nW5I8pWk7ANg/yXIgwFbAM4HL++zPnUmuAI4Ya0uyDbBtVY291tnAea3LPtt8vwYYK6IOAJ6X5DXN\n/jZNf+5a5U0vbW3viL9cJUnSRm/ZsmVrdNt6CyxpcqcCy4ElrbbpbiXzUGs7re/vq6qPTXVRkkPo\njTwV8JaqWj7Fqe8DPgMsm+R9puvPI/zm33mAt1bVxdNc17Pfas9Qv+7AQrUrZtsNc+2GuXbHbLth\nro8ZGRlhZGTksf3R0dFJz3OKoDReAKrqPnojQm9uHfsG8Lpm+/XAf033GsBXgDcl2Qogye8l+d32\niVX1+aqaX1ULpiiuxvrzHeBm4OBm/37g3rF0qtlUAAAIw0lEQVT1VcAbgMvWoD/HjN0wI8kzkzxh\nimskSZI0A45gSeO1R6k+ACxqtR0LLElyPL2bTxw9yTWP7VfVxUl2Ab7ZLJ96gF5h9uMZ9ucf6I2q\njTkK+JemSLp9df0BPk5vwt/yZj3Xj4BD+uiLBuFf/7pjtt0w126Ya3fMthvm2jcfNCxpHB80LEmS\nNnqLfdCwJK3bfI5Id8y2G+baDXPtjtl2w1z75giWpHGS+EtBkiRt1ObMncOKe1ZMe44jWJLWWFX5\nNeSvE088cdb7sKF+ma25rk9f5mq269vXxprr6oqr6VhgSZIkSdKQWGBJ0lpw5513znYXNlhm2w1z\n7Ya5dsdsu2Gu/XMNlqRxXIMlSZK0ZmqSNVgWWJIkSZI0JE4RlCRJkqQhscCSJEmSpCGxwJIkSZKk\nIbHAkgRAkpcluTXJd5O8c7b7s65LclaSlUluaLU9KclFSb6T5CtJtm0de3eS25LckuSAVvuCJDc0\nuX9obX+OdVGSeUm+luSmJDcmObZpN98BJHl8km8lubbJ9h+bdnMdgiSbJFme5IJm31yHIMmdSa5v\nfm6vbNrMdkBJtk3y6Sanm5LsZa7DY4EliSSbAKcDLwV2BV6XZJfZ7dU6bwm9vNreBXy1qp4FfA14\nN0CS5wCvBZ4NvBz4cJKxuw6dCby5qnYGdk4y8TU3Rg8Db6uqXYEXAYuan0fzHUBVPQTsV1Xzgd2A\nlyTZG3MdluOAm1v75jocjwIjVTW/qhY2bWY7uFOBL1XVs4HnA7dirkNjgSUJYCFwW1XdVVW/Bs4F\nXjnLfVqnVdXlwH0Tml8JnN1snw0c0mwfDJxbVQ9X1Z3AbcDCJNsBW1fVVc15/966ZqNVVSuq6rpm\n+0HgFmAe5juwqvrfZvPx9P4PcB/mOrAk84BXAB9vNZvrcIRV/79qtgNIsg2wb1UtAWjy+jnmOjQW\nWJIA5gJ3t/bvadrUn6dU1UroFQnAU5r2ifn+oGmbSy/rMeY+QZIdgd2BK4A55juYZhrbtcAKYFlV\n3Yy5DsM/A+8A2s++MdfhKODiJFcleUvTZraD2Qn4SZIlzbTWjybZEnMdGgssSeqODxocQJInAp8B\njmtGsibmab59qqpHmymC84B9k4xgrgNJciCwshl1XeWBoy3mOjN7V9UCeiOEi5Lsiz+zg9oMWACc\n0WT7C3rTA811SCywJEHvr1Hbt/bnNW3qz8okcwCaqRM/atp/ADytdd5YvlO1b/SSbEavuDqnqr7Q\nNJvvkFTV/cCXgD0x10HtDRyc5HbgU/TWtp0DrDDXwVXVD5vvPwY+T29Kuz+zg7kHuLuqrm72z6dX\ncJnrkFhgSQK4CnhGkh2SbA4cDlwwy31aH4Txf7G+ADiq2X4j8IVW++FJNk+yE/AM4MpmCsbPkyxs\nFgwf2bpmY/evwM1VdWqrzXwHkOTJY3cFS/IEYH/gWsx1IFX1nqravqqeTu9359eq6g3AFzHXgSTZ\nshnJJslWwAHAjfgzO5BmGuDdSXZumv4IuAlzHZrNZrsDkmZfVT2S5G+Ai+j94eWsqrpllru1Tkuy\nFBgBfifJ94ETgfcDn07yJuAuenddoqpuTnIevTuM/Ro4pqrGpl4sAv4N2ILeHZ2+vDY/x7qoubPd\nEcCNzXqhAt4DnAycZ74z9lTg7OY/QpvQGx28pMnYXIfv/ZjroOYAn0tS9P7P+smquijJ1ZjtoI4F\nPpnkccDtwNHAppjrUOQ3+UiSJEmSBuEUQUmSJEkaEgssSZIkSRoSCyxJkiRJGhILLEmSJEkaEgss\nSZIkSRoSCyxJkiRJGhILLEmStE5JcmmSBVMcOzfJ05vtO5NcNuH4dUluaLbfmOS0ad7nzCQvmuLY\nwUn+fuafQtLGygJLkiStF5L8PrBVVd3eNBWwdZK5zfFdmra26R74uRdwxRTHvgi8OslmA3RZ0kbI\nAkuSJE0ryZZJLkxybZIbkrymab8jyclN2xWtkaUnJ/lMkm81X3/Qep2zmnOvSXJw075Fkk8luSnJ\nZ4EtpujK4fQKn7bzmnaA1wFLJxzfvhkR+06SE1qfaRfgu1VVSY5t3vu6JEsBqqqAbwAHzDA2SRsp\nCyxJkrQ6LwN+UFXzq2o34MutY/c1bWcApzZtpwIfrKq9gD8FPt60vxe4pKpeCLwEOCXJE4C/Bn5R\nVbsCJwJ7TtGPfYCrW/sFnA+8qtk/iFULsBc0x58PvKY19fDlrc/xTmD3qtod+KvWtVcBfzhFXyRp\nUhZYkiRpdW4E9k/yviT7VNUDrWPnNt8/Bbyw2f5j4PQk1wIXAE9MsiW90aB3Ne3LgM2B7ekVMZ8A\nqKobgeun6McOwA8ntP0UuC/JYcDNwC8nHL+4qn5WVb8CPkuvSAN4Kb8psK4HliY5Anikde3/ADtO\n0RdJmpTziiVJ0rSq6rZm5OcVwElJvlpVJ40dbp/afN8E2Kuqft1+nSQAr66q2yZpH9c0VVemOHYe\nvRG0I6e4Ztx+M2q2bVWtaNoOpFfkHQy8N8lzq+rR5r2mW8MlSatwBEuSJE0ryVOBX1bVUuAUoH2H\nv8Oa74cD32y2vwIc17r++a32Y1vtuzeb/wkc0bQ9F9htiq7cBWzX7lrz/XPAycBFk1yzf5Lfaoqq\nQ4CvA/sBlzbvF2D7qroMeBewDfDE5tqnNu8pSWvMESxJkrQ6z6O3XupR4P8Yv07pSUmuB35F7yYT\n0CuuzmjaN6VXQB0DnAR8qLmNeoA76I0anQksSXITcAvj11m1XU5vfdbyZr8AqupBeoXfZKNhV9Kb\nGjgXOKeqlje3bv90c3xT4BNJtmn6dGpV3d8cWwhcuNp0JKklvZvkSJIk9SfJHcAeVXXvWnq/pwOn\nVdWBA77O1fSmMD4yzTmhV8i9oKoeHuT9JG1cnCIoSZJmaq3+lbZ5/tX9Y7eDH+B19pyuuGocBJxv\ncSWpX45gSZIkSdKQOIIlSZIkSUNigSVJkiRJQ2KBJUmSJElDYoElSZIkSUNigSVJkiRJQ/L/uDxL\nkuJrs2cAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "co_t, de_t = compression_decompression_times()\n", - "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Compression speed', fontsize=14, y=1.01)\n", - "\n", - "\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", - "x = (nbytes / 1000000) / np.array([co_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='bit shuffle', color='b')\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", - "x = (nbytes / 1000000) / np.array([co_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='no shuffle', color='g')\n", - "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", - "ax.set_yticklabels(labels, rotation=0)\n", - "\n", - "xlim = (0, np.max((nbytes / 1000000) / np.array(co_t)) + 100)\n", - "ax.set_xlim(*xlim)\n", - "ax.set_ylim(0, len(co_t))\n", - "ax.set_xlabel('speed (Mb/s)')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", - "\n", - "fig.tight_layout();" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAMWCAYAAADszSe0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcVMW9///Xm00CiIAaVlkk6nUJyLgLARQ1eqOEGEQU\nlxjUeM034pqH1yWAC2oi/uJN1MQEUaNZ9F7iBQwqCIPRiKhsCkiUTUFxYXO7YmQ+vz+qGs709DbD\nzPQ0fJ6PRz+m+1Sdqs85dcRTXVWnZWY455xzzjnnnNtxjYodgHPOOeecc87tLLyD5ZxzzjnnnHO1\nxDtYzjnnnHPOOVdLvIPlnHPOOeecc7XEO1jOOeecc845V0u8g+Wcc84555xztcQ7WM4553Y6ks6X\n9HGx4ygWSXtKqpDUv9ixOOfcrsY7WM45VySSJsab4K2SvpT0vqSZki6V1KTY8ZW4PwP7FjuIIvMf\nunTOuSLwDpZzzhXXdKAD0A04EZgMjAX+LulrxQysrkhqWtd1mNkWM/uorutp4FTsAJxzblfkHSzn\nnCuuLWb2oZm9Z2aLzOyXwECgDPhpKpOkppLukPSOpM8kvSTppGRBkg6Q9L+SNkn6RNILkg6OaZJ0\no6S3JX0haZGkwYl9u8XRtDMllUv6XNI8Sd+Mr39I+lTSbEn7JPYbLek1SSMlrY77/VXSnok8EyVN\nkfRTSe8A7xRyTJKaSPovSWtjzKsljUukny5pYaxzvaRZkvaOaT+Q9Ena+fmRpDclbYl/L0xLr5B0\nkaTH4rEulzQiV+NJOkTSDEmb4zmfL2lATBsQy/xO3P5/kl6RVJZWxrHxnH8maY2keyXtnpbnp5Le\nise6MD0uSUfEsv9P0qvAUbnids45V3e8g+Wccw2MmS0GngK+n9j8IPAtYDhwMPAQMFnSNwEkdQSe\nB7YCg4DewH8BjeP+lwNXAdcAhwB/BSZJ6pVW/RjgNuBQYBPwR+BXwLXAEUCLWG5Sd2AEcFqsez9g\nQlqeAcA3gW/HPHmPCRgFfBcYBnwDOBNYFo+3PfAnYCLwb7GcPyTqMxJT5CR9Lx7HXbGuu4F7JX0n\nLc4b47npBfwFeEBSF7L7I/AucDjhnI8BvkjL8wvCeT8MWAFMkdQ8xvVN4GngiXh+vhfLeSAR+63A\nBcB/AAcS2uc3kk6J6S2BqcBbhI75tcCd+BRB55wrDjPzl7/85S9/FeFF6BxMzpJ2G/BpfN+T0HHq\nkpbnr8Cv4/tbgZVA4yzlrQGuT9s2C3g4vu8GVAAXJtK/E7d9N7HtfGBz4vNo4F9A58S2vnG/nonj\nfB9oksizbwHHdDcwPcvx9In775Ml/Xzg48Tn54HfZTj/zyU+VwC3JD43Bj4Dzs7RhpuBc7OkDYhl\nDk9sawlsBH4YPz+UIa5D4357ETq0nwN90/L8f8DU+P5iYAPwtUT6iHh++hf7OveXv/zlr13t5Yuo\nnXOuYRLbRyD6xM9LJCXX1TQDno3vDwWeN7OtVQoK0806Af9IS3oeOCVt22uJ9+/HGF5P29ZKUnMz\nS43UrDWztYk8LxE6CAcCy+O2183sq0SeshzHNDO+fxCYLumfwDPA34BpZmbAwnjsiyU9A8wA/tuy\nr7s6kKqjas8TRt0yHr+ZbZX0IfD1LGVCGBGbIOkHMZ7/MbNliXQD5iTK/EzSa8BBcdNhQE9JwxP7\npNo+1bFuDjxV+TTRhNChhjCCt8jM/i+R/iK+Bss554rCO1jOOdcwHUSYTgZhOncFYRraV2n5/o8d\nkz6N7F8Z0jJtq+4U88/SPuc9JjObL6kb26cVPgQsAE40swrgJElHAScBI4HbJPU3s9coXK7jT6Vn\nPVYzGyvpEUJH9WRgtKQfmdmDBdbfCPg9oaOW3iFaS5guCHAqce1ajlidc841AL4GyznnGhhJhxBu\n1h+Pm+YTbr47mtmKtNd7iTz9lOHx7mb2CWGdUN+0pH7AkloIubOkzonPR8V4c5VdyDFhZp+Z2SQz\n+zFhyuIgSd9IpL9kZjeb2RGEYzwzS31LqXr838oTY0HMbLmZ/drMTiWMkiUfniHg6G0fwnqpQxL1\nzgMONrOVGc7DlphvC9A9Q3qqw7UU+KYqP3XyGHwNlnPOFYWPYDnnXHHtFh/Y0AjYGzgB+E/gZWA8\ngJm9KemPwIOSribclLcjPG1wuZk9AdwL/Ah4PD4UYSPhoRRLzGwR4UELYyW9BbwKnEvoYPXJE18h\n08y+AB6SdBVhzdB9hPVBK7LtUMgxSboCeI8wavUVYV3RZmBNHLk6gfCAiPcJUw67AIuzVPkL4DFJ\n8wjTDU8BziI8VKJG4oMq7iR0hFcRHrffjzA9L+kGSR/FY/kZocP0p5h2B/CipPuA3wKfEKYznmpm\nl5jZp5LuBO6U1Ah4DmhF6LRtNbPfEx60cSswUdJNQGfgupoel3POuR3jHSznnCuuEwgjL1sJT+17\nnXAT/ru0NUs/AK4n3JB3ITzUYC5xvZKZvSupP6EjMZMwevEa4QEIEJ781yru357wNL7TzSy5virT\niEchoyArCT/sOwXYk9DpuaiA/XIeE6GzcQ3hCYJGGPU62cy+kLSZMCL1/4A2hOlzN5nZn8jAzP5X\n0k+AqwkPiFgN/IeZ/S3PseY6/q1AW8LDMjoC6wnn4Jq0/a8ldJb3J3QAv5NaL2Vmr8V2uwUoJzxY\nYwXhYR+p2G+UtI7wFMh7gY8Jnc6fx/TP4tMQ7yN0nt8gPOJ/co7YnXPO1RGFtcLOOedc9UkaDXzf\nzNIf977Li7+HNRPY28w2FDse55xz9cPXYDnnnHN1x5/k55xzuxjvYDnnnHN1x6eJOOfcLsanCDrn\nnHPOOedcLfERLOecc84555yrJd7Bcs4555xzzrla4h0s55xzzjnnnKsl/jtYzrlKJPnCTOecc865\nAphZlafFegfLOVeFP/ym9PzgBz/gwQcfLHYYrpq83UqTt1tp8nYrTQ253aTMv8ThUwSdc24n0L17\n92KH4GrA2600ebuVJm+30lSK7eYdLOecc84555yrJT5F0DlXRbYh711R+/bdWLduVbHDyKtNmzbF\nDsHVgLdbafJ2K03ebqWpFNvNO1jOuQx8DVbK+++XRmfz0EMPLXYIrga83UqTt1tp8nYrTaXYbvLF\n7M65pPAUQf93YTv5Qz+cc87Vqe7du7N69epih+Gy6NatG6tWraqyXVLGpwjW2xosSd0kvZYlbZak\nsvqKJa3urpLmSZqW2LayGLFkI2mApIkF5MsZt6RP4t+Okh6L78+X9KualFdgnaMlXZmvnOpIlilp\noqT+efIPkLQptvM8STcUUMcsSV3zpFfrmpU0VNISSc/Gz3+StEDSqHgcp+fZv5BjPVvSwvh6XlKv\nRNp4SYslDahO3M4555yrW6tXr8bM/NVAX9Xt/Nb3Qy4a4tfAQ4BnzOyUxLaGGGchMeXLYwBm9p6Z\nDStgv9qos6F4zszK4uuWIsUwErjQzAZJ6gAcbmaHmtndtVjHCqC/mfUGbgHuTyWY2VXATcAPa7E+\n10CUl5cXOwRXA95upcnbrTR5u7n6Ut8drKaSHonf4j8mqXl6BklnSVoUX7fHbY3it/eL4jfzo+L2\nnpKmx1GAVyT1qEFMbYAP0rZ9mIjnvFjnfEkPxW0TJd0t6QVJb6VGHiS1lDQjxrJQ0uC4vZukpXG/\nZZIelXRi3H+ZpMNjvhaSJkiaI+lVSafFML4ENhdwLB/GcsbGeOdJWiNpQupwEvEkRxO7xhGZZZJ+\nluk85Ksz27lKkrSvpGmSXpY0W9L+klpLWpXI00LS25IaZ8qfof5NhPOTT3UX0qwHtma79qJhkl6S\n9IakvjH+SiOCkqZI6i/pRqAfMEHSz4Gngc6xjfpVClQqk1Qej3uapPaFHquZzTGz1LUyB+iclmUd\n4Zp3zjnnnHN1oL4fcnEAcIGZzYk3/ZcCd6USJXUEbgf6EG4mp8dOyhqgs5n1ivlax10eBcaZ2WRJ\nzahZh7ExUJHcYGZHxXoOAq4DjjGzjZKSN6YdzKyvpAOBycAk4AtgiJl9KmlPwg3u5Ji/J/B9M1si\n6RVgeNx/cKzjdOB64FkzGylpD2CupBlm9iLwYozpMOBHZnZx+oGk4jaz0cDoWMZzQOqGPznalHx/\nBHBwjP9lSVPNbF6qvFwKPFcp98fYl0s6ErgvjubMlzTAzGYDpwJPmdlWSVXyA4PS6r8i9V7SWOBl\nM5uaoe5jJC0A1gLXmNmSPMc1NJZZRuZrD6CxmR0l6RRgDHBiavcM5d0s6XjgSjObL+keYIqZlcVy\nR8a/TQjtNdjM1ksaBowDRlbjWFMuBKalbasgXPN5jEm8HxhfriEbOHBgsUNwNeDtVpq83UqTt5vb\nUeXl5QWNhNZ3B+ttM5sT3z8C/IREB4twoz/LzDYASHoU6E+Y6tRD0t3A34BnJLUCOpnZZAAzK2QU\noxJJAnrHWDI5HnjczDbGOjYl0p6I25ZK+nqqSOA2hXUyFUCnRNrKxE39YmBGfP8a0D2+Pwk4TdI1\n8XMzoCuwLFWpmb0KVOlcZfEIcJeZLciTb3rq2CRNIoy0zCuwjpRc5wpJLYFjgcfjeQdoGv8+BpwJ\nzAaGA/fkyZ9R7Fhm8irQ1cw+j52hJ4BMo2GZrCDt2kukTUqU363A8vKNpB0AHEL4ckGELw3eTc+U\n41hDJdJxwAWEtkxaC+wvaTcz25K9hDF5wnTOOeec27UMHDiwUkd97NixGfMVew1WpvU7VW5A4816\nb6AcuAT4Xba8lQqSLk1MleuQltYIWAkcCDxZUPSVJW9OU3GMAPYC+phZH8LUw+YZ8lckPlewvaMr\nwihXn/jqYWbLqAFJYwgd2ipT9TIopF12VCNgY1wDlTq+Q2LaZOBkSW2BMmBmnvzVYmafmtnn8f00\nwlTVdgXum+3ag+1tuJXtbfgVlf+7qjINNg8BryeOu3fa+sD8BYQHW9xPGAXbmEwzsxXAUmC1pIOr\nGZtrwHxtQWnyditN3m6lqdTarUOH7kiqs1eHDt0LiqNHjx7MnDkzY9rzzz/PgQceWCvHm6uefL74\n4gtOO+002rRpw5lnngnADTfcwN57702nTp1YvXo1jRo1oqKiIk9JtaO+O1jdJKWmnZ0N/D0tfS7Q\nX1I7SY2Bs4DZcbpdYzP7K3ADUGZmnwLvSPougKRmkr6WLMzM7o03qWVmti4trcLMugOvEEZPMpkJ\nnJG6GY8dgExSHaw9gA/MrCKOIHTLkCeXp4HLtu0g1ejB/wprt04ARqUnZdnlRElt4vkbAryQocyl\nearNea7M7BNgpaShiTJ7xbTPCO1wNzDVgqz5qyuxhok41VCJUdIZcWpqtn2rXHvZssa/q4BDFewD\nHJkrtAzblgF7Szo61t8kTr8siMKTD/8HONfMlmdI7wX0IIz+Li60XOecc87Vr/ffX034zrtuXqH8\nHdOvXz+WLt1+i7gjnaQd8d///d98+OGHbNy4kb/85S+888473HXXXbzxxhu8+26YCLR9QlTdq+8O\n1hvAjyUtISy0/03cnnq63TrgWsJowXzCGpMphIX65ZLmA3+IeQDOAy6TtJDQKdh2I10N/wQyjmbE\nKX23Ejp584HxyXiTWePfR4EjYjznEEYK0vNk2j/lZsLoyiKFh1DclJ5B0mFxbVIuVwCdCOup5sXR\nrFz1ziVMd1tAmOZXaXpg7GTklONcJZ0DjFR4KMnrwOBE2l8II4B/TmwbkSN/FQoP9zg1Q9JQSa/H\nuH5JmIaYmiLaE9iQo9hs117Ga8DMXiB0shbHul5Nz5Plc2r/fwFDgTsU1ozNB46pxrHeSLie742j\nt3PT0tsCq8ysfr7CcfXG1xaUJm+30uTtVpq83XZeq1evZv/999/WiVq9ejV77bUXe+6Z9xa2buR6\n5vuu8AKuAW4vdhwN+QV8B/h/xY6jDo7rYODOYsdRz8c8DPhTnjwG5q9tL8w555yrS5n+X1P3/z8u\n7P9v3bt3t9tuu80OOugga9eunf3whz+0LVu2mJlZeXm5denSxczMzj33XGvUqJG1aNHCdt99d/vF\nL35RpayPPvrITj31VGvTpo21a9fO+vfvX6meO++803r16mVt2rSx4cOHb6vnwQcftH79+lUqS5It\nX77cRo8ebc2aNbOmTZva7rvvbr/97W/ta1/7mjVu3Nh23313u+CCC2zVqlXWqFEj27p1q5mZbd68\n2UaOHGkdO3a0Ll262A033GAVFRXVap/E9ir3UvU9gtUQTQL6KvFDw64yM3vSzH5d7Dhqm5ktNrOr\nix1HfZE0Hrga+H0Buf0VX+3bd8t/uhqAUltb4AJvt9Lk7VaavN1q7o9//CPTp09n+fLlLFu2jFtu\n2f6ToqlRo4cffpiuXbsydepUPv74Y66+uuot1vjx49lnn31Yv349H3zwAePGjauU/vjjj/PMM8+w\ncuVKFi5cyIMPPlilnvTPY8aM4brrrmP48OF8/PHHXHzxxUybNo1OnTrx8ccf88ADD1SJ4/zzz6dZ\ns2asWLGC+fPnM336dH7/+wJujwpU308RbHAsrFP5VrHjcK6uWfih4ULz1mUozjnnnCshP/nJT+jU\nqRMA119/PZdddhk33VRlJQuQ+x6iadOmvPfee6xcuZKePXvSt2/fSumjRo2iffuw4ue0005jwYLs\nD8Ku6b3K+++/z7Rp09i8eTO77bYbzZs35/LLL+f+++/noosuqlGZ6XwEyznndgK+tqA0ebuVJm+3\n0uTtVnNdunTZ9r5bt27bHhxRXT/96U/p2bMnJ510Et/4xje44447KqWnOlcALVq04NNPP61ZwDm8\n/fbb/Otf/6Jjx460a9eOtm3bcskll/DRRx/VWh27/AiWc84555xzLrt33nln2/vVq1dvG81Kl+9J\nfS1btuTOO+/kzjvvZMmSJRx33HEceeSRHHfccXn3+/zzz7d9XrduXY2fCrjPPvvQvHlz1q9fX2dP\nFvQRLOec2wn42oLS5O1WmrzdSpO3W83dc889rF27lg0bNjBu3DiGDx+eMV+HDh1YsWJF1nKefPJJ\nli8PvyKz++6706RJExo3bpy3/t69e7N48WIWLVrEli1bsv7Aby6pKYUdOnTgpJNO4oorruCTTz7B\nzFixYgXPPfdctcvMxjtYzjnnnHPONTDhIUvFf4iTJM4+++xt0/r2228/rr/++ox5r732Wm6++Wba\ntWvHXXfdVSX9zTff5IQTTmD33Xenb9++/PjHP6Z///7b6slmv/3242c/+xmDBg1i//3351vfqv7j\nE5LlP/zww3z55ZccdNBBtGvXjjPOOIN169bl2LuadflidudckiTzfxecc865+iPJHzDVgGVrn7i9\nSs/QR7Ccc84555xzrpZ4B8s5V4Ukf+0Crw5dOhT7Utvl+ZqQ0uTtVpq83Vx98acIOueqGlPsAFy1\nrQR6VG+X98e8XyehOOecc7syX4PlnKtEknkHaxcxxn9U2jnnGgJfg9WwNdg1WJK6SXotS9osSWX1\nFUta3V0lzZM0LbFtZTFiyUbSAEkTC8iXM25Jn8S/HSU9Ft+fL+lXNSmvwDpHS7oyXznVkSxT0kRJ\n/fPkHyBpU2zneZJuKKCOWZK65kmv1jUraaikJZKejZ//JGmBpFHxOE7Ps3/eY435/kvSm7HsQxPb\nx0taLGlAdeJ2zjnnnHOFq+81WA2xaz4EeMbMTklsa4hxFhJTvjwGYGbvmdmwAvarjTobiufMrCy+\nbilSDCOBC81skKQOwOFmdqiZ3V1bFUg6BehpZvsBPwJ+k0ozs6uAm4Af1lZ9rgFpUF8LuUL5mpDS\n5O1WmrzdXH2p7w5WU0mPxG/xH5PUPD2DpLMkLYqv2+O2RvHb+0WSFkoaFbf3lDQ9flP/iqRqrkAA\noA3wQdq2DxPxnBfrnC/pobhtoqS7Jb0g6a3UyIOklpJmxFgWShoct3eTtDTut0zSo5JOjPsvk3R4\nzNdC0gRJcyS9Kum0GMaXwOYCjuXDWM7YGO88SWskTUgdTiKe5Ghi1zgis0zSzzKdh3x1ZjtXSZL2\nlTRN0suSZkvaX1JrSasSeVpIeltS40z5M9S/iXB+8qnuT3WvB7Zmu/aiYZJekvSGpL4x/kojgpKm\nSOov6UagHzBB0s+Bp4HOsY36VQpUKpNUHo97mqT21TjW7wIPA5jZS8Aeif0B1hGueeecc845Vwfq\n+yEXBwAXmNmceNN/KbDtV8gkdQRuB/oQbianx07KGqCzmfWK+VrHXR4FxpnZZEnNqFmHsTFQkdxg\nZkfFeg4CrgOOMbONkpI3ph3MrK+kA4HJwCTgC2CImX0qaU9gTkwD6Al838yWSHoFGB73HxzrOB24\nHnjWzEZK2gOYK2mGmb0IvBhjOgz4kZldnH4gqbjNbDQwOpbxHJC64U+ONiXfHwEcHON/WdJUM5uX\nKi+XAs9Vyv0x9uWSjgTui6M58yUNMLPZwKnAU2a2VVKV/MCgtPqvSL2XNBZ42cymZqj7GEkLgLXA\nNWa2JM9xDY1llpH52gNobGZHxVGjMcCJqd0zlHezpOOBK81svqR7gClmVhbLHRn/NiG012AzWy9p\nGDAOGFngsXYG3kl8Xhu3pZ5mUEG45nOblXjfnWo/PMEVgbdRSRo4cGCxQ3A14O1Wmrzd3I4qLy8v\naCS0vkew3jazOfH9I4Rv9JOOAGaZ2QYzqyB0oPoDK4AecdTo28AnkloBncxsMoCZfWlmX1QnGEkC\nehM6cJkcDzxuZhtjHZsSaU/EbUuBr6eKBG6TtBCYAXSSlEpbmbipXxzTAV4j3MICnARcK2k+UA40\nAyqtAzKzVzN1rrJ4BLjLzBbkyTfdzDbF8zeJqu1SiFznCkktgWOBx+Px/RZIjaw8BpwZ3w8H/pIn\nf0ZmNjpL5+pVoKuZHQr8mth2Bapy7SXSJiXKL+zn0POPpB0AHEL4cmE+odPdKT1TjmPNZy2wv6Td\ncuY6LvHyG3fnnHOu3nXo0mGX+amO4447jgceeKDG+19wwQW0a9eOo48+GoD77ruPDh060Lp1azZs\n2ECjRo1YsWLFDsc5cOBAxowZs+2VTX2PYKV/s59p/U6VG1Az2ySpN/Bt4BLgDODyTHkrFSRdClwU\n6/l3M1uXSGtEuHneAjxZjWNI2ZIh5hHAXkAfM6tQeABE8wz5KxKfK9jeDiKMcr1Zg3gqkTSG0KGt\nMlUvg0LaZUc1AjamRmzSTAZuldQWKANmAq1y5K8WM/s08X6apHsltTOzDQXsm+nauzAmp9pwK9vb\n8Csqf3FRZRpsHgJeN7O+1dwvZS2wT+Jzl7gNADNbIWkpsFrSIDNbXMN6XENTg8e0u+IrLy/3b9VL\nkLdbaSq1dnt/7ft1+rMpO8tPdTz//PM8++yzvPvuuzRv3pyvvvqKq666irlz53LIIYcA4Wl/9am+\nR7C6SUpNOzsb+Hta+lygv6R2khoDZwGz43S7xmb2V+AGoCzeNL8j6bsAkppJ+lqyMDO718z6xAcb\nrEtLqzCz7sArbB89STcTOENSu1hH2yz5Uq22B/BB7FwdR+VRjUJa9mngsm07JJ4AVx0Ka7dOAEal\nJ2XZ5URJbeL5GwK8kKHMpXmqzXmuzOwTYKWkoYkye8W0zwjtcDcw1YKs+asruQYpTjVUqnOlsGau\nY459q1x72bLGv6uAQxXsAxyZK7QM25YBe0s6OtbfJE6/LNRk4Ly479HAJjPb9i9oPIc9CKO/3rly\nzjnnXElbtWoV3bt3p3nz8J32unXr2LJlCwceeOC2PPX9CPz67mC9AfxY0hLCQvvUE85ST7dbB1xL\nmB43n7DGZAphDUl5nDL1h5gHwo3kZXFK3gvkmUKWxT+BdpkS4pS+WwmdvPnA+GS8yazx76PAETGe\nc4ClGfJk2j/lZsKDQBYpPITipvQMkg6La5NyuYIwrexlhYcojMlT71zCdLcFhGl+89Lq3DNPfbnO\nVdI5wEiFh5K8DgxOpP2FMAL458S2ETnyV6HwcI9TMyQNlfR6jOuXhGmIqSmiPYFcI1nZrr2M14CZ\nvUDoZC2Odb2anifL59T+/wKGAncorBmbDxxT6LGa2d8IHdO3CNMqL03L0hZYFafgup2Jj16VpFL6\nNt1t5+1WmrzdaqZHjx6MHz+e3r1707ZtW8466yy+/HL7M7d+97vfsd9++7HXXnsxZMgQ3nvvvYzl\nbNmyhXPPPZe99tqLtm3bctRRR/Hhh9ufp7Zq1Sr69etH69atOfnkk9mwIdyezZ49m3322adSWT16\n9GDmzJk88MADXHTRRbz44ou0bt2aESNG8G//9m8AtG3blhNOOKFKHF9++SVXX3013bp1o2PHjlx6\n6aVs2bKlSr4dscv/0LCka4A9zezavJl3UZK+A/Qws18XO5baJOlgwkNXri52LPVF4aEZ3zOzs3Lk\n8R8a3lWM8R8ads65hkAZfshWUp1OESz0/wE9evSgffv2/O///i+77bYbxx57LJdffjkXX3wxM2fO\n5Mwzz2TGjBkcdNBBXHXVVSxcuJDZs2dXKef+++/nySef5LHHHqNZs2YsWLCA/fbbj1atWnHcccex\nZs0annrqKbp06cLJJ5/MMcccw7hx45g9ezbnnnsub7/9dqWYJkyYwPHHH89DDz3EhAkTeO655wBY\nvXo1++67L1999dW2qYGNGjXirbfeYt999+WKK65g5cqVPPTQQzRp0oSzzz6bQw45hFtvvTXrOcjU\nPontVWYk1fcarIZoEvCgpGlpv4XlIjOryRq1Bi9OkduVOlfjgW8B/5k385i6jsY1BO0712TQ39Wm\nUlsT4gJvt9Lk7VZzo0aNon378P+M0047jQULwvPT/vjHPzJy5Eh69+4NwG233Ubbtm15++236dq1\n0nPaaNq0KevXr+ef//wn3/zmN+nTp0+l9AsuuICePXsCMGzYMKZMmbJDMZtZxrVXv/vd73jttdfY\nY489ALj22msZMWJEzg5Wde3yHSwzW0646XRupxZ/aLjQvHUZiqsDfuPgnHOurqQ6VwAtWrTYNg3w\n3Xff5bDDDtuW1rJlS/bcc0/Wrl1bpYN13nnnsWbNGoYPH87mzZsZMWIE48aNo3Hj8OsxHTp0qFTH\np59+Sm1wALmwAAAgAElEQVT78MMP+fzzzyvFXFFRUev3PfW9Bss551wd8M5VafJ2K03ebqXJ2632\nderUidWrV2/7/Nlnn7F+/Xo6d+5cJW/jxo258cYbWbx4Mf/4xz+YOnUqDz/8cN46WrZsyeeff77t\n89atWyut3aqOvfbaixYtWrB48WI2bNjAhg0b2LRpE5s3b65Redl4B8s555xzzjlXbWeddRYTJ05k\n0aJFbNmyheuuu46jjz66yugVhJkWr7/+OhUVFbRq1YqmTZtuG73KZf/99+eLL75g2rRpfPXVV9xy\nyy2VHrKRSbYRKUlcdNFFXH755ds6aWvXruWZZ54p4GgLt8tPEXTOuZ2BTxEsTd5upcnbrTSVWru1\n79y+Tn+rqtB1uLl+Q2rQoEHcfPPNnH766WzatIljjz2WP//5zxnzrlu3jksuuYS1a9fSqlUrhg8f\nzjnnnJO3jtatW3PvvfcycuRIKioq+OlPf0qXLl2qFXPy8x133MHYsWM5+uijt422/cd//AcnnXRS\nzjKrY5d/iqBzrjJJ5v8ulJ5Su3FwgbdbafJ2K00Nud2yPaXONQzVfYqgd7Ccc5V4B8s555yrX97B\natiq28HyNVjOOeecc845V0u8g+Wcq0KSv3bBV4cO3Yt96e1yysvLix2CqwFvt9Lk7ebqiz/kwjmX\ngU9TKD3lwMAdKuH997MvMnbOOedcYXwNlnOuEknmHaxdla8BcM65YlCWNT6uYcjWPnF78dZgSeom\n6bUsabMkldVXLGl1d5U0T9K0xLaVxYglG0kDJE0sIF/OuCV9Ev92lPRYfH++pF/VpLwC6xwt6cp8\n5VRHskxJEyX1z5N/sKSFkuZLekXS8QXUMUtS1R9xqJxerWtW0lBJSyQ9Gz//SdICSaPicZyeZ/9C\njvXseKwLJT0vqVcibbykxZIGVCdu55xzztWtbt26FX2auL+yv7p161at9qzvNVgNsWs+BHjGzE5J\nbGuIcRYSU748BmBm75nZsAL2q406G4IZZtbbzPoAFwD3FymOkcCFZjZIUgfgcDM71MzursU6VgD9\nzaw3cAuJYzWzq4CbgB/WYn2uwSgvdgCuBnxNSGnyditNDbndVq1ahZn5K8Nr1qxZRY9h1apV1WrP\n+u5gNZX0SPwW/zFJzdMzSDpL0qL4uj1uaxS/vV8Uv5kfFbf3lDRdYRTgFUk9ahBTG+CDtG0fJuI5\nT9tHPx6K2yZKulvSC5LeUhx5kNRS0owYy0JJg+P2bpKWxv2WSXpU0olx/2WSDo/5WkiaIGmOpFcl\nnRbD+BLYXMCxfBjLGRvjnSdpjaQJqcNJxJMcTeyqMCKzTNLPMp2HfHVmO1dJkvaVNE3Sy5JmS9pf\nUmtJqxJ5Wkh6W1LjTPkz1L+JcH6yMrPPEx9bAR8VcFzrga3Zrr1omKSXJL0hqW+Mv9KIoKQpkvpL\nuhHoB0yQ9HPgaaBzbKN+aeepTFJ5PO5pklK/BFjIsc4xs9S1MgfonJZlHeGad84555xzdaC+H3Jx\nAHCBmc2JN/2XAnelEiV1BG4H+hBuJqfHTsoaoLOZ9Yr5WsddHgXGmdlkSc2oWYexMVCR3GBmR8V6\nDgKuA44xs42SkjemHcysr6QDgcnAJOALYIiZfSppT8IN7uSYvyfwfTNbIukVYHjcf3Cs43TgeuBZ\nMxspaQ9grqQZZvYi8GKM6TDgR2Z2cfqBpOI2s9HA6FjGc0Dqhj852pR8fwRwcIz/ZUlTzWxeqrxc\nCjxXKffH2JdLOhK4z8JoznxJA8xsNnAq8JSZbZVUJT8wKK3+K1LvJY0FXjazqekVSxoC3AZ0AL5d\nwHENjfuVkfnaA2hsZkdJOgUYA5yY2j1DeTcrTE280szmS7oHmGJmZbHckfFvE0J7DTaz9ZKGAeOA\nkYUea8KFwLS0bRWEaz6PMYn3A9nRhye4+jCw2AG4GmioP3rqcvN2K03ebqWpIbVbeXl5QSOh9d3B\netvM5sT3jwA/IdHBItzozzKzDQCSHgX6E6Y69ZB0N/A34BlJrYBOZjYZwMxyfrOfiSQBvWMsmRwP\nPG5mG2MdmxJpT8RtSyV9PVUkcJvCOpkKoFMibaWZLYnvFwMz4vvXgO7x/UnAaZKuiZ+bAV2BZalK\nzexVoErnKotHgLvMbEGefNNTxyZpEmGkZV6BdaTkOldIagkcCzwezztA0/j3MeBMYDYwHLgnT/6M\nYscyW9oTwBNxtOgPhM5+IVaQdu0l0ibFv68ChU7OzfeYtgOAQwhfLojwpcG76ZlyHSuApOMI0yH7\npSWtBfaXtJuZbclewpg8YTrnnHPO7VoGDhxYqcM3duzYjPmKvQYr0/qdKjeg8Wa9N2GRwSXA77Ll\nrVSQdGliqlyHtLRGwErgQODJgqKvLHlzmopjBLAX0MfCep8PgOYZ8lckPlewvaMrwihXn/jqYWbL\nqAFJYwgd2ipT9TIopF12VCNgo5mVJY7vkJg2GThZUlugDJiZJ3+NmdnzQJM4wlhI/mzXHmxvw61s\nb8OvqPzfVZVpsHkIeD1x3L2t8vrA/AWEB1vcTxgF25hMM7MVwFJgtaSDqxmba9DKix2Aq4GGvCbE\nZeftVpq83UpTKbZbfXewuklKTTs7G/h7WvpcoL+kdpIaA2cBs+PNcGMz+ytwA1BmZp8C70j6LoCk\nZpK+lizMzO6NN6llZrYuLa3CzLoDrxBGTzKZCZwhqV2so22WfKkO1h7AB2ZWEUcQumXIk8vTwGXb\ndpAOLWCfqsGEtVsnAKPSk7LscqKkNvH8DQFeyFDm0jzV5jxXZvYJsFLS0ESZvWLaZ4R2uBuYakHW\n/NUlqWfifVmsc338PCNOTc22b5VrL1vW+HcVcKiCfYAjc4WWYdsyYG9JR8f6m8TplwVRePLh/wDn\nmtnyDOm9gB6E0d/FhZbrnHPOOecKU98drDeAH0taQlho/5u4PfV0u3XAtYSvYucT1phMISzUL5c0\nnzC969q433nAZZIWEjoFqYcBVMc/gXaZEuKUvlsJnbz5wPhkvMms8e+jwBExnnMIIwXpeTLtn3Iz\n4UEgixQeQnFTegZJh8W1SblcAXQirKeaF0ezctU7lzDdbQFhml+l6YGFjPbkOFdJ5wAjFR5K8jow\nOJH2F8II4J8T20bkyF+FwsM9Ts2Q9H1Jr0uaR+jEDY/5RVgbtyFHsdmuvYzXgJm9QOhkLQZ+SZg+\nSK590vb/FzAUuEPSAsJ/B8dU41hvJFzP98bR27lp6W2BVWZWUXVXV9oGFjsAVwMNaW2BK5y3W2ny\nditNpdhuu/wPDcf1Tnua2bV5M++iJH0H6GFmvy52LLUpTpG7wMyuLnYs9SU+NON7ZnZWjjz+Q8O7\nLP+hS+ecc65QKvYPDTdgk4C+SvzQsKvMzJ7c2TpXAGa2eBfrXI0HrgZ+X+xYXF0oL3YArgZKcW2B\n83YrVd5upakU262+nyLY4MR1Kt8qdhzO1TULPzRcoEKWDLqdTfv21fuleuecc85VtctPEXTOVSbJ\n/N8F55xzzrncfIqgc84555xzztUx72A559xOoBTnqDtvt1Ll7VaavN1KUym2m3ewnHPOOeecc66W\n+Bos51wlvgbLOeeccy4/X4PlnHPOOeecc3XMO1jOObcTKMU56s7brVR5u5Umb7fSVIrttsv/DpZz\nrirJfwfLuZT2nduzbs26YofhnHOuRPgaLOdcJZKMMcWOwrkGZAz4/yudc86l8zVYzjnnnHPOOVfH\n6q2DJambpNeypM2SVFZfsaTV3VXSPEnTEttWFiOWbCQNkDSxgHw545b0SfzbUdJj8f35kn5Vk/IK\nrHO0pCvzlVMdyTIlTZTUP0/+wZIWSpov6RVJxxdQxyxJXfOkV+ualTRU0hJJz8bPf5K0QNKoeByn\n59k/77HGfP8l6c1Y9qGJ7eMlLZY0oDpxuxLRoP7VcoUqxbUFztutVHm7laZSbLf6XoPVEOdYDAGe\nMbNrE9saYpyFxJQvjwGY2XvAsAL2q406G4IZZjYZQNI3gb8C3yhCHCOBC83sH5I6AIeb2X4xrrwd\n6EJIOgXoaWb7SToK+A1wNICZXSVpLvBDYHZt1Oecc8455yqr7ymCTSU9Er/Ff0xS8/QMks6StCi+\nbo/bGsVv7xfFkYhRcXtPSdPjN/WvSOpRg5jaAB+kbfswEc95idGPh+K2iZLulvSCpLdSIw+SWkqa\nEWNZKGlw3N5N0tK43zJJj0o6Me6/TNLhMV8LSRMkzZH0qqTTYhhfApsLOJYPYzljY7zzJK2RNCF1\nOIl4kqOJXeOIzDJJP8t0HvLVme1cJUnaV9I0SS9Lmi1pf0mtJa1K5Gkh6W1JjTPlz1D/JsL5ycrM\nPk98bAV8VMBxrQe2Zrv2omGSXpL0hqS+Mf5KI4KSpkjqL+lGoB8wQdLPgaeBzrGN+qWdpzJJ5fG4\np0lqX+ixAt8FHo7H/RKwR2J/gHWEa97tbGryr58ruoEDBxY7BFcD3m6lydutNJViu9X3CNYBwAVm\nNife9F8K3JVKlNQRuB3oQ7iZnB47KWuAzmbWK+ZrHXd5FBhnZpMlNaNmHcbGQEVyg5kdFes5CLgO\nOMbMNkpK3ph2MLO+kg4EJgOTgC+AIWb2qaQ9gTkxDaAn8H0zWyLpFWB43H9wrON04HrgWTMbKWkP\nYK6kGWb2IvBijOkw4EdmdnH6gaTiNrPRwOhYxnNA6oY/OdqUfH8EcHCM/2VJU81sXqq8XAo8Vyn3\nx9iXSzoSuM/MBsUO2QAzmw2cCjxlZlslVckPDEqr/4rUe0ljgZfNbGp6xZKGALcBHYBvF3BcQ+N+\nZWS+9gAam9lRCqNGY4ATU7tnKO9mhamJV5rZfEn3AFPMrCyWOzL+bUJor8Fmtl7SMGAcMLLAY+0M\nvJP4vDZuez9+riBc87nNSrzvjt+8O+ecc26XV15eXtCUxfruYL1tZnPi+0eAn5DoYBFu9GeZ2QYA\nSY8C/YFbgB6S7gb+BjwjqRXQKTX1y8zyfbNfhSQBvWMsmRwPPG5mG2MdmxJpT8RtSyV9PVUkcJvC\nOpkKoFMibaWZLYnvFwMz4vvXCLewACcBp0m6Jn5uBnQFlqUqNbNXgSqdqyweAe4yswV58k1PHZuk\nSYSRlnkF1pGS61whqSVwLPB4PO8ATePfx4AzCdPWhgP35MmfUexYZkt7Angijhb9gdDZL8QK0q69\nRNqk+PdVoFuB5eV7/vkBwCGELxdE+NLg3fRMuY41j7XA/pJ2M7MtWXMdV8PSXfGsxDvCJai8vLwk\nv53d1Xm7lSZvt9LUkNpt4MCBlWIZO3ZsxnzFXoOVaf1OlRtQM9skqTdh5OES4Azg8kx5KxUkXQpc\nFOv5dzNbl0hrRLh53gI8WY1jSEnenKbiGAHsBfQxswqFB0A0z5C/IvG5gu3tIMIo15s1iKcSSWMI\nHdoqU/UyKKRddlQjYGNqxCbNZOBWSW2BMmAmYSpftvw1ZmbPS2oiaU8zW19A/kzX3oUxOdWGW9ne\nhl9ReSS1yjTYPAS8bmZ9q7lfylpgn8TnLnEbAGa2QtJSYLWkQWa2uIb1OOecc865DOp7DVY3hYX3\nAGcDf09Lnwv0l9ROUmPgLGB2nG7X2Mz+CtwAlJnZp8A7kr4LIKmZpK8lCzOze82sj5mVJTtXMa3C\nzLoDrxBGTzKZCZwhqV2so22WfKkO1h7AB7FzdRyVRzUK+eXWp4HLtu2QeAJcdSis3ToBGJWelGWX\nEyW1iedvCPBChjKX5qk257kys0+AlZKGJsrsFdM+I7TD3cBUC7Lmry5JPRPvy2Kd6+PnGXFqarZ9\nq1x72bLGv6uAQxXsAxyZK7QM25YBe0s6OtbfJE6/LNRk4Ly479HAJjNLTQ9MncMehNFf71ztTHz0\nqiQ1lG9lXfV4u5Umb7fSVIrtVt8drDeAH0taQlho/5u4PfV0u3XAtUA5MJ+wxmQKYQ1JuaT5hOld\nqSf+nQdcJmkhoVOQXMxfqH8C7TIlxCl9txI6efOB8cl4k1nj30eBI2I85wBLM+TJtH/KzYQHgSxS\neAjFTekZJB0W1yblcgXQibCeal4czcpV71zCdLcFhGl+laYHxk5GTjnOVdI5wEiFh5K8DgxOpP2F\nMAL458S2ETnyV6HwcI9TMyR9X9LrkuYROnHDY34R1sZtyFFstmsv4zVgZi8QOlmLgV8Spg+Sa5+0\n/f8FDAXukLSA8N/BMYUeq5n9jdAxfQv4LWGdY1JbYJWZVaTv65xzzjnndpx29V+nj+ud9kx7TLtL\nkPQdoIeZ/brYsdQmSQcTHrpydbFjqS/xoRnfM7OzcuQxxtRfTK6W+BqsujMG6ur/lQ1pbYErnLdb\nafJ2K00Nud0kYWZVZiTV9xqshmgS8KCkaWZ2SrGDaYjMrCZr1Bq8OEVuV+pcjQe+Bfxn3sxj6joa\n50pH+841mRzhnHNuV7XLj2A55yqTZP7vgnPOOedcbtlGsOp7DZZzzjnnnHPO7bS8g+WcczuBQn74\n0DU83m6lydutNHm7laZSbDfvYDnnnHPOOedcLfE1WM65SnwNlnPOOedcfr4GyznnnHPOOefqmHew\nnHNuJ1CKc9Sdt1up8nYrTd5upakU281/B8s5V4VUZbTb7eTat+/GunWrih2Gc845V/J8DZZzrhJJ\nBv7vwq5H+P8PnHPOucL5GiznnHPOOeecq2P11sGS1E3Sa1nSZkkqq69Y0uruKmmepGmJbSuLEUs2\nkgZImlhAvpxxS/ok/u0o6bH4/nxJv6pJeQXWOVrSlfnKqY5kmZImSuqfJ/8Bkv4h6YtCY4nXZNc8\n6dW6ZiUNlbRE0rPx858kLZA0Kh7H6Xn2L+RYz5a0ML6el9QrkTZe0mJJA6oTtysV5cUOwNVAKa4t\ncN5upcrbrTSVYrvV9xqshjj/ZAjwjJldm9jWEOMsJKZ8eQzAzN4DhhWwX23U2RCsB35CaOtiGglc\naGb/kNQBONzM9oPQeaqlOlYA/c1ss6STgfuBowHM7CpJc4EfArNrqT7nnHPOOZdQ31MEm0p6JH6L\n/5ik5ukZJJ0laVF83R63NYrf3i+K38yPitt7SpoeRwFekdSjBjG1AT5I2/ZhIp7zYp3zJT0Ut02U\ndLekFyS9lRp5kNRS0owYy0JJg+P2bpKWxv2WSXpU0olx/2WSDo/5WkiaIGmOpFclnRbD+BLYXMCx\nfBjLGRvjnSdpjaQJqcNJxJMcTewaR2SWSfpZpvOQr85s5ypJ0r6Spkl6WdJsSftLai1pVSJPC0lv\nS2qcKX+G+jcRzk9WZvaRmb0KfFXA8aSsB7Zmu/aiYZJekvSGpL4x/kojgpKmSOov6UagHzBB0s+B\np4HOsY36pZ2nMknl8binSWpfjWOdY2apa2UO0DktyzrCNe92OgOLHYCrgYEDBxY7BFcD3m6lydut\nNJViu9X3CNYBwAVmNife9F8K3JVKlNQRuB3oQ7iZnB47KWuAzmbWK+ZrHXd5FBhnZpMlNaNmHcbG\nQEVyg5kdFes5CLgOOMbMNkpK3ph2MLO+kg4EJgOTgC+AIWb2qaQ9CTe4k2P+nsD3zWyJpFeA4XH/\nwbGO04HrgWfNbKSkPYC5kmaY2YvAizGmw4AfmdnF6QeSitvMRgOjYxnPAakb/uRoU/L9EcDBMf6X\nJU01s3mp8nIp8Fyl3B9jXy7pSOA+MxsUO2QDzGw2cCrwlJltlVQlPzAorf4rUu8ljQVeNrOp+eIu\n4LiGxjLLyHztATQ2s6MknQKMAU5M7Z6hvJslHQ9caWbzJd0DTDGzsljuyPi3CaG9BpvZeknDgHHA\nyBoc64XAtLRtFYRrPo8xifcD8Zt355xzzu3qysvLC5qyWN8drLfNbE58/whh2tZdifQjgFlmtgFA\n0qNAf+AWoIeku4G/Ac9IagV0MrPJAGaW85v9TCQJ6B1jyeR44HEz2xjr2JRIeyJuWyrp66kigdsU\n1slUAJ0SaSvNbEl8vxiYEd+/BnSP708CTpN0TfzcDOgKLEtVGkdiqnSusngEuMvMFuTJNz11bJIm\nEUZa5hVYR0quc4WklsCxwOPxvAM0jX8fA84kTFsbDtyTJ39GsWNZ21aQdu0l0ibFv68C3QosL9/z\nzw8ADiF8uSDClwbvpmfKd6ySjgMuILRl0lpgf0m7mdmW7CWMyROma3jK8Y5w6SkvLy/Jb2d3dd5u\npcnbrTQ1pHYbOHBgpVjGjh2bMV+x12BlWr9T5QbUzDZJ6g18G7gEOAO4PFPeSgVJlwIXxXr+3czW\nJdIaEW6etwBPVuMYUpI3p6k4RgB7AX3MrELhARDNM+SvSHyuYHs7iDDK9WYN4qlE0hhCh7bKVL0M\nCmmXHdUI2JgasUkzGbhVUlugDJgJtMqRv95kufYujMmpNtzK9jb8isojqVWmweYh4HUz61uziEHh\nwRb3AyenOrwpZrZC0lJgtaRBZra4pvU455xzzrmq6nsNVjdJqWlnZwN/T0ufC/SX1E5SY+AsYHac\nbtfYzP4K3ACUmdmnwDuSvgsgqZmkryULM7N7zayPmZUlO1cxrcLMugOvEEZPMpkJnCGpXayjbZZ8\nqQ7WHsAHsXN1HJVHNQr55dangcu27SAdWsA+VYMJa7dOAEalJ2XZ5URJbeL5GwK8kKHMpXmqzXmu\nzOwTYKWkoYkye8W0zwjtcDcw1YKs+XdQpXOgsGauY9bMGa69POWuAg5VsA9wZKGxRMuAvSUdHetv\nEqdfFkThyYf/A5xrZsszpPcCehBGf71ztVMZWOwAXA00lG9lXfV4u5Umb7fSVIrtVt8drDeAH0ta\nQlho/5u4PfV0u3XAtYS5LvMJa0ymEBbql0uaD/wh5gE4D7hM0kJCpyD1MIDq+CfQLlNCnNJ3K6GT\nNx8Yn4w3mTX+fRQ4IsZzDrA0Q55M+6fcTHgQyCKFh1DclJ5B0mFxbVIuVwCdCOup5sXRrFz1ziVM\nd1tAmOZXaXpg7GTklONcJZ0DjFR4KMnrwOBE2l8II4B/TmwbkSN/FQoP9zg1w/b2kt4hnJfrFR6i\n0SpOwesJbMhRbLZrL+M1YGYvEDpZi4FfEqYPkmuftP3/BQwF7pC0gPDfwTGFHitwI+F6vjeubZub\nlt4WWGVmFVV3dc4555xzO0pmpfCU7boT1zvtmfaYdpcg6TtADzP7dbFjqU2SDiY8dOXqYsdSX+JD\nM75nZmflyGOl8fR9V1k5OzaKJXb1/x8UQ0NaW+AK5+1WmrzdSlNDbjdJmFmVGUn1vQarIZoEPChp\nmpmdUuxgGiIzq8katQYvTpHblTpX44FvAf9ZQO66Dsc1MO3bF/qcFuecc87lssuPYDnnKpNk/u+C\nc84551xu2Uaw6nsNlnPOOeecc87ttLyD5ZxzO4FCfvjQNTzebqXJ2600ebuVplJsN+9gOeecc845\n51wt8TVYzrlKfA2Wc84551x+vgbLOeecc8455+qYd7Ccc24nUIpz1J23W6nyditN3m6lqRTbzX8H\nyzlXheS/g+V2Xu07t2fdmnXFDsM559xOytdgOecqkWSMKXYUztWhMeD/73POObejfA2Wc84555xz\nztWxeutgSeom6bUsabMkldVXLGl1d5U0T9K0xLaVxYglG0kDJE0sIF/OuCV9Ev92lPRYfH++pF/V\npLwC6xwt6cp85VRHskxJEyX1z5P/AEn/kPRFobHEa7JrnvRqXbOShkpaIunZ+PlPkhZIGhWP4/Q8\n++c91pjvvyS9Gcs+NLF9vKTFkgZUJ25XIhrUv1quUKW4tsB5u5Uqb7fSVIrtVt9rsBrinIwhwDNm\ndm1iW0OMs5CY8uUxADN7DxhWwH61UWdDsB74CaGti2kkcKGZ/UNSB+BwM9sPQuepNiqQdArQ08z2\nk3QU8BvgaAAzu0rSXOCHwOzaqM8555xzzlVW31MEm0p6JH6L/5ik5ukZJJ0laVF83R63NYrf3i+S\ntFDSqLi9p6Tp8Zv6VyT1qEFMbYAP0rZ9mIjnvFjnfEkPxW0TJd0t6QVJb6VGHiS1lDQjxrJQ0uC4\nvZukpXG/ZZIelXRi3H+ZpMNjvhaSJkiaI+lVSafFML4ENhdwLB/GcsbGeOdJWiNpQupwEvEkRxO7\nxhGZZZJ+luk85Ksz27lKkrSvpGmSXpY0W9L+klpLWpXI00LS25IaZ8qfof5NhPOTlZl9ZGavAl8V\ncDwp64Gt2a69aJiklyS9IalvjL/SiKCkKZL6S7oR6AdMkPRz4Gmgc2yjfmnnqUxSeTzuaZLaF3qs\nwHeBh+NxvwTskdgfYB3hmnc7m5r86+eKbuDAgcUOwdWAt1tp8nYrTaXYbvU9gnUAcIGZzYk3/ZcC\nd6USJXUEbgf6EG4mp8dOyhqgs5n1ivlax10eBcaZ2WRJzahZh7ExUJHcYGZHxXoOAq4DjjGzjZKS\nN6YdzKyvpAOBycAk4AtgiJl9KmlPYE5MA+gJfN/Mlkh6BRge9x8c6zgduB541sxGStoDmCtphpm9\nCLwYYzoM+JGZXZx+IKm4zWw0MDqW8RyQuuFPjjYl3x8BHBzjf1nSVDOblyovlwLPVcr9Mfblko4E\n7jOzQbFDNsDMZgOnAk+Z2VZJVfIDg9LqvyL1XtJY4GUzm5ov7gKOa2gss4zM1x5AYzM7SmHUaAxw\nYmr3DOXdLOl44Eozmy/pHmCKmZXFckfGv00I7TXYzNZLGgaMA0YWeKydgXcSn9fGbe/HzxWEa/7/\nZ+/to6wsrnz/zxeUURQVfOHFjK1haQw3RsWoMTra+ouYmGi8KMaXDC7jaLJwqZiYtbyZ/AIoEaMx\nUWd+jtE4JAquQe4lGdQoAul2FEUQmhcFUW/QiDet3igqs4iO9v79UfvA06fPWx+b7n6692ets049\nVbuqdtV+TvezT9WuU5mmTPpA4uE9CIIgCIJ+T3Nzc01bFrvbwfqTmS319CzStq2fZ8qPBprM7G0A\nSbOBE4HpwEGSbgN+DzwmaXdglJnNBzCzat/sd0CSgMNdl1KcAsw1s3e8j82Zst953npJ+xWaBGYo\nxTnXmyQAACAASURBVMm0AaMyZRvNbJ2nnwcWeXot6REWYBxwhqQf+PUg4ABgQ6FTX4np4FyVYRbw\nczNbVUVuYWFskuaRVlpW1thHgUpzhaTdgC8Bc33eAXb29weAb5K2rZ0H/H9V5EvijmVX80eK7r1M\n2Tx/XwE01NhetfPPPwN8jvTlgkhfGvyfYqFPMNbXgUMk/Y2ZfVBW6uQ6Ww96jo2EI5xDmpubc/nt\nbH8n7JZPwm75pDfZrbGxsZ0u06ZNKynX0zFYpeJ3OjyAmtlmSYcDpwHfBSYAk0vJtmtImgRc6v2c\nbmatmbIBpIfnD4CHOzGGAtmH04IeFwL7AEeaWZvSARC7lJBvy1y3sd0OIq1yvVSHPu2QNJXk0HbY\nqleCWuzySRkAvFNYsSliPvATSUOBscAfgN0ryHcbZe69f/Digg0/ZrsNP6L9SmqHbbBVEPCcmR1f\nn8a8Dvxt5vpTngeAmf1R0nrgVUn/j5k9X2c/QRAEQRAEQQm6OwarQSnwHuAC4Imi8mXAiZKGSRoI\nnA887tvtBprZb4EfAWPNbAvwmqRvAEgaJGnXbGNmdoeZHWlmY7POlZe1mdmBwLOk1ZNS/AGYIGmY\n9zG0jFzBwdoTeNOdq5Npv6pRyy+3LgCu3FYhcwJcZ1CK3foycFVxUZkqp0ray+fvLGBJiTbXV+m2\n4lyZ2fvARknnZNr8vJf9J8kOtwEPWaKs/Cek3RwoxcyNLCtc4t6r0u4rwBFK/C1wTK26OBuAfSV9\n0fvfybdf1sp8YKLX/SKw2cwK2wMLc3gQafU3nKu+RKxe5ZLe8q1s0DnCbvkk7JZP8mi37nawXgAu\nl7SOFGh/p+cXTrdrBa4FmoEWUozJg6QYkmZJLcB9LgPpQfJKSatJTkE2mL9WXgSGlSrwLX0/ITl5\nLcAtWX2zov4+Gzja9fkWsL6ETKn6Ba4nHQSyRukQiuuKBSQd5bFJlbgaGEWKp1rpq1mV+l1G2u62\nirTNr932QHcyKlJhrrJ8C7hE6VCS54AzM2VzSCuA/5bJu7CCfAeUDvf4eon84ZJeI83LPyodorG7\nb8EbDbxdodly917Je8DMlpCcrOeBW0nbB6lUp6j+fwHnAD+VtIr0OTiu1rGa2e9JjunLwC9JcY5Z\nhgKvmFlbcd0gCIIgCILgk6P+/mv2Hu+0d9Ex7UEGSV8DDjKzf+5pXboSSf+NdOjKNT2tS3fhh2b8\ndzM7v4KMMbX7dAq6iIjBqp2p0Fv+9/Wm2IKgdsJu+STslk96s90kYWYddiR1dwxWb2Qe8GtJj5jZ\nV3tamd6ImdUTo9br8S1y/cm5ugX4O+B/VBWeuqO1CYKeY/j+9Wx2CIIgCILa6PcrWEEQtEeSxd+F\nIAiCIAiCypRbweruGKwgCIIgCIIgCII+SzhYQRAEfYBafvgw6H2E3fJJ2C2fhN3ySR7tFg5WEARB\nEARBEARBFxExWEEQtCNisIIgCIIgCKoTMVhBEARBEARBEAQ7mHCwgiAI+gB53KMehN3yStgtn4Td\n8kke7Ra/gxUEQQekDqvdQcDw4Q20tr7S02oEQRAEQa8mYrCCIGiHJIP4uxCUQsT/jCAIgiBIRAxW\nEARBEARBEATBDmaHOliSGiStLVPWJGnsjuy/HJIOkLRS0iOZvI09oUs5JJ0kaWYNcp3WW9JVknYp\nU3aRpNs9PUXSxCptXSRpShWZ9zurYzUKbfo91lSDfJOkFyS1uO33qSJfcf69/MFO6jxI0kLvf4Kk\nEyQ959eHlvusZOpXHaukXSU9JGm9pLWSbsiUHeL9zemM3kFeaO5pBYI6yGNsQRB2yytht3ySR7t1\nxwpWb9xPchbwmJl9NZPXG/WsRad69J4MDK6jXr067Ii5tTLpSpxvZkea2Vgz+7+d7KOe8mLGAub9\nzwUuBG4ws7HA1hrbq0XmZjP7LHAkcIKk00gdv2hmnwMOk3RQJ3UPgiAIgiAIaqA7HKydJc2StE7S\nA6VWTiSdL2mNv270vAGSZnreaklXef5oXwVYJenZOh8U9wLeLMp7K6PPRO+zRdJvPG+mpNskLZH0\nsqTxnr+bpEWuy2pJZ3p+g68izJS0QdJsSad6/Q2SvuBygyXdI2mppBWSznA1PgTerWEsb3k70zKr\nM5u8zcG+mtHi8zhB0hXAKKBJ0mKve7HrtBQ4PtP2FtKDfyW2uhyS9pM0z23TIumLhSnNzO01kpa5\nzBTPmyFpUkZmiqTvlZMv4mPg7RrmCTp3v2+bf1+tKsztCkm7ucwQSXPdzvdl9N8oaZinj/LVs32B\n+4CjvZ3LgHOB67N1vc4ASTdJesbHfWmtYzWzrWb2uKc/AlYCnyoSe4P0GQj6FI09rUBQB42NjT2t\nQlAHYbd8EnbLJ3m0W3ecIvgZ4GIzWyrpHmAS8PNCoaSRwI2kb9s3AwvdSdkE7G9mn3e5PbzKbNK3\n/vMlDaI+J3Eg0JbNMLNjvZ8xwA+B48zsHUnZB9ERZna8pM8C84F5wF+Bs8xsi6S9gaVeBjAaONvM\n1kl6FjjP65/pfYwH/hFYbGaXSNoTWCZpkZk9DTztOh0FfMfMLiseSEFvM5sCTPE2/gP4Z+ArwOtm\n9nVvZ4iZvS/paqDRxzcCmEqa//dI+4xWepu3VJtIM3sgc3k70Gxm4yUJ2L0g5v2fChxsZsd4+XxJ\nJwBzgFuBO1z+XGBcOXkzexJ32sxsE3COtz8SuLsw3hL8WtJ/AfPMbHqVcW2bf+D7wCQze1rSYJLN\nAY4AxgCtwBJJXzKzp+i4ymRm9pakfwC+b2YFJ/w44EEzmyepISN/CbDZzI71e3yJpMfM7NVOjBW/\nd88gzW2WNtJnoAJTM+lG4uE9CIIgCIL+TnNzc01bFrtjBetPZrbU07OAE4rKjwaazOxtM2sjOVAn\nAn8EDvJVo9OA9yXtDowys/kAZvahmf2VTuAP6oeTHLhSnALMNbN3vI/NmbLfed56YL9Ck8AMSauB\nRcAoSYWyjWa2ztPPeznAWuBAT48DrpXUQnJuBgEHZBUysxWlnKsyzAJuMbMW7+dUXyE6wcwKsVBi\n+6rSsWyf/49Izk69nAL8i+tsmf4KjHN9VpKcuM+QHKhVwL6SRkj6PPC2mb1eTr5c52b25woOxwVm\ndhjwd8DfSfpWJ8a1BPiFr/4N9fsUYJn3acAqttv0k55xPg6Y6PfEM8AwisZdZaxIGgjcD9xqZq8U\nFW8ifQYqMDXzaqxd86AHae5pBYI6yGNsQRB2yytht3zSm+zW2NjI1KlTt73K0R0rWB2+zS8h0+GB\n1Mw2SzocOA34LjCBFDtU8eHVt5pd6v2cbmatmbIBJMftA+DhToyhwAcldL4Q2Ac40szalA6d2KWE\nfFvmuo3tcy/SKtdLdejTDklTSQ7tvQBm9pLSQSKnA9N9ZazUyk1X/ehRtfggATPM7O4SZXNJNh7B\ndievknyn4p/M7M/+/p+S7geOITmjtdT9qaSHgK+RVpPGeVHWvh+z3aYfsf3Li5KHiVRBwBVmtrCO\nugXuAjaY2T+VKPslsEDSMWb2nU/QRxAEQRAEQVBEd6xgNUg61tMXAE8UlS8DTpQ0zL91Px943Lfb\nDTSz3wI/Asaa2RbgNUnfgG2nsu2abczM7sgcZNBaVNZmZgcCzwLfLKPvH4AJmRiaoWXkCk7JnsCb\n7lydDDSUkKnEAuDKbRWkI2qo01GZFLv1ZeCqTN5IYKuZ3Q/cTDpkAdJWwMKWy2dI8z9U0s4kJ6dU\n+5dn46TKsJi0BbQQRzSkUN3fFwDfLsQwSRrlsUkADwDnAWeTnK1y8vsUtVkVSQP9fsLH+HXgOb8+\nS5mT9srU/7SZPW9mNwHLgUOrdLkROMrTZ9eqZ4YFwCRJO3n/Bxff51X0nQ7sYWZXlxG5BrgknKu+\nRmNPKxDUQR5jC4KwW14Ju+WTPNqtOxysF4DLJa0jBdbf6fkG4E7QtaT9LS3AcjN7ENgfaPZtUve5\nDMBE4ErfkrcEGF6HTi+Stl11wLf0/YTk5LUAhTikcitxs0kHF6wGvgWsLyFTqn6B60kHgaxROqb7\numIBPyjhrgrjAbiadHjFcj9EYSpwGCmmqwX4MVBYvbobeFTSYp//aaTYsSeAdR1aThwK/KWKDpOB\nkyWtITmxYzy/YOuFpG1rT7vMXDxOy+d9CLDJzN6oID8k22YWSSN9pamYvyGt2KwibTXc5HMAKU6u\n2mEik5WOPF9NOvzikRIyWX2uA26XtIy0mlWOcvfEr0h2WOn3xJ0UrTaXG6uk/UnxfWO0/WCObxeJ\nDQVerqBXEARBEARBUCdK4SP9C0k/APY2s2urCgcASJoPjPc4rT6DpHuBq82smvPYJ/AYxDXAOWa2\noYyM9c5fLQgq08yOX8US/fF/xo6kubk5l9/O9nfCbvkk7JZPerPdJGFmHXZVdccKVm9kHnC8Mj80\nHFTGzM7sa84VgJlN7EfO1SGkVeIW0ipuEARBEARB0MX0yxWsIAjKk1awgqAjw4c30Nr6Sk+rEQRB\nEAS9gnIrWN1ximAQBDkjvngJgiAIgiCoj/66RTAIgqBP0Zt+JySonbBbPgm75ZOwWz7Jo93CwQqC\nIAiCIAiCIOgiIgYrCIJ2SLL4uxAEQRAEQVCZOEUwCIIgCIIgCIJgBxMOVhAEQR8gj3vUg7BbXgm7\n5ZOwWz7Jo93CwQqCIAiCIAiCIOgiIgYrCIJ2xO9gBUHQ1xi+/3BaN7X2tBpBEPQxysVghYMVBEE7\nJBlTe1qLIAiCLmRq/L5fEARdTxxyEQRB0JfZ2NMKBHURdssleYwJCcJueSWPdtuhDpakBklry5Q1\nSRq7I/svh6QDJK2U9Egmr1f9m5N0kqSZNch1Wm9JV0napUzZRZJu9/QUSROrtHWRpClVZN7vrI7V\nKLTp91hTDfJNkl6Q1OK236eKfMX59/IHO6nzIEkLvf8Jkk6Q9JxfH1rus5KpX+tYx0paI+lFSbdm\n8g/x/uZ0Ru8gCIIgCIKgdrpjBas3rsmfBTxmZl/N5PVGPWvRqR69JwOD66hXrw47Ym6tTLoS55vZ\nkWY21sz+byf7qKe8mLGAef9zgQuBG8xsLLC1xvZqkfkX4BIzOwQ4RNJppI5fNLPPAYdJOqiTuge9\nnbBoPgm75ZLGxsaeViGog7BbPsmj3brDwdpZ0ixJ6yQ9UGrlRNL5/o37Gkk3et4ASTM9b7Wkqzx/\ntK8CrJL0bJ0PinsBbxblvZXRZ6L32SLpN543U9JtkpZIelnSeM/fTdIi12W1pDM9v0HSeq+3QdJs\nSad6/Q2SvuBygyXdI2mppBWSznA1PgTerWEsb3k70zKrM5u8zcGSHvL8Nb5qcgUwCmiStNjrXuw6\nLQWOz7S9hfTgX4mtLoek/STNc9u0SPpiYUozc3uNpGUuM8XzZkialJGZIul75eSL+Bh4u4Z5gs7d\n79vm31erCnO7QtJuLjNE0ly3830Z/TdKGubpo3z1bF/gPuBob+cy4Fzg+mxdrzNA0k2SnvFxX1rr\nWCWNAIaY2XLPupf0hUKWN0ifgSAIgiAIgqCL2akb+vgMcLGZLZV0DzAJ+HmhUNJI4EbgSGAzsNCd\nlE3A/mb2eZfbw6vMJn3rP1/SIOpzEgcCbdkMMzvW+xkD/BA4zszekZR9EB1hZsdL+iwwH5gH/BU4\ny8y2SNobWOplAKOBs81snaRngfO8/pnex3jgH4HFZnaJpD2BZZIWmdnTwNOu01HAd8zssuKBFPQ2\nsynAFG/jP4B/Br4CvG5mX/d2hpjZ+5KuBhp9fCOAqaT5fw9oBlZ6m7dUm0gzeyBzeTvQbGbjJQnY\nvSDm/Z8KHGxmx3j5fEknAHOAW4E7XP5cYFw5eTN7EnfazGwTcI63PxK4uzDeEvxa0n8B88xsepVx\nbZt/4PvAJDN7WtJgks0BjgDGAK3AEklfMrOn6LjKZGb2lqR/AL5vZgUn/DjgQTObJ6khI38JsNnM\njvV7fImkx8zs1RrGuj/ps1Ngk+dlaSN9BsqT3Yh4IPEtex7YSNgpj4Tdcklzc3Muv1Xv74Td8klv\nsltzc3NNMWHd4WD9ycyWenoWcAUZBws4Gmgys7cBJM0GTgSmAwdJug34PfCYpN2BUWY2H8DMPuys\nMv6gfrjrUopTgLlm9o73sTlT9jvPWy9pv0KTwAxJJ5IeXEdlyjaa2TpPPw8s8vRa0mMrwDjgDEk/\n8OtBwAHAhkKnZrYC6OBclWEWcIuZtUjaAvxM0gzgYXdMCjoXVpWOpf38zwEOrrGvYk4B/t51NqA4\n9moccKqkld7/biQHaqakfd3Z2w9428xelzS5lDzwJCUwsz8D5ZyrC8zsz776NE/St8ys3D1QzBLg\nF35vznPdAJZ5n0haRbLpU2RW7OpkHGkb3wS/3oM07lcLAlXGWo1NpM/As2UlTq6z5SAIgiAIgj5K\nY2NjO2dv2rRpJeW6w8Hq8G1+CZkOD6RmtlnS4cBpwHeBCaTYoYoPr77V7FLv53Qza82UDQD+CHwA\nPNyJMRT4oITOFwL7AEeaWZvSoRO7lJBvy1y3sX3uRVrleqkOfdohaSrJob0XwMxeUjpI5HRguq+M\nlVq5+aQOQYFq8UECZpjZ3SXK5pJsPIK0olVNvlPxTwVHyMz+U9L9wDGUd7KL6/5U0kPA10irSeO8\nKGvfj9lu04/YvrJa8jCRKgi4wswW1lH3deBvM9ef8rwsvwQWSDrGzL5TRx9BbyRWQfJJ2C2X9JZv\n04POEXbLJ3m0W3fEYDVIOtbTFwBPFJUvA06UNEzSQOB84HHfbjfQzH4L/AgYa2ZbgNckfQO2ncq2\na7YxM7sjc5BBa1FZm5kdSPrm/ptl9P0DMCETQzO0jFzBKdkTeNOdq5OBhhIylVgAXLmtgnREDXU6\nKpNit74MXJXJGwlsNbP7gZtJhyxA2gpY2HL5DGn+h0rameTklGr/8mycVBkWk7aAFuKIhhSq+/sC\n4NuFGCZJozw2CeAB4DzgbJKzVU5+n6I2qyJpoN9P+Bi/Djzn12dJuqFK/U+b2fNmdhOwHDi0Spcb\ngaM8fXatemZYAEyStJP3f3DxfV4Ov+fflVTYVjkR+PcisWtIh2CEcxUEQRAEQdDFdIeD9QJwuaR1\npMD6Oz3fYNsD4bWk2J8WYLmZPUiKG2mW1EI6HOBarzcRuFLSatLWreF16PQiMKxUgW/p+wnJyWsB\nCnFI5VbiZpMOLlgNfAtYX0KmVP0C15MOAlmjdEz3dcUCSgcl3FVhPABXkw6vWK50iMJU4DBSTFcL\n8GPStkuAu4FHJS32+Z9Gih17AljXoeXEocBfqugwGThZ0hqSEzvG8wu2XgjcDzztMnPxOC2f9yHA\nJjN7o4L8kGybWSSN9JWmYv6GtGKzihRftsnnAFKcXLXDRCZLWus2/hB4pIRMVp/rgNslLSOtZpWj\n3D3xK5IdVvo9cSdFq80VxgpwOXAP6T5/ycweLSofCrxcQa8gj/SqH5oIaibslkvy+Ls8Qdgtr+TR\nbuqPv2zu8U57m9m1VYUDACTNB8abWSWHIXdIuhe42syqOY99Al/VWgOcY2YbysgYU7tVraAriMMS\n8knYrXuYCl35vNObgu6D2gm75ZPebDdJmFmHXVX91cEaDfwa2FL0W1hB0GeRdAhpK+Ya4CIr8+GX\n1P/+KARB0KcZvv9wWje1VhcMgiDoBOFgBUFQE5LK+V5BEARBEASBU87B6o4YrCAIgmAHk8c96kHY\nLa+E3fJJ2C2f5NFu4WAFQRAEQRAEQRB0EbFFMAiCdsQWwSAIgiAIgurEFsEgCIIgCIIgCIIdTDhY\nQRAEfYA87lEPwm55JeyWT8Ju+SSPdgsHKwiCIAiCIAiCoIuIGKwgCNoRv4MV5JHhwxtobX2lp9UI\ngiAI+hHxO1hBENREcrDi70KQN0T8PwuCIAi6kzjkIgiCoE/T3NMKBHWQx9iCIOyWV8Ju+SSPdtuh\nDpakBklry5Q1SRq7I/svh6QDJK2U9Egmb2NP6FIOSSdJmlmDXKf1lnSVpF3KlF0k6XZPT5E0sUpb\nF0maUkXm/c7qWI1Cm36PNdUg/4ikFknPSfqVpJ2qyFecfy9/sJM6D5K00O+9CZJOcH1WSjq03Gcl\nU7/qWCXtKukhSeslrZV0Q6bsEO9vTmf0DoIgCIIgCGqnO1aweuOejbOAx8zsq5m83qhnLTrVo/dk\nYHAd9erVYUfMrZVJl2OCmR1pZp8D9gK+2ck+6ikvZixgZjbWzOYCFwI3mNlYYGuN7dUic7OZfRY4\nEjhB0mmkjl/08R8m6aBO6h70ehp7WoGgDhobG3tahaAOwm75JOyWT/Jot+5wsHaWNEvSOkkPlFo5\nkXS+pDX+utHzBkia6XmrJV3l+aN9FWCVpGfrfFDcC3izKO+tjD4Tvc8WSb/xvJmSbpO0RNLLksZ7\n/m6SFrkuqyWd6fkNvoowU9IGSbMlner1N0j6gssNlnSPpKWSVkg6w9X4EHi3hrG85e1Mc31XStrk\nbQ721YwWn8cJkq4ARgFNkhZ73Ytdp6XA8Zm2t5Ae/Cux1eWQtJ+keW6bFklfLExpZm6vkbTMZaZ4\n3gxJkzIyUyR9r5x8ER8Db1ebJDMr6LgzMAj4S5Uq2+bfV6sKc7tC0m4uM0TSXLfzfRn9N0oa5umj\nlFZr9wXuA472di4DzgWuz9b1OgMk3STpGR/3pbWO1cy2mtnjnv4IWAl8qkjsDdJnIAiCIAiCIOhq\nzGyHvYAGoA34ol/fA3zP002kb/RHAq8Cw0gO32LgTC97LNPWHv6+FDjT04OAXerQaxowuUzZGOAF\nYKhf7+XvM4E5nv4s8JKnBwK7e3rvTH4D6SF9jF8/C9zj6TOBeZ7+CXCBp/cENgC7Ful0FHBXjWPb\nE1hNWr0YD/wyUzbE3/+YGd+IzPzvBDwJ3F6nvf8NuNLTyvT3nr+fWtDHyx8ETgCOAJoz7TwP7F9O\n3q/fL9H/SOChCvo9SnKs5nRyXPOB4zw92O/Tk4B3vE8BTwFfyszvsIzt/uDpk4D5mXZnAuMz98sa\nT18K/DBzjy8HGjoz1sK9C/xv4MCi/MXAFyrUM5iSeTUZWLx6/au/2wnLI01NTT2tQlAHYbd8EnbL\nJ73Jbk1NTTZlypRtL//fQ/GrYhxKF/EnM1vq6VnAFcDPM+VHA01m9jaApNnAicB04CBJtwG/Bx6T\ntDswyszmk0b0YWeVkSTgcNelFKcAc83sHe9jc6bsd563XtJ+hSaBGZJOJDmTozJlG81snaefBxZ5\nei1woKfHAWdI+oFfDwIOIDlaeH8rgMtqHOIs4BYza5G0BfiZpBnAw2b2ZEbnwqrSsbSf/znAwTX2\nVcwpwN+7zgYUx16NA06VtNL73w042MxmStpX0ghgP+BtM3td0uRS8iQnsANm9mfg6+WUM7OvSBoE\nPCBpopndW+O4lgC/8HtznusGsMz7RNIqkk2fIrNiVyfjSNv4Jvj1HqRxv5oZS8WxShoI3A/camav\nFBVvIn0Gni2vwtTOax0EQRAEQdCHaWxsbLdlcdq0aSXlusPBsirXUOKB1Mw2SzocOA34LjCBFDtU\n8eHVt5pd6v2cbmatmbIBpNWFD4CHOzGGAh+U0PlCYB/gSDNrUzp0YpcS8m2Z6za2z72As83spTr0\naYekqSSH9l4AM3tJ6SCR04HpkhaZ2fRSVT9p304p2xb3M8PM7i5RNpdk4xHAnBrkq/VVWkGzDyX9\nL+AYoCYHy8x+Kukh4GvAEknjvChr34/ZbtOP2L79tuRhIlUQcIWZLayjboG7gA1m9k8lyn4JLJB0\njJl95xP0EfQqGntagaAO8hhbEITd8krYLZ/k0W7dEYPVIOlYT18APFFUvgw4UdIw/9b9fOBxSXsD\nA83st8CPgLGW4mhek/QN2HYq267ZxszsDkuHGYzNOlde1mZmB5K+uS93yMEfgAmZGJqhZeQKTsme\nwJvuXJ1M2upVLFOJBcCV2ypIR9RQp6MyKXbry8BVmbyRwFYzux+4mbTtEuA90qoIwDOk+R/q8UkT\nKIGky7NxUmVYDExy+QGShhSq+/sC4NuFGCZJozw2CeAB4DzgbJKzVU5+n6I2q6IUJzfC0zuRHKVV\nfn2WMiftlan/aTN73sxuIm3XO7RKlxtJWwPx8XSWBcAk1xVJBxff51X0nU7aUnt1GZFrgEvCuQqC\nIAiCIOh6usPBegG4XNI6UkzInZ5vAO4EXUv6EZcWYLmZPUiKwWmW1EI6HOBarzcRuFLSatLWreF1\n6PQiKeaoA76l7yckJ68FuCWrb1bU32eTDi5YDXwLWF9CplT9AteTDgJZo3RM93XFAn5Qwl0VxgNw\nNenwiuV+iMJU4DBgmY/jx6RtlwB3A49KWuzzP40U2/YEsK5Dy4lDqX4wxGTgZElrSE7sGM8v2Hoh\nadva0y4zF9jdy9YBQ4BNZvZGBfkh2TazSBrpK03F7AbM9218K4DXgH/1stFUP0xkstKR56tJcXWP\nlJDJ6nMdcLukZaTVrHKUuyd+RbLDSr8n7qRotbncWCXtD/wQGJM5mOPbRWJDgZcr6BXkkuaeViCo\ngzz+vksQdssrYbd8kke7KYXK9C883mlvM7u2qnAAgKT5pAMZKjkMuUPSvcDVZlbNeewTeAziGuAc\nM9tQRsbq3IEZ9CjN9O9tgiKP/8+am5tzuf2lvxN2yydht3zSm+0mCTPrsKuqvzpYo4FfA1us/W9h\nBUGfRdIhpK2Ya4CLrMyHPzlYQZAvhg9voLX1lZ5WIwiCIOhHhIMVBEFNSCrnewVBEARBEAROOQer\nO2KwgiAIgh1MHveoB2G3vBJ2yydht3ySR7uFgxUEQRAEQRAEQdBFxBbBIAjaEVsEgyAIgiAIqhNb\nBIMgCIIgCIIgCHYw4WAFQRD0AfK4Rz0Iu+WVsFs+CbvlkzzaLRysIAiCIAiCIAiCLiJisIIgaEf8\nDlYQBL2B4fsPp3VTa0+rEQRBUJb4HawgCGpCkjG1p7UIgqDfMxXiGSUIgt5MHHIRBEHQl9nYsMjF\nvgAAIABJREFU0woEdRF2yyV5jAkJwm55JY9226EOlqQGSWvLlDVJGrsj+y+HpAMkrZT0SCavV/2b\nk3SSpJk1yHVab0lXSdqlTNlFkm739BRJE6u0dZGkKVVk3u+sjtUotOn3WFMN8o9IapH0nKRfSdqp\ninzF+ffyBzup8yBJC/3emyDpBNdnpaRDy31WMvVrHetYSWskvSjp1kz+Id7fnM7oHQRBEARBENRO\nd6xg9cb1/bOAx8zsq5m83qhnLTrVo/dkYHAd9erVYUfMrZVJl2OCmR1pZp8D9gK+2ck+6ikvZixg\nZjbWzOYCFwI3mNlYYGuN7dUi8y/AJWZ2CHCIpNNIHb/o4z9M0kGd1D3o7YRF80nYLZc0Njb2tApB\nHYTd8kke7dYdDtbOkmZJWifpgVIrJ5LO92/c10i60fMGSJrpeaslXeX5o30VYJWkZ+t8UNwLeLMo\n762MPhO9zxZJv/G8mZJuk7RE0suSxnv+bpIWuS6rJZ3p+Q2S1nu9DZJmSzrV62+Q9AWXGyzpHklL\nJa2QdIar8SHwbg1jecvbmeb6rpS0ydscLOkhz1/jqyZXAKOAJkmLve7FrtNS4PhM21tID/6V2Opy\nSNpP0jy3TYukLxamNDO310ha5jJTPG+GpEkZmSmSvldOvoiPgberTZKZFXTcGRgE/KVKlW3z76tV\nhbldIWk3lxkiaa7b+b6M/hslDfP0UUqrtfsC9wFHezuXAecC12frep0Bkm6S9IyP+9JaxyppBDDE\nzJZ71r2kLxSyvEH6DARBEARBEARdTMVtUl3EZ4CLzWyppHuAScDPC4WSRgI3AkcCm4GF7qRsAvY3\ns8+73B5eZTbpW//5kgZRn5M4EGjLZpjZsd7PGOCHwHFm9o6k7IPoCDM7XtJngfnAPOCvwFlmtkXS\n3sBSLwMYDZxtZuskPQuc5/XP9D7GA/8ILDazSyTtCSyTtMjMngaedp2OAr5jZpcVD6Sgt5lNAaZ4\nG/8B/DPwFeB1M/u6tzPEzN6XdDXQ6OMbAUwlzf97QDOw0tu8pdpEmtkDmcvbgWYzGy9JwO4FMe//\nVOBgMzvGy+dLOgGYA9wK3OHy5wLjysmb2ZO402Zmm4BzvP2RwN2F8RYj6VHgaGCRmT1aZVzb5h/4\nPjDJzJ6WNJhkc4AjgDFAK7BE0pfM7Ck6rjKZmb0l6R+A75tZwQk/DnjQzOZJasjIXwJsNrNj/R5f\nIukxM3u1hrHuT/rsFNjkeVnaSJ+B8mQ3Ih5IfMueBzYSdsojYbdc0tzcnMtv1fs7Ybd80pvs1tzc\nXFNMWHc4WH8ys6WengVcQcbBIj3wNpnZ2wCSZgMnAtOBgyTdBvweeEzS7sAoM5sPYGYfdlYZf1A/\n3HUpxSnAXDN7x/vYnCn7neetl7RfoUlghqQTSQ+uozJlG81snaefBxZ5ei3psRVgHHCGpB/49SDg\nAGBDoVMzWwF0cK7KMAu4xcxaJG0BfiZpBvCwOyYFnQurSsfSfv7nAAfX2FcxpwB/7zobUBx7NQ44\nVdJK7383kgM1U9K+7uztB7xtZq9LmlxKHniSEpjZn4GSzpWXf8UdlgckTTSze2sc1xLgF35vznPd\nAJZ5n0haRbLpU2RW7OpkHGkb3wS/3oM07lczY6k41ipsIn0Gni0rcXKdLQdBEARBEPRRGhsb2zl7\n06ZNKynXHQ5Wh2/zS8h0eCA1s82SDgdOA74LTCDFDlV8ePWtZpd6P6ebWWumbADwR+AD4OFOjKHA\nByV0vhDYBzjSzNqUDp3YpYR8W+a6je1zL9Iq10t16NMOSVNJDu29AGb2ktJBIqcD031lbHqpqp+0\nb6dafJCAGWZ2d4myuSQbjyCtaFWTryuuy8w+lPS/gGNI2+dqqfNTSQ8BXyOtJo3zoqx9P2a7TT9i\n+8pqycNEqiDgCjNbWEfd14G/zVx/yvOy/BJYIOkYM/tOHX0EvZFYBcknYbdc0lu+TQ86R9gtn+TR\nbt0Rg9Ug6VhPXwA8UVS+DDhR0jBJA4Hzgcd9u91AM/st8CNgrMfRvCbpG7DtVLZds42Z2R1+mMHY\nrHPlZW1mdiDpm/tyhxz8AZiQiaEZWkau4JTsCbzpztXJQEMJmUosAK7cVkE6ooY6HZVJsVtfBq7K\n5I0EtprZ/cDNpEMWIG0FLGy5fIY0/0M9PmkCJZB0eTZOqgyLSVtAC3FEQwrV/X0B8O1CDJOkUR6b\nBPAAcB5wNsnZKie/T1GbVVGKkxvh6Z1IjtIqvz5L0g1V6n/azJ43s5uA5cChVbrcCBzl6bNr1TPD\nAmCS64qkg4vv83L4Pf+upMK2yonAvxeJXUM6BCOcqyAIgiAIgi6mOxysF4DLJa0jBdbf6fkG2x4I\nryXF/rQAy83sQVLcSLOkFtLhANd6vYnAlZJWk7ZuDa9DpxeBYaUKfEvfT0hOXgtQiEMqtxI3m3Rw\nwWrgW8D6EjKl6he4nnQQyBqlY7qvKxZQOijhrgrjAbiadHjFcqVDFKYCh5FiulqAH5O2XQLcDTwq\nabHP/zRS7NgTwLoOLScOpfrBEJOBkyWtITmxYzy/YOuFwP3A0y4zF4/T8nkfAmwyszcqyA/JtplF\n0khfaSpmN1L81ipgBfAa8K9eNprqh4lMlrTWbfwh8EgJmaw+1wG3S1pGWs0qR7l74lckO6z0e+JO\nilabK4wV4HLgHtJ9/lKJeLOhwMsV9ArySK/6oYmgZsJuuSSPv8sThN3ySh7tpv74K+ke77S3mV1b\nVTgAQNJ8YLyZVXIYcoeke4Grzaya89gn8FWtNcA5ZrahjIwxtVvVCrqCOCwhn4TdyjMVeuszSm8K\nug9qJ+yWT3qz3SRhZh12VfVXB2s08GtgS9FvYQVBn0XSIaStmGuAi6zMh19S//ujEARBr2P4/sNp\n3dRaXTAIgqCHCAcrCIKakFTO9wqCIAiCIAiccg5Wd8RgBUEQBDuYPO5RD8JueSXslk/Cbvkkj3YL\nBysIgiAIgiAIgqCLiC2CQRC0I7YIBkEQBEEQVCe2CAZBEARBEARBEOxgwsEKgiDoA+Rxj3oQdssr\nYbd8EnbLJ3m0WzhYQRAEQRAEQRAEXUTEYAVB0I74HaygtzN8eAOtra/0tBpBEARBPyd+BysIgppI\nDlb8XQh6MyL+dwVBEAQ9TRxyEQRB0Kdp7mkFgjrIY2xBEHbLK2G3fJJHu+1QB0tSg6S1ZcqaJI3d\nkf2XQ9IBklZKeiSTt7EndCmHpJMkzaxBrtN6S7pK0i5lyi6SdLunp0iaWKWtiyRNqSLzfmd1rEah\nTb/HmmqQny7pT5Leq7H9ivPv5Q/WrjFIGiRpod97EySdIOk5vz603GclU7/qWCXtKukhSeslrZV0\nQ6bsEO9vTmf0DoIgCIIgCGqnO1aweuM+jrOAx8zsq5m83qhnLTrVo/dkYHAd9erVYUfMrZVJl2M+\ncPQn6KOe8mLGAmZmY81sLnAhcIOZjQW21theLTI3m9lngSOBEySdRur4RTP7HHCYpIM6qXvQ62ns\naQWCOmhsbOxpFYI6CLvlk7BbPsmj3brDwdpZ0ixJ6yQ9UGrlRNL5ktb460bPGyBppuetlnSV54/2\nVYBVkp6t80FxL+DNory3MvpM9D5bJP3G82ZKuk3SEkkvSxrv+btJWuS6rJZ0puc3+CrCTEkbJM2W\ndKrX3yDpCy43WNI9kpZKWiHpDFfjQ+DdGsbylrczzfVdKWmTtznYVzNafB4nSLoCGAU0SVrsdS92\nnZYCx2fa3kJ68K/EVpdD0n6S5rltWiR9sTClmbm9RtIyl5nieTMkTcrITJH0vXLyRXwMvF1tksxs\nmZm9UU0uw7b599WqwtyukLSbywyRNNftfF9G/42Shnn6KKXV2n2B+4CjvZ3LgHOB67N1vc4ASTdJ\nesbHfWmtYzWzrWb2uKc/AlYCnyoSe4P0GQiCIAiCIAi6GjPbYS+gAWgDvujX9wDf83QT6Rv9kcCr\nwDCSw7cYONPLHsu0tYe/LwXO9PQgYJc69JoGTC5TNgZ4ARjq13v5+0xgjqc/C7zk6YHA7p7eO5Pf\nQHpIH+PXzwL3ePpMYJ6nfwJc4Ok9gQ3ArkU6HQXcVePY9gRWk1YvxgO/zJQN8fc/ZsY3IjP/OwFP\nArfXae9/A670tDL9vefvpxb08fIHgROAI4DmTDvPA/uXk/fr90v0PxJ4qIqO79UxrvnAcZ4e7Pfp\nScA73qeAp4AvZeZ3WMZ2f/D0ScD8TLszgfGZ+2WNpy8Ffpi5x5cDDXWMdS/gfwMHFuUvBr5QoZ7B\nlMyrycDi1etf/clOWF+hqampp1UI6iDslk/CbvmkN9mtqanJpkyZsu3l/48ofu3EjudPZrbU07OA\nK4CfZ8qPBprM7G0ASbOBE4HpwEGSbgN+DzwmaXdglJnNJ43ow84qI0nA4a5LKU4B5prZO97H5kzZ\n7zxvvaT9Ck0CMySdSHImR2XKNprZOk8/Dyzy9FrgQE+PA86Q9AO/HgQcQHK08P5WAJfVOMRZwC1m\n1iJpC/AzSTOAh83syYzOhVWlY2k//3OAg2vsq5hTgL93nQ0ojr0aB5wqaaX3vxtwsJnNlLSvpBHA\nfsDbZva6pMml5ElOYAfM7M/A1+vUvRJLgF/4vTnPdQNY5n0iaRXJpk+RWbGrk3GkbXwT/HoP0rhf\nLQhUG6ukgcD9wK1m9kpR8SbSZ+DZ8ipM7bzWQRAEQRAEfZjGxsZ2WxanTZtWUq47HCyrcg0lHkjN\nbLOkw4HTgO8CE0ixQxUfXn2r2aXez+lm1popG0BaXfgAeLgTYyjwQQmdLwT2AY40szalQyd2KSHf\nlrluY/vcCzjbzF6qQ592SJpKcmjvBTCzl5QOEjkdmC5pkZlNL1X1k/btlLJtcT8zzOzuEmVzSTYe\nAcypQb5aX12Gmf1U0kPA14AlksZ5Uda+H7Pdph+xffttycNEqiDgCjNbWI++zl3ABjP7pxJlvwQW\nSDrGzL7zCfoIehWNPa1AUAd5jC0Iwm55JeyWT/Jot+6IwWqQdKynLwCeKCpfBpwoaZh/634+8Lik\nvYGBZvZb4EfAWDPbArwm6Ruw7VS2XbONmdkdZnakpYMEWovK2szsQNI3998so+8fgAmZGJqhZeQK\nTsmewJvuXJ1M2upVLFOJBcCV2ypIR9RQp6MyKXbry8BVmbyRwFYzux+4mbTtEuA90qoIwDOk+R8q\naWeSk1Oq/cuzcVJlWAxMcvkBkoYUqvv7AuDbhRgmSaM8NgngAeA84GySs1VOfp+iNjtLu3qSzlLm\npL2SFaRPm9nzZnYTabveoVX62EjaGghpPJ1lATBJ0k7e/8HF93kVfaeTttReXUbkGuCScK6CIAiC\nIAi6nu5wsF4ALpe0jhQTcqfnG4A7QdeSfsSlBVhuZg+SYnCaJbWQDge41utNBK6UtJq0dWt4HTq9\nSIo56oBv6fsJyclrAW7J6psV9ffZpIMLVgPfAtaXkClVv8D1pINA1igd031dsYAflHBXhfEAXE06\nvGK5H6IwFTgMWObj+DFp2yXA3cCjkhb7/E8jxbY9Aazr0HLiUOAvVXSYDJwsaQ3JiR3j+QVbLyRt\nW3vaZeYCu3vZOmAIsMn8MIoy8kOybWaRNNJXmjog6aeSXgN2VTqu/cdeNJrqh4lMVjryfDUpru6R\nEjJZfa4Dbpe0jLSaVY5y98SvSHZY6ffEnRStNpcbq6T9gR8CYzIHc3y7SGwo8HIFvYJc0tzTCgR1\nkMffdwnCbnkl7JZP8mg3pVCZ/oXHO+1tZtdWFQ4AkDSfdCBDJYchd0i6F7jazKo5j30Cj0FcA5xj\nZhvKyFg37sAMuoxm+s82QdFX/nc1NzfncvtLfyfslk/CbvmkN9tNEmbWYVdVf3WwRgO/BrZY+9/C\nCoI+i6RDSFsx1wAXWZkPfzhYQe+n7zhYQRAEQX4JBysIgppIDlYQ9F6GD2+gtfWVnlYjCIIg6OeU\nc7C6IwYrCIKcUeo3HeLVu19NTU09rkN3vfqSc5XH2IIg7JZXwm75JI92CwcrCIIgCIIgCIKgi4gt\ngkEQtEOSxd+FIAiCIAiCysQWwSAIgiAIgiAIgh1MOFhBEAR9gDzuUQ/Cbnkl7JZPwm75JI92Cwcr\nCIIgCIIgCIKgi4gYrCAI2hExWEEQBEEQBNUpF4O1U08oEwRB70bq8LciCIKgJMP3H07rptaeViMI\ngqDXECtYQRC0Q5Ixtae1CDrNRuCgnlYi6DR9wW5T02/n9Seam5tpbGzsaTWCThJ2yye92W5ximAQ\nBEEQBEEQBMEOZoc6WJIaJK0tU9YkaeyO7L8ckg6QtFLSI5m8jT2hSzkknSRpZg1yndZb0lWSdilT\ndpGk2z09RdLEKm1dJGlKFZn3O6tjNQpt+j3WVIP8dEl/kvReje1XnH8vf7B2jUHSIEkL/d6bIOkE\nSc/59aHlPiuZ+rWOdaykNZJelHRrJv8Q729OZ/QOckLeV0H6K2G3XNJbv00PKhN2yyd5tFt3rGD1\nxn0DZwGPmdlXM3m9Uc9adKpH78nA4Drq1avDjphbK5Mux3zg6E/QRz3lxYwFzMzGmtlc4ELgBjMb\nC2ytsb1aZP4FuMTMDgEOkXQaqeMXzexzwGGS4rEuCIIgCIJgB9AdDtbOkmZJWifpgVIrJ5LO92/c\n10i60fMGSJrpeaslXeX5o30VYJWkZ+t8UNwLeLMo762MPhO9zxZJv/G8mZJuk7RE0suSxnv+bpIW\nuS6rJZ3p+Q2S1nu9DZJmSzrV62+Q9AWXGyzpHklLJa2QdIar8SHwbg1jecvbmeb6rpS0ydscLOkh\nz1/jqyZXAKOAJkmLve7FrtNS4PhM21tID/6V2OpySNpP0jy3TYukLxamNDO310ha5jJTPG+GpEkZ\nmSmSvldOvoiPgberTZKZLTOzN6rJZdg2/75aVZjbFZJ2c5khkua6ne/L6L9R0jBPH6W0WrsvcB9w\ntLdzGXAucH22rtcZIOkmSc/4uC+tdaySRgBDzGy5Z91L+kIhyxukz0DQl+hVa/BBzYTdckkef5cn\nCLvllTzarTtOEfwMcLGZLZV0DzAJ+HmhUNJI4EbgSGAzsNCdlE3A/mb2eZfbw6vMJn3rP1/SIOpz\nEgcCbdkMMzvW+xkD/BA4zszekZR9EB1hZsdL+ixpRWQe8FfgLDPbImlvYKmXAYwGzjazdZKeBc7z\n+md6H+OBfwQWm9klkvYElklaZGZPA0+7TkcB3zGzy4oHUtDbzKYAU7yN/wD+GfgK8LqZfd3bGWJm\n70u6Gmj08Y0AppLm/z2gGVjpbd5SbSLN7IHM5e1As5mNlyRg94KY938qcLCZHePl8yWdAMwBbgXu\ncPlzgXHl5M3sSdxpM7NNwDne/kjg7sJ4PwnZ+Qe+D0wys6clDSbZHOAIYAzQCiyR9CUze4qOq0xm\nZm9J+gfg+2ZWcMKPAx40s3mSGjLylwCbzexYv8eXSHrMzF6tYaz7kz47BTZ5XpY20megPNmNiAcS\n25iCIAiCIOj3NDc31+TwdYeD9SczW+rpWcAVZBws0ratJjN7G0DSbOBEYDpwkKTbgN8Dj0naHRhl\nZvMBzOzDzirjD+qHuy6lOAWYa2bveB+bM2W/87z1kvYrNAnMkHQi6cF1VKZso5mt8/TzwCJPryU9\ntgKMA86Q9AO/HgQcAGwodGpmK4AOzlUZZgG3mFmLpC3AzyTNAB52x6Sgc2FV6Vjaz/8c4OAa+yrm\nFODvXWcDimOvxgGnSlrp/e9GcqBmStrXnb39gLfN7HVJk0vJA09SAjP7M/CJnasSLAF+4ffmPNcN\nYJn3iaRVJJs+RWbFrk7GkbbxTfDrPUjjfrUg8AnHuon0GXi2rMTJdbYc9BzhBOeTsFsuyWNMSBB2\nyyu9yW6NjY3t9Jk2bVpJue5wsDp8m19CpsMDqZltlnQ4cBrwXWACKXao4sOrbzW71Ps53cxaM2UD\ngD8CHwAPd2IMBT4oofOFwD7AkWbWpnToxC4l5Nsy121sn3uRVrleqkOfdkiaSnJo7wUws5eUDhI5\nHZjuK2PTS1X9pH071eKDBMwws7tLlM0l2XgEaUWrmny3xcyZ2U8lPQR8jbSaNM6Lsvb9mO02/Yjt\nK6slDxOpgoArzGxhHXVfB/42c/0pz8vyS2CBpGPM7Dt19BEEQRAEQRCUoTtisBokHevpC4AnisqX\nASdKGiZpIHA+8LhvtxtoZr8FfgSMNbMtwGuSvgHbTmXbNduYmd1hZkf6QQKtRWVtZnYg6Zv7b5bR\n9w/AhEwMzdAycgWnZE/gTXeuTgYaSshUYgFw5bYK0hE11OmoTIrd+jJwVSZvJLDVzO4HbiYdsgBp\nK2Bhy+UzpPkfKmlnkpNTqv3Ls3FSZVhM2gJaiCMaUqju7wuAbxdimCSN8tgkgAeA84CzSc5WOfl9\nitrsLO3qSTpL0g0VK0ifNrPnzewmYDlwaJU+NgJHefrsOnRcAEyStJP3f3DxfV4Ov+fflVTYVjkR\n+PcisWtIh2CEc9WXiFiefBJ2yyV5jAkJwm55JY926w4H6wXgcknrSIH1d3q+wbYHwmtJsT8twHIz\ne5AUN9IsqYV0OMC1Xm8icKWk1aStW8Pr0OlFYFipAt/S9xOSk9cCFOKQyq3EzSYdXLAa+BawvoRM\nqfoFricdBLJG6Zju64oF/KCEuyqMB+Bq0uEVy/0QhanAYaSYrhbgx6RtlwB3A49KWuzzP40UO/YE\nsK5Dy4lDgb9U0WEycLKkNSQndoznF2y9ELgfeNpl5uJxWj7vQ4BNhcMoysgPybaZRdJIX2nqgKSf\nSnoN2FXpuPYfe9Foqh8mMlnSWrfxh8AjJWSy+lwH3C5pGWk1qxzl7olfkeyw0u+JOylaba40VuBy\n4B7Sff6SmT1aVD4UeLmCXkEQBEEQBEGdqL/9+jqAxzvtbWbXVhUOAJA0HxhvZpUchtwh6V7gajOr\n5jz2CXxVaw1wjpltKCNjTO1WtYIgyDNToT8+SwRBEEjCzDrsquqvDtZo4NfAlqLfwgqCPoukQ0hb\nMdcAF1mZD7+k/vdHIQiCuhm+/3BaN7VWFwyCIOhjhIMVBEFNSCrnewW9mObm5l510lJQG2G3fBJ2\nyydht3zSm+1WzsHqjhisIAiCIAiCIAiCfkGsYAVB0I5YwQqCIAiCIKhOrGAFQRAEQRAEQRDsYMLB\nCoIg6APk8XdCgrBbXgm75ZOwWz7Jo93CwQqCIAiCIAiCIOgiIgYrCIJ2RAxWEARBEARBdcrFYO3U\nE8oEQdC7Sb9HHATBJ2X48AZaW1/paTWCIAiCbiS2CAZBUAKLV+5eTb1Ah3gVv95441UqkcfYgiDs\nllfCbvkkj3YLBysIgiAIgiAIgqCL6BIHS1KDpLVlypokje2KfjqLpAMkrZT0SCZvY0/oUg5JJ0ma\nWYPcRn8vO9dF8kMkvSbp9mwbkoZ1Qreqc+X2PaBC+UWS/qnWPmvU66LCuCRNkTSxivzRklr8tVrS\nN2voY6akE6uUj++k3idIes7vyb+RdLOktZJ+6uP4XpX6tYz1y5Ke9XEul3Rypuz7kl6oZfxBHmns\naQWCOmhsbOxpFYI6CLvlk7BbPsmj3boyBsu6sK2u4izgMTO7NpPXG/WsRScrky7H9cDjdfTzSeR3\ndDv1shY4yszaJI0AnpP0P83s427W40LgBjO7H0DSpcBQMzNJU7qoj7eAr5tZq6T/BiwAPgVgZrdI\nehK4GZjTRf0FQRAEQRAEGbpyi+DOkmZJWifpAUm7FAtIOl/SGn/d6HkDfDVgjX/rfpXnj5a0UNIq\n/0b+oDp02gt4syjvrYw+E73PFkm/8byZkm6TtETSy4VVCkm7SVqUWR040/MbJK33ehskzZZ0qtff\nIOkLLjdY0j2SlkpaIekMV+ND4N0axvJWcYakuzMrM29K+n89/yhgP+Cx4irAld7/akmHZMb2r26D\nVZL+e7k+S/AX4GNv5yve9ipJC0vou4+k/ynpGX8dp8RGSXtk5F6UtG8p+RL9bwG2VlLQzP5qZm1+\nuSvwbg3O1WaSbZB0o688rZJ0U0bmpBL3yUmSHsyM5Z/8PrsEOBe4XtJ9kv4d2B1YIWlC0Tx9WtIj\nvgL1eMFOwPs1jHW1mbV6+nlgF0k7Z0RagT2rjD3IJc09rUBQB3mMLQjCbnkl7JZP8mi3rlzB+gxw\nsZktlfT/s3fvUXZUZf7/3x8yiQiBhBAIkDEX4ncwcUhMAiIjQiOgw8yAiIAgN10sQC6CXBwZvBAE\nEUX4fqP+wOEyMUIECQNMEDMkQDq6IBFyJyQEgTgQmXARgmQGQejP74/aB6tP17l006G7Os9rrV59\nTtVTVc+uXZ2cffbeVTcApwNXVVZK2hm4HJhI9uF1bmqkrAOG2x6f4ioftGeQfds/S9IAutYY7Ae0\n5RfY3isdZxxwIbC37ZclDc6F7WT7o5LGArOA24E/AYfZ3ihpe2BhWgcwBviM7VWSFgFHp+0PTcc4\nHPgacJ/tkyQNAh6SdK/tBcCClNNk4FTbp1QXpJJ31bKT03YjgNnANEkCvk/WW3JQwTl53vZkSacB\n5wOnAN8ANuTqYFCtYxbkcETaZihwLbCP7aerzmfFVOAq2w9Keh9wj+1xku4EPg1Ml/Rh4He2X5A0\nozoeGFd1/CsrryWdmi3ytdUHTvv9N2A08LkmynVO2m4IWb1/IL3fNhdWdJ1AQY+d7Rsk7QPcZfv2\ntK8/2p6UXud7sK4luw6eTHlfAxxgO//3VLOsuZgjgCW2/5xb3EZTf/dTcq9biOFnIYQQQtjctba2\nNtXg684G1tO2F6bXNwFfItfAAvYE5tl+CSB9eN4XuBQYLWkq8EtgjqSBwC62ZwHYfqOzyaSGxoSU\nS5GPAzNtv5yOsSG37s60bLWkHSu7BL6jbF5OG7BLbt1a26vS60eBe9PrR4BR6fUngEMkfSW9HwCM\nANZUDmp7MVmDpzPl3BKYCZxpe52kM4C7bT+bnQKq77d9R/q9mKxRA3Ag8Pa8HNvN9KhEg1wQAAAg\nAElEQVRV+wgw3/bTaR8bCmIOBMamugEYKGkr4Fbgm8B04Gj+MnytVnwh2/9aZ91DwN9K2g24R9I8\n239solyvAK9Juh64G/hFbl3RdfKOSNoa+DtgZq7c/avj6pU17eeDwHfo2Mh+EdhB0uAadZRMaT7p\n0Eu09HQCoQvKOLcgRL2VVdRbOfWmemtpaWmXz8UXX1wYtynnYBXNu+nwcB3bGyRNAD4JfBE4Evhy\nUWy7HUmnAyen4/xDZVhUWrcF8BTwOtmH4s56vSDnY4GhwMQ0l2ctsGVBfFvufb63QGS9XL/tQj71\nXAPcZnteer83sE86P9uQDd181faFVbm+Rfc/B63Rw5ME7FXVowKwQNmQ0KFk8+a+VS9e7+AZTbbX\nSHoS+D9kjcxG8W+lXqQDyK7NM9NrKL5O3qR9b2uHobINbAG8XOnZ6gpJf03Wm3a87d/l19l+TdIt\nwFOSPmu7w1DOEEIIIYTQdd05B2ukpMqQss8Bv65a/xCwr6QhkvoBxwDz03C7frbvAL4OTLK9EXhG\n0qcAJA2Q9N78zmxfbXui7Un5xlVa12Z7FLCIXM9MlfuBI9MQMCRtVyOu8sF5ENnwujZld2YbWRBT\nzz3AWW9vIH2oiW3qSr1VA21fUVlm+zjbo2zvSjYE8Ke5xlUtc4EzcvvtMLxP2fyznevsYyHwMUkj\nU3zR+ZwDnJ3b54TcujvIejxX5XpW6sU3TdKodM2R8ns/8Nv0frrSPLka224NDLb9n8C5wPhaoen3\nfwHjJPVP5/GAGvH5bd5m+1VgbRreV8mh1jGL8h1E1sv21VyPcn79YLK/ieHRuOprWns6gdAFZZxb\nEKLeyirqrZzKWG/d2cB6DDhD0iqym0v8OC03QGoEXUD2KWAp8LDtu4DhQKukpcCNKQbgBLIbMiwH\nHgCGdSGnx4HC25KnIX3fJmvkLQUqc3lq9cTNAPZM+RwHrC6IKdq+4hKy3qQVym6z/q3qAEmTJdWc\nU1PgPGB3ZTe5WCKp0fDCWrldCgxRdsvwpVSNNUpD1cYAL9Xcsf0i2fDGO9I+bikIOxvYQ9kNNlYC\np+bW3UrWS3hLk/EdSDq1xjnYB1guaUk6zim54YHjgWfr7HYb4Bep3n8FnJOWF14nttelY6xMZVlS\nHVPnfcVxwEnKbqqxEji0OqBOWc8kq6tv5q6Lobn1g4DnbNe9WUYIIYQQQuga2T19B+1NJ8132r7q\nNu2hk9J8ni/YPr+nc+lOkrYBrre92TwXKg13nGq76I6MlRj3/J31Q+grRF/+fzaEEDZnkrDdYURS\nX29gjQF+Amy0fXAPpxNCj5J0Hlkv4RW2b64T13f/UQjhXTZs2EjWr/9dT6cRQghhE9gsG1ghhM6T\n5Ph3oXxaW1t71Z2WQnOi3sop6q2cot7KqTfXW60GVnfOwQohhBBCCCGEzVr0YIUQ2okerBBCCCGE\nxqIHK4QQQgghhBA2sWhghRBCH1DG54SEqLeyinorp6i3cipjvUUDK4QQQgghhBC6SczBCiG0E3Ow\nQgghhBAaqzUH6696IpkQQu8mdfi3IoQQQgibyLDhw1i/bn1PpxG6SfRghRDakWSm9HQWodPWAqN7\nOonQaVFv5RT1Vk69ud6mQHwmLxbPwQohhBBCCCGEzVi3NLAkjZT0SI118yRN6o7jdJakEZKWSJqd\nW7a2J3KpRdJ+kqY1Ebc2/a55rqvit5H0jKQf5PchaUgncmt4rlL9jqiz/kRJP2z2mE3mdWKlXJIu\nknRCg/g9JS1NP8slfbaJY0yTtG+D9Yd3Mu99JK1M1+R7JF0h6RFJ303lOLfB9g3LmuL+RdJvJa2W\n9Inc8vMkPdZM+UMJ9dZvZUN9UW/lFPVWTlFvpdRbe6/q6c4erN7Yr3kYMMf2wbllvTHPZnJyjde1\nXALM78Jx3kn8pt5PVz0CTLY9Efgk8P9J6tcDeRwLXGZ7ku3XgZOB8ba/2l0HkDQWOAoYCxwMXK00\nocr2lcCJwBnddbwQQgghhNBedzaw+ku6SdIqSbdK2rI6QNIxklakn8vTsi1Sb8CK1Ltwdlo+RtJc\nScskLZLUle8dBgPPVy17IZfPCemYSyVNT8umSZoq6QFJT1R6KSRtLenelMtySYem5SNTT8E0SWsk\nzZB0UNp+jaQ9UtxWkm6QtFDSYkmHpDTeAF5poiwvVC+QdF2uZ+Z5Sd9IyycDOwJzqjcBzkrHXy7p\nb3Jl+7dUB8skfbrWMQv8AXgr7efv076XSZpbkO9QSbdJ+k362VuZtZK2zcU9LmmHoviC428EXquX\noO0/2W5Lb98LvGL7rQbl2kBWN0i6PPU8LZP0vVzMfgXXyX6S7sqV5YfpOjuJrOFziaQbJf0HMBBY\nLOnIqvO0q6TZkh6WNL9ST8CrjcoKfAq4xfabtn8H/Bb4cG79emBQg32EMupVffOhaVFv5RT1Vk5R\nb6VUxudgdeddBHcDvmB7oaQbgNOBqyorJe0MXA5MJPvwOjc1UtYBw22PT3GVD9ozyL7tnyVpAF1r\nDPYD2vILbO+VjjMOuBDY2/bLkgbnwnay/dHUGzALuB34E3CY7Y2StgcWpnUAY4DP2F4laRFwdNr+\n0HSMw4GvAffZPknSIOAhSffaXgAsSDlNBk61fUp1QSp5Vy07OW03ApgNTEu9Fd8n6y05qOCcPG97\nsqTTgPOBU4BvABtydTCo1jELcjgibTMUuBbYx/bTVeezYipwle0HJb0PuMf2OEl3Ap8Gpkv6MPA7\n2y9ImlEdD4yrOv6VldeSTs0W+drqA6f9/hvZAIHPNVGuc9J2Q8jq/QPp/ba5sKLrBAp67GzfIGkf\n4C7bt6d9/dH2pPT6olz4tWTXwZMp72uAA2zn/55qlXU46XpKfp+WVbTRzN/9vNzrUcSwihBCCCFs\n9lpbW5tq8HVnA+tp2wvT65uAL5FrYAF7AvNsvwSQPjzvC1wKjJY0FfglMEfSQGAX27MAbL/R2WRS\nQ2NCyqXIx4GZtl9Ox9iQW3dnWrZa0o6VXQLfUTYvpw3YJbdure1V6fWjwL3p9SNkH08BPgEcIukr\n6f0AYASwpnJQ24vJGjydKeeWwEzgTNvrJJ0B3G372ewUUH1nkzvS78VkjRqAA4G35+XYbqZHrdpH\ngPm2n0772FAQcyAwNtUNwEBJWwG3At8EpgNHAz9vEF/I9r/WWfcQ8LeSdgPukTTP9h+bKNcrwGuS\nrgfuBn6RW1d0nbwjkrYG/g6YmSt3/+q4emVt4EVgB0mDa9RRZv8u7j30nGgEl1PUWzlFvZVT1Fsp\n9aY5WC0tLe3yufjiiwvjurOBVf2tfdG8mw63MbS9QdIEsrkxXwSOBL5cFNtuR9LpZHNYDPyD7fW5\ndVsATwGvk30o7qzXC3I+FhgKTLTdpuwGEFsWxLfl3ud7C0TWy/XbLuRTzzXAbbYrfQ57A/uk87MN\n2dDNV21fWJXrW3T/c9AaPTxJwF62/1y1fIGyIaFDyebNfatevN7BM5psr5H0JPB/yBqZjeLfSr1I\nB5Bdm2em11B8nbxJ+97WDkNlG9gCeLnSs9UFvwfel3v/12kZALZfk3QL8JSkz9ruMJQzhBBCCCF0\nXXfOwRopqTKk7HPAr6vWPwTsK2mIshsMHAPMT8Pt+tm+A/g6MMn2RuAZSZ8CkDRA0nvzO7N9te2J\n6YYB66vWtdkeBSwi1zNT5X7gyDQEDEnb1YirfHAeRDa8rk3S/sDIgph67gHOensD6UNNbFNX6q0a\naPuKyjLbx9keZXtXsiGAP801rmqZS+7GB0XD+5TNP9u5zj4WAh+TNDLFF53POcDZuX1OyK27g6zH\nc1WuZ6VefNMkjUrXHCm/95PNTULSdKV5cjW23RoYbPs/gXOB8bVC0+//AsZJ6p/O4wE14vPbvM32\nq8BaSUfkcqh1zCKzgKPT38xosrI+lNvXYLK/ieHRuOpjYm5BOUW9lVPUWzlFvZVSGedgdWcD6zHg\nDEmryG4u8eO03ACpEXQB0AosBR62fRfZ/JBWSUuBG1MMwAlkN2RYDjwADOtCTo8DhbclT0P6vk3W\nyFsKVOby1OqJmwHsmfI5DlhdEFO0fcUlZL1JK5TdZv1b1QGSJkvqMH+ojvOA3ZXd5GKJpEbDC2vl\ndikwRNktw5cCLVV5iWye2Us1d2y/SDa88Y60j1sKws4G9lB2g42VwKm5dbeS9RLe0mR8B5JOrXEO\n9gGWS1qSjnNKbnjgeODZOrvdBvhFqvdfAeek5YXXie116RgrU1mWVMfUeV9xHHCSsptqrAQOrQ6o\nVdZ0Xd8KrCIbcnu62z+5cBDwnO1GN8sIIYQQQghdoL781Og032l72xc0DA41Sfog2Q1Mzu/pXLqT\npG2A621vNs+FSsMdp9ouuiNjJcZMefdyCiGEEDZ7U6AvfybvqyRhu8OIpL7ewBoD/ATYWPUsrBA2\nO5LOI+slvML2zXXi+u4/CiGEEEIvNGz4MNavW984MPQqm2UDK4TQeZIc/y6UT2tra6+601JoTtRb\nOUW9lVPUWzn15nqr1cDqzjlYIYQQQgghhLBZix6sEEI70YMVQgghhNBY9GCFEEIIIYQQwiYWDawQ\nQugDyvickBD1VlZRb+UU9VZOZay3aGCFEEIIIYQQQjeJOVghhHZiDlYIIYQQQmO15mD9VU8kE0Lo\n3aQO/1aEsFkaNmwk69f/rqfTCCGEUCIxRDCEUMDxU7qfeb0gh77389xz/8WmVMa5BSHqrayi3sqp\njPUWDawQQgghhBBC6Cbd0sCSNFLSIzXWzZM0qTuO01mSRkhaIml2btnansilFkn7SZrWRNza9Lvm\nua6K30bSM5J+kN+HpCGdyK3huUr1O6LO+hMl/bDZYzaZ14mVckm6SNIJDeKHSLpf0qv589Fgm2mS\n9m2w/vBO5r2PpJXpmnyPpCskPSLpu6kc5zbYvpmyHihpkaTlkh6WtH9u3XmSHpP02c7kHcqipacT\nCF3Q0tLS0ymELoh6K6eot3IqY711Zw+Wu3Ff3eUwYI7tg3PLemOezeTkGq9ruQSY34XjvJP4Tb2f\nrvoT8HXgvB7O41jgMtuTbL8OnAyMt/3VbjzGC8A/2Z4AfB64sbLC9pXAicAZ3Xi8EEIIIYSQ050N\nrP6SbpK0StKtkrasDpB0jKQV6efytGyL1BuwIn3rfnZaPkbSXEnL0jfyo7uQ02Dg+aplL+TyOSEd\nc6mk6WnZNElTJT0g6YlKL4WkrSXdm+sdODQtHylpddpujaQZkg5K26+RtEeK20rSDZIWSlos6ZCU\nxhvAK02U5YXqBZKuS7kvlfS8pG+k5ZOBHYE51ZsAZ6XjL5f0N7my/Vuqg2WSPl3rmAX+ALyV9vP3\nad/LJM0tyHeopNsk/Sb97K3MWknb5uIel7RDUXzB8TcCr9VL0Pb/2n4QeL2J8lRsIKsbJF2eep6W\nSfpeLma/gutkP0l35cryw3SdnQQcBVwi6UZJ/wEMBBZLOrLqPO0qaXbqgZpfqSfg1SbKutz2+vT6\nUWBLSf1zIeuBQZ04D6E0Wns6gdAFZZxbEKLeyirqrZzKWG/deRfB3YAv2F4o6QbgdOCqykpJOwOX\nAxPJPrzOTY2UdcBw2+NTXOWD9gyyb/tnSRpA1xqD/YC2/ALbe6XjjAMuBPa2/bKkwbmwnWx/VNJY\nYBZwO1kvyGG2N0raHliY1gGMAT5je5WkRcDRaftD0zEOB74G3Gf7JEmDgIck3Wt7AbAg5TQZONX2\nKdUFqeRdtezktN0IYDYwTZKA75P1lhxUcE6etz1Z0mnA+cApwDeADbk6GFTrmAU5HJG2GQpcC+xj\n++mq81kxFbjK9oOS3gfcY3ucpDuBTwPTJX0Y+J3tFyTNqI4HxlUd/8rKa0mnZot8baO8myjXOWmf\nQ8jq/QPp/ba5sKLrBAp67GzfIGkf4C7bt6d9/dH2pPT6olz4tWTXwZPpfFwDHGA7//fUsKySjgCW\n2P5zbnEbTf3dT8m9biGGn4UQQghhc9fa2tpUg687G1hP216YXt8EfIlcAwvYE5hn+yWA9OF5X+BS\nYLSkqcAvgTmSBgK72J4FYPuNziaTGhoTUi5FPg7MtP1yOsaG3Lo707LVknas7BL4jrJ5OW3ALrl1\na22vSq8fBe5Nrx8BRqXXnwAOkfSV9H4AMAJYUzmo7cVkDZ7OlHNLYCZwpu11ks4A7rb9bHYKqL7f\n9h3p92KyRg3AgcDb83JsN9OjVu0jwHzbT6d9bCiIORAYm+oGYKCkrYBbgW8C04GjgZ83iC9k+1+7\nkHcjrwCvSboeuBv4RW5d0XXyjkjaGvg7YGau3P2r4xqVVdIHge/QsZH9IrCDpME16iiZ0nzSoZdo\n6ekEQheUcW5BiHorq6i3cupN9dbS0tIun4svvrgwrjsbWNXf2hfNu+nwcB3bGyRNAD4JfBE4Evhy\nUWy7HUmnk81hMfAPlWFRad0WwFNkQ8Lu7kQZKvJDySp5HAsMBSbablN2A4gtC+Lbcu/zvQUi6+X6\nbRfyqeca4Dbb89L7vYF90vnZhmzo5qu2L6zK9S26/zlojR6eJGCvqh4VgAXKhoQOJZs396168XoX\nn9Fk+63Ui3QA2bV5ZnoNxdfJm7Tvbe0wVLaBLYCXKz1bXSHpr8l60463/bv8OtuvSboFeErSZ213\nGMoZQgghhBC6rjvnYI2UVBlS9jng11XrHwL2VXZHt37AMcD8NNyun+07yG5EMMn2RuAZSZ8CkDRA\n0nvzO7N9te2J6YYB66vWtdkeBSwi1zNT5X7gyDQEDEnb1YirfHAeRDa8rk3ZndlGFsTUcw9w1tsb\nSB9qYpu6Um/VQNtXVJbZPs72KNu7kg0B/GmucVXLXHI3Piga3qds/tnOdfaxEPiYpJEpvuh8zgHO\nzu1zQm7dHWQ9nqtyPSv14ruqXV1Jmq40T64wOOtRGmz7P4FzgfEN9vtfwDhJ/dN5PKBGfIdcAGy/\nCqxNw/sqOdQ6ZlG+g8h62b6a61HOrx9M9jcxPBpXfU1rTycQuqCMcwtC1FtZRb2VUxnrrTsbWI8B\nZ0haRXZziR+n5QZIjaALyD4FLAUetn0XMBxolbSU7I5nF6TtTiC7IcNy4AFgWBdyehwovC15GtL3\nbbJG3lKgMpenVk/cDGDPlM9xwOqCmKLtKy4h601aoew269+qDpA0WVJn5g+dB+yu7CYXSyQ1Gl5Y\nK7dLgSHKbhm+lKqxRmmo2hjgpZo7tl8kG954R9rHLQVhZwN7KLvBxkrg1Ny6W8l6CW9pMr4DSafW\nOgepx/FK4ERJT0v6QFo1Hni2zm63AX6R6v1XwDlpeeF1YntdKsvKVJYl1TF13lccB5yk7KYaK4FD\nC8pTq6xnktXVN3PXxdDc+kHAc7br3iwjhBBCCCF0jeyevoP2ppPmO21v+4KGwaGmNJ/nC7bP7+lc\nupOkbYDrbW82z4VKwx2n2i66I2Mlxj1/Z/0QegvRl/+fDCGE0HWSsN1hRFJfb2CNAX4CbKx6FlYI\nmx1J55H1El5h++Y6cdHACuFt0cAKIYRQrFYDqzuHCPY6tp+0/bFoXIWQ3dI+zVms2bj6C8VP/MQP\nYtiw/HTb7lfGuQUh6q2sot7KqYz11t13kQsh9AHxjX35tLa29qpb2YYQQgibqz49RDCE0HmSHP8u\nhBBCCCHUt1kOEQwhhBBCCCGEd1M0sEIIoQ8o4xj1EPVWVlFv5RT1Vk5lrLdoYIUQQgghhBBCN4k5\nWCGEdmIOVgghhBBCYzEHK4QQQgghhBA2sbhNewihA6nDlzEhhBBCqQwbPoz169a//T4eZ1FOZay3\naGCFEDqa0tMJhE5bC4zu6SRCp0W9lVPUWyk8N+W5nk4hbKZiDlYIoR1JjgZWCCGE0psC8Tk3bEqb\ndA6WpJGSHqmxbp6kSd1xnM6SNELSEkmzc8vW9kQutUjaT9K0JuLWpt81z3VV/DaSnpH0g/w+JA3p\nRG4Nz1Wq3xF11p8o6YfNHrPJvE6slEvSRZJOaBA/RNL9kl7Nn48G20yTtG+D9Yd3Mu99JK1M1+R7\nJF0h6RFJ303lOLfB9g3LmuL+RdJvJa2W9Inc8vMkPSbps53JO4QQQgghNK87b3LRG78iOAyYY/vg\n3LLemGczObnG61ouAeZ34TjvJH5T76er/gR8HTivh/M4FrjM9iTbrwMnA+Ntf7W7DiBpLHAUMBY4\nGLhaaUKV7SuBE4Ezuut4oRfpVV8dhaZFvZVT1FsplfF5SqGc9dadDaz+km6StErSrZK2rA6QdIyk\nFenn8rRsi9QbsELScklnp+VjJM2VtEzSIkldGe08GHi+atkLuXxOSMdcKml6WjZN0lRJD0h6otJL\nIWlrSfemXJZLOjQtH5l6CqZJWiNphqSD0vZrJO2R4raSdIOkhZIWSzokpfEG8EoTZXmheoGk61Lu\nSyU9L+kbaflkYEdgTvUmwFnp+Msl/U2ubP+W6mCZpE/XOmaBPwBvpf38fdr3MklzC/IdKuk2Sb9J\nP3srs1bStrm4xyXtUBRfcPyNwGv1ErT9v7YfBF5vojwVG8jqBkmXp56nZZK+l4vZr+A62U/SXbmy\n/DBdZyeRNXwukXSjpP8ABgKLJR1ZdZ52lTRb0sOS5lfqCXi1UVmBTwG32H7T9u+A3wIfzq1fDwzq\nxHkIIYQQQgid0J03udgN+ILthZJuAE4HrqqslLQzcDkwkezD69zUSFkHDLc9PsVVPmjPIPu2f5ak\nAXStMdgPaMsvsL1XOs444EJgb9svSxqcC9vJ9kdTb8As4HayXpDDbG+UtD2wMK0DGAN8xvYqSYuA\no9P2h6ZjHA58DbjP9kmSBgEPSbrX9gJgQcppMnCq7VOqC1LJu2rZyWm7EcBsYFrqrfg+WW/JQQXn\n5HnbkyWdBpwPnAJ8A9iQq4NBtY5ZkMMRaZuhwLXAPrafrjqfFVOBq2w/KOl9wD22x0m6E/g0MF3S\nh4Hf2X5B0ozqeGBc1fGvrLyWdGq2yNc2yruJcp2T9jmErN4/kN5vmwsruk6goMfO9g2S9gHusn17\n2tcfbU9Kry/KhV9Ldh08mc7HNcABtvN/T7XKOpx0PSW/T8sq2mjm735e7vUoYjJ3GUQdlVPUWzlF\nvZVS2e5EFzK9qd5aW1ub6lHrzgbW07YXptc3AV8i18AC9gTm2X4JIH143he4FBgtaSrwS2COpIHA\nLrZnAdh+o7PJpIbGhJRLkY8DM22/nI6xIbfuzrRstaQdK7sEvqNsXk4bsEtu3Vrbq9LrR4F70+tH\nyD6eAnwCOETSV9L7AcAIYE3loLYXkzV4OlPOLYGZwJm210k6A7jb9rPZKaB64t0d6fdiskYNwIHA\n2/NybDfTo1btI8B820+nfWwoiDkQGJvqBmCgpK2AW4FvAtOBo4GfN4gvZPtfu5B3I68Ar0m6Hrgb\n+EVuXdF18o5I2hr4O2Bmrtz9q+PeQVlfBHaQNLhGHWX27+LeQwghhBD6qJaWlnYNvosvvrgwblPO\nwSqad9PhLhvpQ94EoBX4InBdrdh2O5JOT0PjlkjaqWrdFmQjpMeSfSjurPxQskoexwJDgYm2J5IN\nPdyyIL4t9z7fWyCyXq6J6We07TW8c9cAt9mu9DnsDZwp6SmynqzjJV1WULa36P7b9Dd6eJKAvXLn\nYEQavrcAGJN6wQ4D/r1efDfnXJftt8iG2N0G/BPwn7nVRdfJm7T/u+owVLaBLYCX0zytSrn/thPb\n/x54X+79X6dlANh+DbgFeEpSUQ9nKKuYE1JOUW/lFPVWSmWcyxPKWW/d2cAaKakypOxzwK+r1j8E\n7Kvsjm79gGOA+Wm4XT/bd5DdiGCS7Y3AM5I+BSBpgKT35ndm++r04XOS7fVV69psjwIWkeuZqXI/\ncGQaAoak7WrEVT44DyIbXtcmaX9gZEFMPfcAZ729gfShJrapK/VWDbR9RWWZ7eNsj7K9K9kQwJ/a\nvrDBruaSu/FB0fA+ZfPPdq6zj4XAxySNTPFF53MOcHZunxNy6+4g6/FcletZqRffVe3qStJ0pXly\nhcFZj9Jg2/8JnAuMb7Df/wLGSeqfzuMBzeYCYPtVYK2kI3I51DpmkVnA0elvZjTwfrK/vcq+BpP9\nTQy33WGeXAghhBBCeGe6s4H1GHCGpFVkN5f4cVpugNQIuoCsp2op8LDtu8jmh7RKWgrcmGIATiC7\nIcNy4AFgWBdyehwovC15GtL3bbJG3lKgMpenVk/cDGDPlM9xwOqCmKLtKy4huxHICmW3Wf9WdYCk\nyZI6M3/oPGD3XE9eo+GFtXK7FBii7JbhS4GWqrxENs/spZo7tl8kG954R9rHLQVhZwN7KLvBxkrg\n1Ny6W8l6CW9pMr4DSafWOgfKbjl/JXCipKclfSCtGg88W2e32wC/SPX+K+CctLzwOrG9LpVlZSrL\nkuqYOu8rjgNOUnZTjZXAoQXlKSxruq5vBVaRDbk93e0fAjIIeC71ZIW+JOaElFPUWzlFvZVSb5rL\nE5pXxnrr0w8aTvOdtrd9QcPgUJOkD5LdwOT8ns6lO0naBrje9mbzXKh004yptovuyFiJiQcNhxBC\nKL8p8aDhsGmpxoOG+3oDawzwE2Bj1bOwQtjsSDqPrJfwCts314nru/8ohBBC2GwMGz6M9ev+Mouk\ntbW1lL0hm7veXG+1GljdfZODXsX2k8DHejqPEHqDdEv7KxsGEt/4lVFv/g8o1Bb1Vk5RbyGEevp0\nD1YIofMkOf5dCCGEEEKor1YPVnfe5CKEEEIIIYQQNmvRwAohhD6gjM8JCVFvZRX1Vk5Rb+VUxnqL\nBlYIIYQQQgghdJOYgxVCaCfmYIUQQgghNBZzsEIIIYQQQghhE4sGVgihA0nxU9KfnXYa1dOXT+iE\nMs4tCFFvZRX1Vk5lrLc+/RysEEJXxRDB8mkFWnjuuQ4jFUIIIYTwLoo5WCGEdiQ5GlhlpnhQdAgh\nhPAu6FVzsCSNlPRIjXXzJE16t3NKxx4haYmk2blla3sil1ok7SdpWhNxa9Pvmh8kNx0AACAASURB\nVOe6Kn4bSc9I+kFu2TxJIxpsN03Svg3yvavR8Tsjv09JJ0q6qIltvivpEUkrJB3VRPxFkk5osP7c\nTua9m6SlkhZLGi3pLEmrJN2YyvHDBts3LKukCZIeTGVdli+rpGMkPSbpnM7kHUIIIYQQmteTc7B6\n41eshwFzbB+cW9Yb82wmJ9d4XcslwPyupdOpXDbFPuvuX9I/AB8CxgMfAc6XNHAT5NTIYcBM25Nt\nrwVOAw60fXxa39l6LfI/wPG2dwcOBv6fpG0BbN8M7AdEA6tPau3pBEIXlHFuQYh6K6uot3IqY731\nZAOrv6Sb0jf4t0rasjogfeO+Iv1cnpZtkXpNVkhaLunstHyMpLnpW/tFkkZ3IafBwPNVy17I5XNC\nOuZSSdPTsmmSpkp6QNITkg5Py7eWdG/KZbmkQ9PykZJWp+3WSJoh6aC0/RpJe6S4rSTdIGlh6vE4\nJKXxBvBKE2V5oXqBpOtS7kslPS/pG2n5ZGBHYE7VJn8A3mpwnA0pJyTtmcqxLOW9ddXxC8skaYGk\nsbm4eZIm1TkHea8BGxvkOA74lTP/C6wA/r7BNq+mfZN6mh5N5fpZLuaDKdcnJH0pxbbrMZR0Xurt\nOhj4MnCapPskXQPsCsyuXMO5bYZKuk3Sb9LP3s2W1fYTtp9Mr/+b7HreIbf+OWBQg7KHEEIIIYQu\n6smbXOwGfMH2Qkk3AKcDV1VWStoZuByYSPYhfm5qpKwDhtsen+K2TZvMAC6zPUvSALrWeOwHtOUX\n2N4rHWcccCGwt+2XJQ3Ohe1k+6OpkTALuB34E3CY7Y2StgcWpnUAY4DP2F4laRFwdNr+0HSMw4Gv\nAffZPknSIOAhSffaXgAsSDlNBk61fUp1QSp5Vy07OW03ApgNTJMk4PvAscBBVfFHNDphts9J++wP\n3AIcaXtJ6iF6rSq8sExpu88CUyTtlM7nEknfrhGfP/6tldepATbZ9pSq4y4HvinpKmBrYH/g0Qbl\nuir39qvAKNt/zl1vkF3DLWQNljWSrq5s3nF3ni3px8CrlX1L+iTQkq6nE3PxU4GrbD8o6X3APcC4\nJstKLubDQP9Kgyunib+N/G5b0k/o3Vp6OoHQBS0tLT2dQuiCqLdyinorp95Ub62trU31qPVkA+tp\n2wvT65uAL5FrYAF7AvNsvwQgaQawL3ApMFrSVOCXwJz0YX4X27MAbL/R2WRSQ2NCyqXIx8mGd72c\njrEht+7OtGy1pB0ruwS+o2x+UhuwS27dWtur0utHgUqj4RFgVHr9CeAQSV9J7wcAI4A1lYPaXgx0\naFw1KOeWwEzgTNvrJJ0B3G372ewU0NVbkO0GPGt7ScptYzpePqZWmWaS9Z5NAY4CbmsQX8j2XUCH\n+V6250raE3iQrEfnQRr3zOUtB34m6U5SXSd3234T+IOk54BhndgnZOe66HwfCIzVX07eQElbpd43\noHZZ395x9gXFT4HjC1a/JGlMQcMrZ0rD5EMIIYQQNictLS3tGnwXX3xxYVxvmoNVNLekw4fP1LCZ\nQDbh4IvAdbVi2+1IOj0NjVuSekny67YA1gJjgbubyr691wtyPhYYCky0PZHsg/2WBfFtufdt/KXR\nK7JeronpZ7TtNbxz1wC32Z6X3u8NnCnpKbKerOMlXdbFfTdqnBWWyfazwIuSdifryfp5bptuOQe2\nL0v7+CTZdf94Jzb/R+BHwCTg4XS9QMd6/CvgTbKe0IoOQ1+bIGCvXLlH5BtXDTeWtgF+AfyL7YcL\nQqYCyyR9vgu5hV6rtacTCF1QxrkFIeqtrKLeyqmM9daTDayRkirD2D4H/Lpq/UPAvpKGSOoHHAPM\nT8Pt+tm+A/g6MCn1ljwj6VMAkgZIem9+Z7avTh9WJ9leX7WuzfYoYBHZB/wi9wNHShqSjrFdjbhK\nI2MQ8LztNkn7AyMLYuq5Bzjr7Q2kDzWxTV2pt2qg7Ssqy2wfZ3uU7V2B84Gf2r6wYNvpSvPDalgD\n7JSGLSJpYKq3vHpl+jnwz8C2tlc2Ed80ZfP2KvU2HtidNN9M0mWV66bGtgJG2J4PXABsC9S7QcZz\nwA6StpP0HuCfupDyHODteVmSJjS7YRqqeScwPf2NFLkQeL/tn3QhtxBCCCGEUEdPNrAeA86QtIrs\n5hI/TssNkBpBF5B9LbsUeDgNixoOtEpaCtyYYgBOAM6StBx4gM4P14KsV2NI0Yo0pO/bZI28pcCV\n+Xzzoen3DGDPlM9xwOqCmKLtKy4huxHIinTThG9VB0iaLOnaOuWpdh6we64nrzPDC8cDz9ZaafvP\nZI3TH0laRtZIeE9VWL0y/Tsde68urRPfgaRDJE0pWNUf+LWklWTX2XG2K3PtdgfWF2xT0Q+4KdXj\nYmCq7T8WxFWu2zdTng+TNRBXF8S226bA2cAeym6OshI4tTqgTlmPAvYBPp+r5/FVMQPSzS5Cn9LS\n0wmELuhNcwtC86LeyinqrZzKWG/xoOGcNNdne9sXNAzejKQhZ9fbrtW7V1qSZlfdlr9PS/MAl9ve\nuU5MPGi41OJBwyGEEMK7Qb3pQcO92O3AR5V70HAA26/2xcYVwGbWuDqGrGfxe01Ex09Jf4YNy49G\nDr1dGecWhKi3sop6K6cy1ltP3kWw10l3VftYT+cRwqaQHjR8c5Oxmzib0N1aW1tLOYwihBBC6Gti\niGAIoR1Jjn8XQgghhBDqiyGCIYQQQgghhLCJRQMrhBD6gDKOUQ9Rb2UV9VZOUW/lVMZ6iwZWCCGE\nEEIIIXSTmIMVQmgn5mCFEEIIITQWc7BCCCGEEEIIYROL27SHEDqQOnwZE0IouWHDh7F+3fqeTqNP\niMcilFPUWzmVsd6igRVC6GhKTycQOm0tMLqnkwid9i7W23NTnnt3DhRCCJu5mIMVQmhHkqOBFUIf\nNCUeIh5CCN2pV83BkjRS0iM11s2TNOndzikde4SkJZJm55at7YlcapG0n6RpTcStTb9rnuuq+G0k\nPSPpB7ll8ySNaLDdNEn7Nsj3rkbH74z8PiWdKOmiJrb5rqRHJK2QdFQT8RdJOqHB+nM7mfdukpZK\nWixptKSzJK2SdGMqxw8bbN9sWU+U9LikNfkySDpG0mOSzulM3iGEEEIIoXk9eZOL3vg12mHAHNsH\n55b1xjybyck1XtdyCTC/a+l0KpdNsc+6+5f0D8CHgPHAR4DzJQ3cBDk1chgw0/Zk22uB04ADbR+f\n1ne2XjuQtB3wTWBPYC/gIkmDAGzfDOwHRAOrL+pVXwWFpkW9lVIZn8sTot7Kqoz11pMNrP6Sbkrf\n4N8qacvqgPSN+4r0c3latkXqNVkhabmks9PyMZLmSlomaZGkroxqHww8X7XshVw+J6RjLpU0PS2b\nJmmqpAckPSHp8LR8a0n3plyWSzo0LR8paXXabo2kGZIOStuvkbRHittK0g2SFqYej0NSGm8ArzRR\nlheqF0i6LuW+VNLzkr6Rlk8GdgTmVG3yB+CtBsfZkHJC0p6pHMtS3ltXHb+wTJIWSBqbi5snaVKd\nc5D3GrCxQY7jgF8587/ACuDvG2zzato3qafp0VSun+ViPphyfULSl1Jsux5DSeel3q6DgS8Dp0m6\nT9I1wK7A7Mo1nNtmqKTbJP0m/ezdibJ+kuxLgldsbyCr07fLavs5YFCDfYQQQgghhC7qyZtc7AZ8\nwfZCSTcApwNXVVZK2hm4HJhI9iF+bmqkrAOG2x6f4rZNm8wALrM9S9IAutZ47Ae05RfY3isdZxxw\nIbC37ZclDc6F7WT7o6mRMAu4HfgTcJjtjZK2BxamdQBjgM/YXiVpEXB02v7QdIzDga8B99k+KfVA\nPCTpXtsLgAUpp8nAqbZPqS5IJe+qZSen7UYAs4FpkgR8HzgWOKgq/ohGJ8z2OWmf/YFbgCNtL0k9\nRK9VhReWKW33WWCKpJ3S+Vwi6ds14vPHv7XyOjXAJtueUnXc5cA3JV0FbA3sDzzaoFxX5d5+FRhl\n+8+56w2ya7iFrMGyRtLVlc077s6zJf0YeLWyb0mfBFrS9XRiLn4qcJXtByW9D7gHGNdkWYcDz+Te\n/z4ty2v8tzEv93oUcfOEMog6Kqeot1Iq2x3NQibqrZx6U721trY21aPWkw2sp20vTK9vAr5EroFF\nNsRpnu2XACTNAPYFLgVGS5oK/BKYkz7M72J7FoDtNzqbTGpoTEi5FPk42fCul9MxNuTW3ZmWrZa0\nY2WXwHeUzU9qA3bJrVtre1V6/ShQaTQ8QvZxFuATwCGSvpLeDwBGAGsqB7W9GOjQuGpQzi2BmcCZ\nttdJOgO42/az2Smgq/fn3g141vaSlNvGdLx8TK0yzSTraZkCHAXc1iC+kO27gA7zvWzPlbQn8CBZ\nD+WDNO6Zy1sO/EzSnaS6Tu62/SbwB0nPAcM6sU/IznXR+T4QGKu/nLyBkrZKvW9A7bI26SVJY2w/\nWTNi/y7uOYQQQgihj2ppaWnX4Lv44osL43rTHKyiuSUdPnymhs0EoBX4InBdrdh2O5JOT0PjlqRe\nkvy6LchGwo8F7m4q+/ZeL8j5WGAoMNH2RLIP9lsWxLfl3rfxl0avyHq5Jqaf0bbX8M5dA9xmu9JH\nsTdwpqSnyHqyjpd0WRf33ahxVlgm288CL0ranawn6+e5bbrlHNi+LO3jk2TX/eOd2PwfgR8Bk4CH\n0/UCHevxr4A3yXpCKzoMfW2CgL1y5R6Rb1w18HvaN0L/Oi3Lmwosk/T5LuQWequYy1NOUW+lVMY5\nISHqrazKWG892cAaKakyjO1zwK+r1j8E7CtpiKR+wDHA/DTcrp/tO4CvA5NSb8kzkj4FIGmApPfm\nd2b76vRhdZLt9VXr2myPAhaRfcAvcj9wpKQh6Rjb1YirNDIGAc/bbpO0PzCyIKaee4Cz3t5A+lAT\n29SVeqsG2r6issz2cbZH2d4VOB/4qe0LC7adrjQ/rIY1wE5p2CKSBqZ6y6tXpp8D/wxsa3tlE/FN\nUzZvr1Jv44HdSfPNJF1WuW5qbCtghO35wAXAtkC9G2Q8B+wgaTtJ7wH+qQspzwHenpclaUIntr0H\nOEjSoHSNHpSW5V0IvN/2T7qQWwghhBBCqKMnG1iPAWdIWkV2c4kfp+UGSI2gC8h6qpYCD6dhUcOB\nVklLgRtTDMAJwFmSlgMP0PnhWpD1agwpWpGG9H2brJG3FLgyn28+NP2eAeyZ8jkOWF0QU7R9xSVk\nNwJZkW6a8K3qAEmTJV1bpzzVzgN2z/XkdWZ44Xjg2Vorbf+ZrHH6I0nLyBoJ76kKq1emf6dj79Wl\ndeI7kHSIpCkFq/oDv5a0kuw6O852Za7d7sD6gm0q+gE3pXpcDEy1/ceCuMp1+2bK82Gyhs3qgth2\n2xQ4G9hD2c1RVgKnVgfUKmsawnoJ2ZcFvwEurhrOCjAg3ewi9CUxl6ecot5KqTfNCQnNi3orpzLW\nWzxoOCfN9dne9gUNgzcjkrYBrrddq3evtCTNrrotf5+W5gEut71znZh40HAIfdGUeNBwCCF0J9V4\n0HA0sHIkjQF+AmzcnD50h82DpGPI7og43fb/rRMX/yiE0AcNGz6M9evqddiHZrW2tpbyW/XNXdRb\nOfXmeqvVwOrJuwj2Oumuah/r6TxC2BTSg4ZvbjJ2E2cTultv/g8o1Bb1FkIIfU/0YIUQ2pHk+Hch\nhBBCCKG+Wj1YPXmTixBCCCGEEELoU6KBFUIIfUAZnxMSot7KKuqtnKLeyqmM9RYNrBBCCCGEEELo\nJjEHK4TQTszBCiGEEEJoLOZghRBCCCGEEMImFrdpDyF0IHX4MiaEsAlst90wXnopnk1VNnF7/XKK\neiunMtZbNLBCCAViiGD5tAItPZxD6KyXX44vM0IIoa+JOVghhHYkORpYIbxbFA/2DiGEkupVc7Ak\njZT0SI118yRNerdzSsceIWmJpNm5ZWt7IpdaJO0naVoTcWvT75rnuip+G0nPSPpBbtk8SSMabDdN\n0r4N8r2r0fE7I79PSSdKuqiJbd5KdbtU0p1NxF8k6YQG68/tZN67peMvljRa0lmSVkm6MZXjhw22\nb1hWSRMkPSjpEUnLJB2VW3eMpMckndOZvEMIIYQQQvN68iYXvfEru8OAObYPzi3rjXk2k5NrvK7l\nEmB+19LpVC6bYp/N7P9/bE+yPdH2YZsgn2YcBsy0Pdn2WuA04EDbx6f1na3XIv8DHG97d+Bg4P9J\n2hbA9s3AfkA0sPqk1p5OIITNRhmfyxOi3sqqjPXWkw2s/pJuSt/g3yppy+qA9I37ivRzeVq2Reo1\nWSFpuaSz0/Ixkuamb+0XSRrdhZwGA89XLXshl88J6ZhLJU1Py6ZJmirpAUlPSDo8Ld9a0r0pl+WS\nDk3LR0panbZbI2mGpIPS9msk7ZHitpJ0g6SFqcfjkJTGG8ArTZTlheoFkq5LuS+V9Lykb6Tlk4Ed\ngTlVm/wBeKvBcTaknJC0ZyrHspT31lXHLyyTpAWSxubi5kmaVOcc5L0GbGyQI0BnJzq8mvZN6ml6\nNJXrZ7mYD6Zcn5D0pRTbrsdQ0nmpt+tg4MvAaZLuk3QNsCswu3IN57YZKuk2Sb9JP3s3W1bbT9h+\nMr3+b7LreYfc+ueAQZ08FyGEEEIIoUk9eZOL3YAv2F4o6QbgdOCqykpJOwOXAxPJPsTPTY2UdcBw\n2+NT3LZpkxnAZbZnSRpA1xqP/YC2/ALbe6XjjAMuBPa2/bKkwbmwnWx/NDUSZgG3A38CDrO9UdL2\nwMK0DmAM8BnbqyQtAo5O2x+ajnE48DXgPtsnSRoEPCTpXtsLgAUpp8nAqbZPqS5IJe+qZSen7UYA\ns4FpkgR8HzgWOKgq/ohGJ8z2OWmf/YFbgCNtL5E0kNRAySksU9rus8AUSTul87lE0rdrxOePf2vl\ndWqATbY9pSDV96Rz/QbwXdv/0aBcV+XefhUYZfvPuesNsmu4hazBskbS1ZXNO+7OsyX9GHi1sm9J\nnwRa0vV0Yi5+KnCV7QclvQ+4BxjXibJWYj4M9K80uHKa+NvI77aFuHlCGbT0dAIhbDbKdkezkIl6\nK6feVG+tra1N9aj1ZAPradsL0+ubgC+Ra2ABewLzbL8EIGkGsC9wKTBa0lTgl8Cc9GF+F9uzAGy/\n0dlkUkNjQsqlyMfJhne9nI6xIbfuzrRstaQdK7sEvqNsflIbsEtu3Vrbq9LrR4FKo+ERYFR6/Qng\nEElfSe8HACOANZWD2l4MdGhcNSjnlsBM4Ezb6ySdAdxt+9nsFHS6p6diN+BZ20tSbhvT8fIxtco0\nk6z3bApwFHBbg/hCtu8Cas33Gmn7v1PP5v2SVqRhes1YDvxM2dyt/Pytu22/CfxB0nPAsCb3VyGK\nz/eBwFj95eQNlLSV7f+tBDQoa+ULip8CxxesfknSmIKGV86UhsmHEEIIIWxOWlpa2jX4Lr744sK4\n3jQHq2huSYcPn6lhM4FswsEXgetqxbbbkXR6Ghq3JPWS5NdtAawFxgJ3N5V9e68X5HwsMBSYaHsi\n2VCtLQvi23Lv2/hLo1dkvVwT089o22t4564BbrM9L73fGzhT0lNkPVnHS7qsi/tu1DgrLJPtZ4EX\nJe1O1pP189w23XIO0nA5UqOqlaxntFn/CPwImAQ8nK4X6FiPfwW8SdYTWtFh6GsTBOyVK/eIfOOq\n4cbSNsAvgH+x/XBByFRgmaTPdyG30Gu19nQCIWw2yjgnJES9lVUZ660nG1gjJVWGsX0O+HXV+oeA\nfSUNkdQPOAaYn4bb9bN9B/B1YFLqLXlG0qcAJA2Q9N78zmxfnT6sTrK9vmpdm+1RwCKyD/hF7geO\nlDQkHWO7GnGVRsYg4HnbbZL2B0YWxNRzD3DW2xtIH2pim7pSb9VA21dUltk+zvYo27sC5wM/tX1h\nwbbTleaH1bAG2CkNW0TSwFRvefXK9HPgn4Ftba9sIr5pkganYaNIGgp8FFiV3l9WuW5qbCtghO35\nwAXAtsDAOod7DthB0naS3gP8UxdSngO8PS9L0oRmN0xDNe8Epqe/kSIXAu+3/ZMu5BZCCCGEEOro\nyQbWY8AZklaR3Vzix2m5AVIj6AKyr2WXAg+nYVHDgVZJS4EbUwzACcBZkpYDD9D54VoAjwNDilak\nIX3fJmvkLQWuzOebD02/ZwB7pnyOA1YXxBRtX3EJ2Y1AVqSbJnyrOkDSZEnX1ilPtfOA3XM9eZ0Z\nXjgeeLbWStt/Jmuc/kjSMrJGwnuqwuqV6d/p2Ht1aZ34DiQdImlKwaqxwKJUb/eRzdV7LK3bHVhf\nsE1FP+CmVI+Lgam2/1gQV7lu30x5PkzWQFxdENtumwJnA3souznKSuDU6oA6ZT0K2Af4fK6ex1fF\nDEg3uwh9SktPJxDCZqM3zQkJzYt6K6cy1ls8aDgnzfXZ3vYFDYM3I2nI2fW2a/XulZak2VW35e/T\n0jzA5bZ3rhMTDxoO4V0TDxoOIYSyUm960HAvdjvwUeUeNBzA9qt9sXEFsJk1ro4h61n8Xk/nEjaF\n1p5OIITNRhnnhISot7IqY7315F0Ee510V7WP9XQeIWwK6UHDNzcX3dWbSYYQOmO77boymj2EEEJv\nFkMEQwjtSHL8uxBCCCGEUF8MEQwhhBBCCCGETSwaWCGE0AeUcYx6iHorq6i3cop6K6cy1ls0sEII\nIYQQQgihm8QcrBBCOzEHK4QQQgihsZiDFUIIIYQQQgibWDSwQgihDyjjGPUQ9VZWUW/lFPVWTmWs\nt3gOVgihAymegxVCCGHzNWz4MNavW9/TaYSSijlYIYR2JJkpPZ1FCCGE0IOmQHxGDo3EHKwQQggh\nhBBC2MR6pIElaaSkR2qsmydp0rudUzr2CElLJM3OLVvbE7nUImk/SdOaiFubftc811Xx20h6RtIP\ncsvmSRrRYLtpkvZtkO9djY7fGfl9SjpR0kVNbPNWqtulku5sIv4iSSc0WH9uJ/PeLR1/saTRks6S\ntErSjakcP2ywfbNlPVHS45LW5Msg6RhJj0k6pzN5h5LoVf9ShaZFvZVT1Fs5Rb2VUszB6pze2O96\nGDDH9gW5Zb0xz2Zyco3XtVwCzO9aOp3KZVPss5n9/4/tHmm45xwGzLR9GYCk04ADbD8r6UQ6X68d\nSNoO+CYwCRCwWNJ/2H7F9s2S7gceBv7vOylICCGEEEIo1pNDBPtLuil9g3+rpC2rA9I37ivSz+Vp\n2Rap12SFpOWSzk7Lx0iaK2mZpEWSRnchp8HA81XLXsjlc0I65lJJ09OyaZKmSnpA0hOSDk/Lt5Z0\nb8pluaRD0/KRklan7dZImiHpoLT9Gkl7pLitJN0gaWHq8TgkpfEG8EoTZXmheoGk61LuSyU9L+kb\naflkYEdgTtUmfwDeanCcDSknJO2ZyrEs5b111fELyyRpgaSxubh5kibVOQd5rwEbG+QIWWOjM15N\n+yb1ND2ayvWzXMwHU65PSPr/2Tv3cC2rMv9/vpCIipKHBGUGNKbL0UlN0J85Org1D6l5GJOMNK3x\nMhtNOziao05hmodKr/E3jlrpoCNoanlAiUSMTaYiChvQQLKiPA1ooxSa5/39/fGsF5797vf82/ju\nB+7Pde1rP+9a91rrXut+2Dz3c99rvacn2R4RQ0lnpmjXIcBXgH+W9ICka4APAtNL93CuzVaSfizp\n0fSzVxNzPZjsJcGfbK8ks+nHS5W2VwBDm1yLoAi08hcvaD9ht2ISdismYbdC0tHR0W4VmqadEawd\ngM/bniPpeuBU4IpSpaRtgEuB3cge4u9PTspzwAjbuyS5zVKTKcDFtqdKGkRrzuNAoDtfYHvPNM5O\nwLnAXrZfkfT+nNhw23snJ2EqcAfwBnCU7VclbQnMSXUAo4FP2l4s6XHg06n9EWmMo4HzgAdsnyRp\nKDBX0kzbjwCPJJ3GAqfY/kL5REp6l5WdnNqNBKYDkyQJ+B5wHHBgmfwx9RbM9ldTnxsAPwLG254v\naQjJQclRcU6p3bHAREnD03rOl/TtKvL58W8rXScHbKztiRVU3TCt9VvAZbbvrjOvK3Ifvw5sZ/vt\n3P0G2T3cQeawLJV0dal57+48XdK1wKpS35IOBjrS/XRiTv5K4ArbD0v6a+A+YKcG5zoCeDb3+flU\nlqf+v41ZuevtiP+UgiAIgiBY7+ns7GwoZbGdDtYztuek68nA6eQcLGAPYJbtlwEkTQHGARcB20u6\nEvgpMCM9zG9reyqA7beaVSY5GrsmXSqxP1l61ytpjJW5urtS2RJJW5e6BC5Rtj+pG9g2V7fM9uJ0\n/Sug5DQ8QfY4C3AQcLiks9LnQcBIYGlpUNvzgF7OVZ15DgZuB75k+zlJpwHTUppaSe9W2AF4wfb8\npNuraby8TLU53U4WaZkIfAr4cR35iti+B6i232uU7f9Jkc2fS1pku9Fs7IXAzcr2buX3b02z/Q7w\nv5JWAMMa7K+EqLzeBwA7as3iDZG0se2/lATqzLUeL0sabfu3VSX2a7HnoH0sIxzhIhJ2KyZht2IS\ndisknZ2d/SaK1dHR0UOXCy64oKJcf9qDVWlvSa+HT9srJe1Klgr1RWA8WepVTcdA0qnAyWmcQ20v\nz9UNAH4HvAlMa2IOJd6soPNxwFbAbra7lR06MbiCfHfuczdrbCKyKNfTLehTi2uAH9suxSj2AvZJ\n67MpWermKtvnttB3Pees6pwk/VHSzmSRrFNyVb3kU5SrKWz/T/q9TFInWWS0UQfrMDLn/gjgPEkf\nTuXldnwf8A5ZJLREr9TXBhCwp+23W2j7PFlUrcRf0TMeBVmEbIGk023f0MIYQRAEQRAEQRXauQdr\nlKRSGttngAfL6ucC4yRtIWkgMAGYndLtBtq+EzgfGJOiJc9KOhJA0iBJG+U7s3217d1sj8k7V6mu\n2/Z2wONkD/iV+DkwXtIWaYzNq8iVnIyhwIvJudoPGFVBphb3AWesbiB9pIE2NUnRqiG2v1sqs328\n7e1sfxD4F+C/KzlXkm5U2h9WhaXA8JS2iKQhyW55as3pVuBsYDPbTzYg70Ac1gAAIABJREFU3zCS\n3p/SRpG0FbA3sDh9vrh031RpK2Ck7dnAOcBmwJAaw60APiBpc0kbAp9oQeUZwOp9WemFQqPcBxwo\naWi6Rw9MZXnOBf4mnKt1jHgrW0zCbsUk7FZMwm6FpL9Er5qhnQ7WU8BpkhaTHS5xbSo3QHKCzgE6\ngS7gsZQWNQLolNQF3JRkAE4AzpC0EHiI5tO1AH4NbFGpIqX0fZvMyesCLs/rmxdNv6cAeyR9jgeW\nVJCp1L7EhWTRpEXp0IRvlQtIGivpBzXmU86ZwM7KDrmYL6mZ9MJdgBeqVaZoy7HAVZIWkDkJG5aJ\n1ZrTT1L7W3NlF9WQ74WkwyVNrFC1I/B4stsDZHv1nkp1OwO1vqp9IDA52XEecKXtP1eQK9237yQ9\nHyNzbJZUkO3RpgJfBnZXdjjKk/SM6AHV55pSWC8ke1nwKHBBWTorwKB02EUQBEEQBEHQxyi+pXoN\naa/PlmXHtK/3SNoUuM52teheYZE03fYh7dbjvSLtA1xoe5saMmbie6dT0EfE3oJiEnYrJmG3YtKM\n3SZCPCP3D/rTHqxyJGG7V2ZaO/dg9UfuAG5Y3x6662F7FdVTJwvN+mRnSRPITkT8Tl3hiWtbmyAI\ngiDovwwb0UoiVBBkRAQrCIIeSHL8XQiCIAiCIKhNtQhWO/dgBUEQBEEQBEEQrFOEgxUEQbAO0MgX\nHwb9j7BbMQm7FZOwWzEpot3CwQqCIAiCIAiCIOgjYg9WEAQ9iD1YQRAEQRAE9Yk9WEEQBEEQBEEQ\nBGuZcLCCIAjWAYqYox6E3YpK2K2YhN2KSRHtFt+DFQRBL6Re0e4gCNrIsGGjWL789+1WIwiCIGiA\n2IMVBEEPJBni70IQ9C9E/H8dBEHQv4g9WEEQBEEQBEEQBGuZtjhYkkZJeqJK3SxJY95rndLYIyXN\nlzQ9V7asHbpUQ9K+kiY1ILcs/a661mXym0p6VtL/zZXNkjSyTrtJksbV0feeeuM3Q75PSSdK+mYD\nbaZLekXS1AbH+KakE+rUf61xrUHSDpK6JM2TtL2kMyQtlnRTmsd/1Glfd66SdpX0sKQnJC2Q9Klc\n3QRJT0n6ajN6B0Whs90KBC3R2W4FghYo4p6QIOxWVIpot3ZGsPpjrsNRwAzbh+TK+qOejejkKtfV\nuBCY3Zo6TemyNvpspP/vAMevBT2a4SjgdttjbS8D/hk4wPZnU32zdq3Ea8Bnbe8MHAL8u6TNAGzf\nAuwLhIMVBEEQBEGwlming7WBpMnpDf5tkgaXC6Q37ovSz6WpbECKmiyStFDSl1P5aEn3p7f2j0va\nvgWd3g+8WFb2Uk6fE9KYXZJuTGWTJF0p6SFJv5F0dCrfRNLMpMtCSUek8lGSlqR2SyVNkXRgar9U\n0u5JbmNJ10uakyIehyc13gL+1MBcXiovkPTDpHuXpBcl/VsqHwtsDcwoa/K/wLt1xlmZdELSHmke\nC5Lem5SNX3FOkh6RtGNObpakMTXWIM/rwKt1dMT2rEbkcqxKfZMiTb9K87o5J/N3SdffSDo9yfaI\nGEo6M0W7DgG+AvyzpAckXQN8EJheuodzbbaS9GNJj6afvRqdq+3f2P5tuv4fsvv5A7n6FcDQJtYh\nKAwd7VYgaImOdisQtEBHR0e7VQhaIOxWTIpot3aeIrgD8HnbcyRdD5wKXFGqlLQNcCmwG9lD/P3J\nSXkOGGF7lyS3WWoyBbjY9lRJg2jNeRwIdOcLbO+ZxtkJOBfYy/Yrkt6fExtue+/kJEwF7gDeAI6y\n/aqkLYE5qQ5gNPBJ24slPQ58OrU/Io1xNHAe8IDtkyQNBeZKmmn7EeCRpNNY4BTbXyifSEnvsrKT\nU7uRwHRgkiQB3wOOAw4skz+m3oLZ/mrqcwPgR8B42/MlDSE5KDkqzim1OxaYKGl4Ws/5kr5dRT4/\n/m2l6+SAjbU9sZ7eDczritzHrwPb2X47d79Bdg93kDksSyVdXWreuztPl3QtsKrUt6SDgY50P52Y\nk78SuML2w5L+GrgP2KnZuUr6P8AGJYcrRwP/NvLddhAPgUEQBEEQrO90dnY2lLLYTgfrGdtz0vVk\n4HRyDhawBzDL9ssAkqYA44CLgO0lXQn8FJiRHua3tT0VwPZbzSqTHI1dky6V2J8sveuVNMbKXN1d\nqWyJpK1LXQKXKNuf1A1sm6tbZntxuv4VUHIangC2S9cHAYdLOit9HgSMBJaWBrU9D+jlXNWZ52Dg\nduBLtp+TdBowzfYL2RLQ6vncOwAv2J6fdHs1jZeXqTan28miZxOBTwE/riNfEdv3AH263yuxELhZ\n0l0kWyem2X4H+F9JK4BhTfYrKq/3AcCOWrN4QyRtbPsvJYF6c00vKP4b+GyF6pclja7geOWYWFf5\noL/RSTjCRaSTsFvx6OzsLORb9fWdsFsx6U926+jo6KHLBRdcUFGunQ5Wr7f8FWR6PXzaXilpV+Bg\n4IvAeLLUq5qOgaRTgZPTOIfaXp6rGwD8DngTmNbEHEq8WUHn44CtgN1sdys7dGJwBfnu3Odu1thE\nZFGup1vQpxbXAD9OKXMAewH7pPXZlCx1c5Xtc1vou55zVnVOkv4oaWeySNYpuape8inK9V5yGJlz\nfwRwnqQPp/JyO74PeIcsElqiV+prAwjY0/bbLbRF0qbAvcC/2n6sgsiVwAJJp9u+oZUxgiAIgiAI\ngsq0cw/WKEmlNLbPAA+W1c8FxknaQtJAYAIwO6XbDbR9J3A+MCZFS56VdCSApEGSNsp3Zvtq27vZ\nHpN3rlJdt+3tgMfJHvAr8XNgvKQt0hibV5ErORlDgReTc7UfMKqCTC3uA85Y3UD6SANtapKiVUNs\nf7dUZvt429vZ/iDwL8B/V3KuJN2otD+sCkuB4SltEUlDkt3y1JrTrcDZwGa2n2xAvhV6RYwkXVy6\nbyo2yKJII23PBs4BNgOG1BhjBfABSZtL2hD4RAt6zgBW78tKLxQaIqVq3gXcmP6NVOJc4G/CuVrX\n6Gi3AkFLdLRbgaAF+svb9KA5wm7FpIh2a6eD9RRwmqTFZIdLXJvKDZCcoHPI8ie6gMdSWtQIoFNS\nF3BTkgE4AThD0kLgIZpP1wL4NbBFpYqU0vdtMievC7g8r29eNP2eAuyR9DkeWFJBplL7EheSRZMW\npUMTvlUuIGmspB/UmE85ZwI7KzvkYr6kZtILdwFeqFaZoi3HAldJWkDmJGxYJlZrTj9J7W/NlV1U\nQ74Xkg6XNLFK3S9S3/tLekZSab/ZzsDySm0SA4HJyY7zgCtt/7mCXOm+fSfp+RiZg7ikgmyPNhX4\nMrC7ssNRnqRnRK80n2pz/RSwD/C5nJ13KZMZlA67CIIgCIIgCPoYxTfDryHt9dnS9jl1hdcjUsrZ\ndbarRfcKi6TpZcfyr9OkfYALbW9TQ8b989sJgtp0EtGQItJJY3YT8f91/6E/7QkJGifsVkz6s90k\nYbtXZlo792D1R+4AbljfHrrrYXsV1VMnC836ZGdJE8hORPxOA9JrW50gCJpg2LBR9YWCIAiCfkFE\nsIIg6IEkx9+FIAiCIAiC2lSLYLVzD1YQBEEQBEEQBME6RThYQRAE6wCNfPFh0P8IuxWTsFsxCbsV\nkyLaLRysIAiCIAiCIAiCPiL2YAVB0IPYgxUEQRAEQVCf2IMVBEEQBEEQBEGwlgkHKwiCYB2giDnq\nQditqITdiknYrZgU0W7xPVhBEPRCiu/BCoIgCIK+ZNiIYSx/bnm71QjeA2IPVhAEPZBkJrZbiyAI\ngiBYx5gI8dy9bhF7sIIgCIIgCIIgCNYybXGwJI2S9ESVulmSxrzXOqWxR0qaL2l6rmxZO3SphqR9\nJU1qQG5Z+l11rcvkN5X0rKT/myubJWlknXaTJI2ro+899cZvhnyfkk6U9M0G2kyX9IqkqQ2O8U1J\nJ9Sp/1rjWoOkHSR1SZonaXtJZ0haLOmmNI//qNO+0bmeKOnXkpbm5yBpgqSnJH21Gb2DgtCv/lIF\nDRN2KyZht2ISdiskRdyD1c4IVn+MkR4FzLB9SK6sP+rZiE6ucl2NC4HZranTlC5ro89G+v8OcPxa\n0KMZjgJutz3W9jLgn4EDbH821Tdr115I2hz4BrAHsCfwTUlDAWzfAuwLhIMVBEEQBEGwlming7WB\npMnpDf5tkgaXC6Q37ovSz6WpbECKmiyStFDSl1P5aEn3S1og6XFJ27eg0/uBF8vKXsrpc0Ias0vS\njalskqQrJT0k6TeSjk7lm0iamXRZKOmIVD5K0pLUbqmkKZIOTO2XSto9yW0s6XpJc1LE4/CkxlvA\nnxqYy0vlBZJ+mHTvkvSipH9L5WOBrYEZZU3+F3i3zjgrk05I2iPNY0HSe5Oy8SvOSdIjknbMyc2S\nNKbGGuR5HXi1jo7YntWIXI5VqW9SpOlXaV4352T+Lun6G0mnJ9keEUNJZ6Zo1yHAV4B/lvSApGuA\nDwLTS/dwrs1Wkn4s6dH0s1cTcz2Y7CXBn2yvJLPpx3PrsAIY2sQ6BEWhlb94QfsJuxWTsFsxCbsV\nko6Ojnar0DTtPEVwB+DztudIuh44FbiiVClpG+BSYDeyh/j7k5PyHDDC9i5JbrPUZApwse2pkgbR\nmvM4EOjOF9jeM42zE3AusJftVyS9Pyc23PbeyUmYCtwBvAEcZftVSVsCc1IdwGjgk7YXS3oc+HRq\nf0Qa42jgPOAB2yelCMRcSTNtPwI8knQaC5xi+wvlEynpXVZ2cmo3EpgOTJIk4HvAccCBZfLH1Fsw\n219NfW4A/AgYb3u+pCEkByVHxTmldscCEyUNT+s5X9K3q8jnx7+tdJ0csLG2J9bTu4F5XZH7+HVg\nO9tv5+43yO7hDjKHZamkq0vNe3fn6ZKuBVaV+pZ0MNCR7qcTc/JXAlfYfljSXwP3ATs1ONcRwLO5\nz8+nsjz1/23Myl1vR/ynFARBEATBek9nZ2dDKYvtdLCesT0nXU8GTifnYJGlOM2y/TKApCnAOOAi\nYHtJVwI/BWakh/ltbU8FsP1Ws8okR2PXpEsl9idL73oljbEyV3dXKlsiaetSl8AlyvYndQPb5uqW\n2V6crn8FlJyGJ8geZwEOAg6XdFb6PAgYCSwtDWp7HtDLuaozz8HA7cCXbD8n6TRgmu0XsiWg1fO5\ndwBesD0/6fZqGi8vU21Ot5NFWiYCnwJ+XEe+IrbvAfp0v1diIXCzpLtItk5Ms/0O8L+SVgDDmuxX\nVF7vA4AdtWbxhkja2PZfSgL/n3N9WdJo27+tKrFfiz0H7WMZ4QgXkbBbMQm7FZOwWyHp7OzsN1Gs\njo6OHrpccMEFFeXa6WD1estfQabXw6ftlZJ2JUuF+iIwniz1qqZjIOlU4OQ0zqG2l+fqBgC/A94E\npjUxhxJvVtD5OGArYDfb3coOnRhcQb4797mbNTYRWZTr6Rb0qcU1wI9TyhzAXsA+aX02JUvdXGX7\n3Bb6ruecVZ2TpD9K2pksknVKrqqXfIpyvZccRubcHwGcJ+nDqbzcju8D3iGLhJbolfraAAL2tP12\nC22fJ4uqlfgresajIIuQLZB0uu0bWhgjCIIgCIIgqEI792CNklRKY/sM8GBZ/VxgnKQtJA0EJgCz\nU7rdQNt3AucDY1K05FlJRwJIGiRpo3xntq+2vZvtMXnnKtV1294OeJzsAb8SPwfGS9oijbF5FbmS\nkzEUeDE5V/sBoyrI1OI+4IzVDaSPNNCmJilaNcT2d0tlto+3vZ3tDwL/Avx3JedK0o1K+8OqsBQY\nntIWkTQk2S1PrTndCpwNbGb7yQbkW6FXxEjSxaX7pmKDLIo00vZs4BxgM2BIjTFWAB+QtLmkDYFP\ntKDnDGD1vqz0QqFR7gMOlDQ03aMHprI85wJ/E87VOka8lS0mYbdiEnYrJmG3QtJfolfN0E4H6yng\nNEmLyQ6XuDaVGyA5QecAnUAX8FhKixoBdErqAm5KMgAnAGdIWgg8RPPpWgC/BraoVJFS+r5N5uR1\nAZfn9c2Lpt9TgD2SPscDSyrIVGpf4kKyaNKidGjCt8oFJI2V9IMa8ynnTGBnZYdczJfUTHrhLsAL\n1SpTtOVY4CpJC8ichA3LxGrN6Sep/a25sotqyPdC0uGSJlap+0Xqe39Jz0gq7TfbGaj1teoDgcnJ\njvOAK23/uYJc6b59J+n5GJljs6SCbI82FfgysLuyw1GepGdErzSfinNNKawXkr0seBS4oCydFWBQ\nOuwiCIIgCIIg6GMU3yi9hrTXZ0vb59QVXo+QtClwne1q0b3CIml62bH86zRpH+BC29vUkDET3zud\ngj4i9hYUk7BbMQm7FZN2220ixHN38/SnPVjlSMJ2r8y0du7B6o/cAdywvj1018P2KqqnThaa9cnO\nkiaQnYj4nbrCE9e2NkEQBEGwfjFsRCvJVUERiQhWEAQ9kOT4uxAEQRAEQVCbahGsdu7BCoIgCIIg\nCIIgWKcIBysIgmAdoJEvPgz6H2G3YhJ2KyZht2JSRLuFgxUEQRAEQRAEQdBHxB6sIAh6EHuwgiAI\ngiAI6hN7sIIgCIIgCIIgCNYy4WAFQRCsAxQxRz0IuxWVsFsxCbsVkyLaLb4HKwiCXki9ot3BOsaw\nYaNYvvz37VYjCIIgCNY5Yg9WEAQ9kGSIvwvrPiL+/gdBEARB68QerCAIgiAIgiAIgrVMQw6WpFGS\nnqhSN0vSmL5VqzEkjZQ0X9L0XNmyduhSDUn7SprUgNyynPw91WQkbdGHelUcJ1dfU+90X8yqI9Pn\n90e+z0bsLWkXSQ9LWijpbklDGmhTs19JqxrXeHWb70p6QtJlkraSNEfSPEn7NGLbBuf6HUlLJC2Q\n9BNJm+XqfiFprqStm9U9KAKd7VYgaIEi7i0Iwm5FJexWTIpot2YiWP0xl+QoYIbtQ3Jl/VHPRnRy\nletm+2mGev01q3c7aGT864Czbe8K3Amc3Qf9tjLvk4FdbH8dOABYZHus7V822F8jMjOAv7P9EeBp\n4F9XN7bHAfOAw5rWPAiCIAiCIGiIZhysDSRNlrRY0m2SBpcLSJogaVH6uTSVDZA0KZUtlPTlVD5a\n0v3pTfvjkrZvQf/3Ay+Wlb2U0+eENGaXpBtT2SRJV0p6SNJvJB2dyjeRNDPpslDSEal8VIoITJK0\nVNIUSQem9ksl7Z7kNpZ0fS4qcXhS4y3gTw3M5aXc9VBJ90p6StLVufLVOZ6SvpaiIYtya7pxateV\nysen8j2SvguSfpvkB5Y0LUUCuyStlPTZBvV+F3g59TEgF6FZIOm0cuG0bg+nNb416XuwpNtyMqsj\na5IOKpevs27V+FByYgBmAp9soM1LSYfhkman9Vkkae81quqiNNeHJX0gFU4q3VPp86r0+25gCDBP\n0tnAZcBRqd/B9LTtcZIeTXXXSKtPnKg7V9szbXenj3OAvyoTWU727yZY5+hotwJBC3R0dLRbhaAF\nwm7FJOxWTIpot2ZOEdwB+LztOZKuB04FrihVStoGuBTYDVgJ3J+clOeAEbZ3SXKllKUpwMW2p0oa\nRGv7wQYC3fkC23umcXYCzgX2sv2KpPxD5XDbe0vaEZgK3AG8ARxl+1VJW5I9nE5N8qOBT9peLOlx\n4NOp/RFpjKOB84AHbJ8kaSgwV9JM248AjySdxgKn2P5C+URKeif2AHYEngHuk3S07TtKlcrS405M\ncgOBRyV1Jj2ft/2JJLeppA2AHwHjbc9Xlh73etnYh+X6/S/gLturSnpXw/ZzwDHp4xeAUWQRGpet\nN2lNzwc+Zvv15GR8DbgE+L6kjWy/DhwL3Jzkz6sgf1G1dZM0DTjJ9vIyVX8l6QjbU4FP0dvpqDS3\nUr+fAX5m+5Lk6JScvE2Ah22fL+kysujUxZW6Sv0dKenPtkupjSuAsbbPSJ9Lc/jbtAZ/b/tdSf8J\nHAdMbnCuef6JzPZ5usnumTpMzF13EA/vQRAEQRCs73R2djaUstiMU/OM7TnpejKwT1n9HsAs2y+n\nN+hTgHHA74DtU9ToYGBVesjfNj3wYvst2280oQvpYXdXMgeuEvsDt9t+JY2xMld3VypbApT2owi4\nRNJCsijHtlqzV2WZ7cXp+lepHuAJYLt0fRBwjqQuss0Qg4CReYVsz6vkXFVgru0/ODvi6xZ6r/U+\nwJ2237D9GpmD+A9JnwMlXSJpn+Qk7QC8YHt+0uHVXIRjNZK2Am4CJqR2zXIA8P2kc/l6A3wU2Al4\nKK3RCcBI2+8CPwMOlzSQLH1tajX5WgrYPqyKw/FPwGmSHiNzjN5qYl6PAZ+X9A0y5/G1VP6m7Z+m\n63msuQ/KafS881L638eAMcBjad77Ax/sJVx9rtmg0nnA27ZvLqt6HtilvjoTcz8d9cWDfkBnuxUI\nWqCIewuCsFtRCbsVk/5kt46ODiZOnLj6pxrNRLDK939U2g/S62HS9kpJuwIHA18ExgNfqSTboyPp\nVLKogIFD8w+TkgaQOW5vAtOamEOJNyvofBywFbCb7W5lBwoMriDfnfvczZo1FFmU6+kW9CmnkbXu\n3ch+OkWhDgUulPQAmTNZb60HkDlyE5PTuTYQ2X654yrU3Qp8CXgFeMz2a8mBribfFLZ/TXb/IelD\nNLEHyfaDksalNjdIutz2ZODtnNi7rLkP3iG9uEhz2KBJdQXcaPu8Jtut6UD6HNk9sH+F6juAb0ha\nbHunVscIgiAIgiAIKtNMBGuUpHza1INl9XOBcZK2SJGICcDslOo10PadZCliY2y/Cjwr6UgASYMk\nbZTvzPbVtnezPab8Tb3tbtvbAY+TpVNV4ufAeKWT2SRtXkWu5HwMBV5MztV+ZOlu5TK1uA84Y3UD\n6SMNtKnGnsr2fg0gm1/5Wj9Itn9nsLL9VP8IPJjSNF9PUYvvkUVClgLDU3oikoYk++S5DFho+/ZK\nyijbw3VjHZ3vB04p9V1hvecAe0saneo3Ts4OwOyk68msSWmrJd8Uuf1RA8juwWvT520lzazTdiTZ\nfXE92WEZpRMRq90Tvwd2T9dH0tPBqnUfleoeAI7J6bx50qEhJH0cOAs4wvabFUROAKaHc7Uu0tFu\nBYIWKOLegiDsVlTCbsWkiHZrxsF6iizNajHZJvlrU3kpJWw5cA5ZnkoXWSTiHmAE0JnSnW5KMpA9\n6J2RUvIeAoa1oP+vgYpHW6eUvm+TOXldwOV5ffOi6fcUYI+kz/HAkgoyldqXuJDsIJBFyo60/1a5\ngKSxkn5QYz4l5gJXkaUj/tb2XfmxbXcBN5Clrz0C/MD2QmBnsr1fXcA3gItsv03mpF0laQHZKXMb\nlo13JnCQskMu5kv6RFn9SOAvdXS+DngWWJTGn1Cm8x+BzwG3pDV+mCx9kZSyeC/w8fS7pjxVbKDs\nsI7hFaomSFoKLCbbo3ZDKt+GnpGoSnQACyXNJ9u/9e+1dAB+COyb1uCjwGu5ulqRyNI6LSFzAmek\nec8Aes2pxlz/g+wwjfuTLa8uq9+c7HTBIAiCIAiCYC2gtGWmkEg6C9jS9jl1hYOWSYc43GT7yXbr\n0pcoO+nwD7bvbbcu7xXp0IxFtr9fQ8btP30/aJ5OmotiiSL//V9X6OzsLOTb2fWdsFsxCbsVk/5s\nN0nY7pWh1MwerP7IHWT7YqaXfRdW0Iek721a57D9n+3W4b1E0myyfYOVTjsMgiAIgiAI+oBCR7CC\nIOh7sghWsK4zbNgoli//fbvVCIIgCILCsq5GsIIgWAvEi5cgCIIgCILWaOXLfYMgCIJ+Rn/6npCg\nccJuxSTsVkzCbsWkiHYLBysIgiAIgiAIgqCPiD1YQRD0QJLj70IQBEEQBEFtqu3BighWEARBEARB\nEARBHxEOVhAEwTpAEXPUg7BbUQm7FZOwWzEpot3CwQqCIAiCIAiCIOgjYg9WEAQ9iO/BCoKgGsNG\nDGP5c8vbrUYQBEG/oNoerHCwgiDogSQzsd1aBEHQL5kY35MXBEFQIg65CIIgWJdZ1m4FgpYIuxWS\nIu4JCcJuRaWIdmvIwZI0StITVepmSRrTt2o1hqSRkuZLmp4r61f/XUnaV9KkBuSW5eTvqSYjaYs+\n1KviOLn6mnqn+2JWHZk+vz/yfTZib0m7SHpY0kJJd0sa0kCbmv1KWtW4xqvbfFfSE5Iuk7SVpDmS\n5knapxHbNjjXzSXNkLRU0n2ShubqfiFprqStm9U9CIIgCIIgaIxmIlj9MSfgKGCG7UNyZf1Rz0Z0\ncpXrZvtphnr9Nat3O2hk/OuAs23vCtwJnN0H/bYy75OBXWx/HTgAWGR7rO1fNthfIzLnADNt7wD8\nHPjX1Y3tccA84LCmNQ/6P9u3W4GgJcJuhaSjo6PdKgQtEHYrJkW0WzMO1gaSJktaLOk2SYPLBSRN\nkLQo/VyaygZImpTKFkr6ciofLel+SQskPS6plf9m3g+8WFb2Uk6fE9KYXZJuTGWTJF0p6SFJv5F0\ndCrfRNLMpMtCSUek8lGSlqR2SyVNkXRgar9U0u5JbmNJ1+eiEocnNd4C/tTAXF7KXQ+VdK+kpyRd\nnStfneMp6WspGrIot6Ybp3ZdqXx8Kt8j6bsg6bdJfmBJ01IksEvSSkmfbVDvd4GXUx8DchGaBZJO\nKxdO6/ZwWuNbk74HS7otJ7M6sibpoHL5OutWjQ8lJwZgJvDJBtq8lHQYLml2Wp9FkvZeo6ouSnN9\nWNIHUuGk0j2VPq9Kv+8GhgDzJJ0NXAYclfodTE/bHifp0VR3jaRSXSNzPRK4MV3fSPYSIs9ysn83\nQRAEQRAEwVrgfU3I7gB83vYcSdcDpwJXlColbQNcCuwGrATuT07Kc8AI27skuc1SkynAxbanShpE\na/vBBgLd+QLbe6ZxdgLOBfay/Yqk/EPlcNt7S9oRmArcAbwBHGX7VUlbAnNSHcBo4JO2F0t6HPh0\nan9EGuNo4DzgAdsnpbSsuZJm2n4EeCTpNBY4xfYXyidS0juxB7Aj8Axwn6Sjbd9RqlSWHndikhsI\nPCqpM+n5vO1PJLlNJW0A/AgYb3u+svS418vGPizX738Bd9leVdKW2VSCAAAgAElEQVS7GrafA45J\nH78AjCKL0LhsvUlrej7wMduvJyfja8AlwPclbWT7deBY4OYkf14F+YuqrZukacBJtsuPuPqVpCNs\nTwU+BfxVrXmV9fsZ4Ge2L0mOTsnJ2wR42Pb5ki4ji05dXKmr1N+Rkv5su5TauAIYa/uM9Lk0h79N\na/D3tt+V9J/AccDkBue6te0Vaczl6p0O2E12z9Qmn/i5HfGWvQgsI+xURMJuhaSzs7OQb9XXd8Ju\nxaQ/2a2zs7OhPWHNOFjP2J6TricDp5NzsMge9mfZLkU0pgDjyB6It5d0JfBTYEZ6yN82PfBi+60m\n9CD1L2DXpEsl9gdut/1KGmNlru6uVLYk9wAq4BJJ48geQrfN1S2zvThd/4osCgLwBNnjJ8BBwOGS\nzkqfBwEjgaWlQW3PI3NE6jHX9h/SPG8B9iFzAkvsA9xp+40kcwfwD8B9wPckXQJMs/1LSR8GXrA9\nP+nwamrTY0BJWwE3Acck56pZDgCucTpeqmy9AT4K7AQ8lGy3AZmD8q6kn5Gt3U/I0tfOAjoqyddS\noOQoVuCfgP+Q9G9kTnMz99tjwPXJUb3b9sJU/qbtn6breWTzr0Svk2WqUEr/+xgwBngszXswsKKX\ncPW5Vuu3xPNka1ub/RrsPQiCIAiCYD2ho6Ojh7N3wQUXVJRrxsEqf1CrtB+k18Ok7ZWSdgUOBr4I\njAe+Ukm2R0fSqWRRAQOH5t/USxoA/A54E5jWxBxKvFlB5+OArYDdbHcrO1BgcAX57tznbtasocii\nXE+3oE85jax170b20ykKdShwoaQHyJzJems9ALgFmGh7SQv6NoLI9ssdV6HuVuBLwCvAY7ZfS85F\nNfmmsP1rsvsPSR+iiT1Ith9MTvdhwA2SLrc9GXg7J/Yua+6Dd0jR2Jxj2AwCbrR9XpPtSqyQNMz2\nCknD6Z1CewfwDUmLbe/U4hhBfySiIMUk7FZI+svb9KA5wm7FpIh2ayYtb5SkfNrUg2X1c4FxkraQ\nNBCYAMxOqV4Dbd9JliI2JkVRnpV0JICkQZI2yndm+2rbu9keU54GZbvb9nbA42TpVJX4OTBe6WQ2\nSZtXkSs5H0OBF5NztR9Zulu5TC3uA85Y3UD6SANtqrGnsr1fA8jmV77WD5Lt3xmsbD/VPwIPpjTN\n123fDHyPLBKyFBie0hORNCTZJ89lwELbt1dSRtkerhsr1eW4Hzil1HeF9Z4D7C1pdKrfODk7ALOT\nrieTpTPWk2+K3P6oAWT34LXp87aSZtZpO5Lsvrie7LCM0omI1e6J3wO7p+sj6elg1bqPSnUPAMfk\ndN486dAoU4HPpesTgbvL6k8ApodzFQRBEARBsHZoxsF6CjhN0mKyTfLXpvJSSthyshPMOoEuskjE\nPcAIoFNSF1kK2jmp3QnAGZIWAg8Bw1rQ/9dAxaOtU0rft8mcvC7g8ry+edH0ewqwR9LneGBJBZlK\n7UtcSHYQyCJlR9p/q1xA0lhJP6gxnxJzgavI0hF/a/uu/Ni2u4AbyNLXHgF+kFLXdibb+9UFfAO4\nyPbbZE7aVZIWADOADcvGOxM4SNkhF/MlfaKsfiTwlzo6Xwc8CyxK408o0/mPZA/+t6Q1fphsXx+2\nu4F7gY+n3zXlqWIDZYd1DK9QNUHSUmAx2R61G1L5NvSMRFWiA1goaT7Z/q1/r6UD8ENg37QGHwVe\ny9XVikSW1mkJmRM4I817BtBrTjXmehlwYJrvx8j2RebZHOiLKGvQ3+hXX1ARNEzYrZAU8Xt5grBb\nUSmi3VTkb2RP+522tH1OXeGgZdIhDjfZfrLduvQlyk46/IPte9uty3tFOjRjke3v15AxE987nYI+\nIg5LKCZFs9tEKPJzQ1/RnzbdB40Tdism/dlukrDdK0Op6A7WaLJIzqtl34UVBEEZkmaT7Rs83vbz\nNeSK+0chCIK1yrARw1j+XPnhpUEQBOsn66SDFQRB3yPJ8XchCIIgCIKgNtUcrFa+eyoIgiDoZxQx\nRz0IuxWVsFsxCbsVkyLaLRysIAiCIAiCIAiCPiJSBIMg6EGkCAZBEARBENQnUgSDIAiCIAiCIAjW\nMuFgBUEQrAMUMUc9CLsVlbBbMQm7FZMi2i0crCAIgiAIgiAIgj4i9mAFQdCD+B6sIOjfDBs2iuXL\nf99uNYIgCNZ74nuwgiBoiMzBir8LQdB/EfF/dxAEQfuJQy6CIAjWaTrbrUDQEp3tViBogSLuCQnC\nbkWliHZryMGSNErSE1XqZkka07dqNYakkZLmS5qeK1vWDl2qIWlfSZMakFuWk7+nmoykLfpQr4rj\n5Opr6p3ui1l1ZPr8/sj32Yi9JX1T0nPpXpkv6eMNtKnZr6RVjWu8us13JT0h6TJJW0maI2mepH0a\nsW2Dc/2OpCWSFkj6iaTNcnW/kDRX0tbN6h4EQRAEQRA0RjMRrP6Yj3AUMMP2Ibmy/qhnIzq5ynWz\n/TRDvf6a1bsdNDr+FbbHpJ+f9UG/rcz7ZGAX218HDgAW2R5r+5cN9teIzAzg72x/BHga+NfVje1x\nwDzgsKY1DwpAR7sVCFqio90KBC3Q0dHRbhWCFgi7FZMi2q0ZB2sDSZMlLZZ0m6TB5QKSJkhalH4u\nTWUDJE1KZQslfTmVj5Z0f3rT/rik7VvQ//3Ai2VlL+X0OSGN2SXpxlQ2SdKVkh6S9BtJR6fyTSTN\nTLoslHREKh+VIgKTJC2VNEXSgan9Ukm7J7mNJV2fi0ocntR4C/hTA3N5KXc9VNK9kp6SdHWufHWO\np6SvpWjIotyabpzadaXy8al8j6TvgqTfJvmBJU1LkZ0uSSslfbZBvd8FXk59DMhFaBZIOq1cOK3b\nw2mNb036HizptpzM6siapIPK5eusWy165cfW4aWkw3BJs9P6LJK09xpVdVGa68OSPpAKJ5XuqfR5\nVfp9NzAEmCfpbOAy4KjU72B62vY4SY+mumsklerqztX2TNvd6eMc4K/KRJaT/bsJgiAIgiAI1gLv\na0J2B+DztudIuh44FbiiVClpG+BSYDdgJXB/clKeA0bY3iXJlVKWpgAX254qaRCt7QcbCHTnC2zv\nmcbZCTgX2Mv2K5LyD5XDbe8taUdgKnAH8AZwlO1XJW1J9nA6NcmPBj5pe7Gkx4FPp/ZHpDGOBs4D\nHrB9kqShwFxJM20/AjySdBoLnGL7C+UTKemd2APYEXgGuE/S0bbvKFUqS487MckNBB6V1Jn0fN72\nJ5LcppI2AH4EjLc9X9IQ4PWysQ/L9ftfwF22V5X0robt54Bj0scvAKPIIjQuW2/Smp4PfMz268nJ\n+BpwCfB9SRvZfh04Frg5yZ9XQf6iausmaRpwku3lFdT9UnIcHwfOtF3Tecz1+xngZ7YvSY5Oycnb\nBHjY9vmSLiOLTl1cqavU35GS/my7lNq4Ahhr+4z0uTSHv01r8Pe235X0n8BxwOQm5lrin8hsn6eb\n7J6pw8TcdQfxlr0IdBJ2KiKdhN2KR2dnZyHfqq/vhN2KSX+yW2dnZ0N7wppxsJ6xPSddTwZOJ+dg\nkT3sz7JdimhMAcaRPRBvL+lK4KfAjPSQv63tqQC232pCD1L/AnZNulRif+B226+kMVbm6u5KZUu0\nZj+KgEskjSN7CN02V7fM9uJ0/StgZrp+AtguXR8EHC7prPR5EDASWFoa1PY8MkekHnNt/yHN8xZg\nHzInsMQ+wJ2230gydwD/ANwHfE/SJcA027+U9GHgBdvzkw6vpjY9BpS0FXATcExyrprlAOAap6Ot\nytYb4KPATsBDyXYbkDko70r6Gdna/YQsfe0ssieOXvK1FCg5ihW4GvhWcvwuIrtvT2pwXo8B1ydH\n9W7bC1P5m7Z/mq7nkc2/Eo1Gzkrpfx8DxgCPpXkPBlb0Eq4+12xQ6Tzgbds3l1U9T0NPcxPriwRB\nEARBEKxHdHR09HD2LrjggopyzThY5fs/Ku0H6fUwaXulpF2Bg4EvAuOBr1SS7dGRdCpZVMDAofk3\n9ZIGAL8D3gSmNTGHEm9W0Pk4YCtgN9vdyg4UGFxBvjv3uZs1ayiyKNfTLehTTiNr3buR/XSKQh0K\nXCjpATJnst5aDwBuASbaXtKCvo0gsv1yx1WouxX4EvAK8Jjt15JzUU2+KWznU+t+CFQ93KNC2weT\n030YcIOky21PBt7Oib3LmvvgHVI0NucYNoOAG22f12S7NR1InyO7B/avUH0H8A1Ji23v1OoYQX+k\no90KBC3R0W4FghboL2/Tg+YIuxWTItqtmbS8UZLyaVMPltXPBcZJ2kLSQGACMDuleg20fSdZitiY\nFEV5VtKRAJIGSdoo35ntq23vlg4lWF5W1217O7J0r2Or6PtzYLzSyWySNq8iV3I+hgIvJudqP7J0\nt3KZWtwHnLG6gfSRBtpUY09le78GkM2vfK0fJNu/M1jZfqp/BB5MaZqvp6jF98giIUuB4Sk9EUlD\nkn3yXAYstH17JWWU7eG6sY7O9wOnlPqusN5zgL0ljU71G0v6UKqbnXQ9mTUpbbXkm0LS8NzHo4En\nU/m2kmZWbrW67Uiy++J64LqkJ1S/J34P7J6uj6Sng1XrPirVPQAcozV7ujZPOjSEshMSzwKOsP1m\nBZETgOnhXAVBEARBEKwdmnGwngJOk7SYbJP8tam8lBK2HDiHLKG8iywScQ8wAuiU1EWWgnZOancC\ncIakhcBDwLAW9P81UPFo65TS920yJ68LuDyvb140/Z4C7JH0OR5YUkGmUvsSF5IdBLJI2ZH23yoX\nkDRW0g9qzKfEXOAqsnTE39q+Kz+27S7gBrL0tUeAH6TUtZ3J9n51Ad8ALrL9NpmTdpWkBWSnzG1Y\nNt6ZwEHKDrmYL+kTZfUjgb/U0fk64FlgURp/QpnOfwQ+B9yS1vhhsn19pEMZ7gU+nn7XlKeKDZQd\n1jG8QtV3kl0WAPsCX03l29AzElWJDmChpPnAp4B/r6UDWYRs37QGHwVey9XVikSW1mkJ2YuIGWne\nM4Bec6ox1/8gO0zj/mTLq8vqNyc7XTBY5+hstwJBS3S2W4GgBYr4vTxB2K2oFNFuKvK3waf9Tlva\nPqeucNAy6RCHm2w/2W5d+hJlJx3+wfa97dblvSIdmrHI9vdryLj9p+8HzdNJpJsVkU6at5so8v/d\n6wL9adN90Dhht2LSn+0mCdu9MpSK7mCNJovkvFr2XVhBEJQhaTbZvsHjbT9fQ664fxSCYD1g2LBR\nLF/++3arEQRBsN6zTjpYQRD0PZIcfxeCIAiCIAhqU83BauW7p4IgCIJ+RhFz1IOwW1EJuxWTsFsx\nKaLdwsEKgiAIgiAIgiDoIyJFMAiCHkSKYBAEQRAEQX0iRTAIgiAIgiAIgmAtEw5WEATBOkARc9SD\nsFtRCbsVk7BbMSmi3cLBCoIgCIIgCIIg6CNiD1YQBD2I78EKgiAIho0YxvLnlrdbjSDo18T3YAVB\n0BCSzMR2axEEQRC0lYkQz4hBUJs45CIIgmBdZlm7FQhaIuxWTMJuhaSIe3mCYtqtIQdL0ihJT1Sp\nmyVpTN+q1RiSRkqaL2l6rqxf/dmTtK+kSQ3ILcvJ31NNRtIWfahXxXFy9TX1TvfFrDoyfX5/5Pts\nxN6SvinpuXSvzJf08Qba1OxX0qrGNV7d5ruSnpB0maStJM2RNE/SPo3YtsG5bi5phqSlku6TNDRX\n9wtJcyVt3azuQRAEQRAEQWM0E8Hqj3Hio4AZtg/JlfVHPRvRyVWum+2nGer116ze7aDR8a+wPSb9\n/KwP+m1l3icDu9j+OnAAsMj2WNu/bLC/RmTOAWba3gH4OfCvqxvb44B5wGFNax70f7ZvtwJBS4Td\niknYrZB0dHS0W4WgBYpot2YcrA0kTZa0WNJtkgaXC0iaIGlR+rk0lQ2QNCmVLZT05VQ+WtL9khZI\nelxSK3+u3g+8WFb2Uk6fE9KYXZJuTGWTJF0p6SFJv5F0dCrfRNLMpMtCSUek8lGSlqR2SyVNkXRg\nar9U0u5JbmNJ1+eiEocnNd4C/tTAXF7KXQ+VdK+kpyRdnStfneMp6WspGrIot6Ybp3ZdqXx8Kt8j\n6bsg6bdJfmBJ01Jkp0vSSkmfbVDvd4GXUx8DchGaBZJOKxdO6/ZwWuNbk74HS7otJ7M6sibpoHL5\nOutWi175sXV4KekwXNLstD6LJO29RlVdlOb6sKQPpMJJpXsqfV6Vft8NDAHmSTobuAw4KvU7mJ62\nPU7So6nuGkmlukbmeiRwY7q+kewlRJ7lZP9ugiAIgiAIgrXA+5qQ3QH4vO05kq4HTgWuKFVK2ga4\nFNgNWAncn5yU54ARtndJcpulJlOAi21PlTSI1vaDDQS68wW290zj7AScC+xl+xVJ+YfK4bb3lrQj\nMBW4A3gDOMr2q5K2BOakOoDRwCdtL5b0OPDp1P6INMbRwHnAA7ZPSmlZcyXNtP0I8EjSaSxwiu0v\nlE+kpHdiD2BH4BngPklH276jVKksPe7EJDcQeFRSZ9LzedufSHKbStoA+BEw3vZ8SUOA18vGPizX\n738Bd9leVdK7GrafA45JH78AjCKL0LhsvUlrej7wMduvJyfja8AlwPclbWT7deBY4OYkf14F+Yuq\nrZukacBJtisde/Sl5Dg+Dpxpu6bzmOv3M8DPbF+SHJ2Sk7cJ8LDt8yVdRhadurhSV6m/IyX92XYp\ntXEFMNb2GelzaQ5/m9bg722/K+k/geOAyQ3OdWvbK9KYy9U7HbCb7J6pTT7xczvibW0RWEbYqYiE\n3YpJ2K2QdHZ2FjIasr7Tn+zW2dnZ0J6wZhysZ2zPSdeTgdPJOVhkD/uzbJciGlOAcWQPxNtLuhL4\nKTAjPeRva3sqgO23mtCD1L+AXZMuldgfuN32K2mMlbm6u1LZktwDqIBLJI0jewjdNle3zPbidP0r\nYGa6foLs8RPgIOBwSWelz4OAkcDS0qC255E5IvWYa/sPaZ63APuQOYEl9gHutP1GkrkD+AfgPuB7\nki4Bptn+paQPAy/Ynp90eDW16TGgpK2Am4BjknPVLAcA1zgdOVS23gAfBXYCHkq224DMQXlX0s/I\n1u4nZOlrZwEdleRrKVByFCtwNfCt5PhdRHbfntTgvB4Drk+O6t22F6byN23/NF3PI5t/JRqNnJXS\n/z4GjAEeS/MeDKzoJVx9rtX6LfE82drWZr8Gew+CIAiCIFhP6Ojo6OHsXXDBBRXlmnGwyh/UKu0H\n6fUwaXulpF2Bg4EvAuOBr1SS7dGRdCpZVMDAofk39ZIGAL8D3gSmNTGHEm9W0Pk4YCtgN9vdyg4U\nGFxBvjv3uZs1ayiyKNfTLehTTiNr3buR/XSKQh0KXCjpATJnst5aDwBuASbaXtKCvo0gsv1yx1Wo\nuxX4EvAK8Jjt15JzUU2+KWznU+t+CFQ93KNC2weT030YcIOky21PBt7Oib3LmvvgHVI0NucYNoOA\nG22f12S7EiskDbO9QtJweqfQ3gF8Q9Ji2zu1OEbQH4m36cUk7FZMwm6FpL9EQYLmKKLdmknLGyUp\nnzb1YFn9XGCcpC0kDQQmALNTqtdA23eSpYiNSVGUZyUdCSBpkKSN8p3Zvtr2bulQguVldd22tyNL\n9zq2ir4/B8YrncwmafMqciXnYyjwYnKu9iNLdyuXqcV9wBmrG0gfaaBNNfZUtvdrANn8ytf6QbL9\nO4OV7af6R+DBlKb5uu2bge+RRUKWAsNTeiKShiT75LkMWGj79krKKNvDdWOluhz3A6eU+q6w3nOA\nvSWNTvUbS/pQqpuddD2ZLJ2xnnxTJEejxNHAk6l8W0kzK7da3XYk2X1xPXBd0hOq3xO/B3ZP10fS\n08GqdR+V6h4AjtGaPV2bJx0aZSrwuXR9InB3Wf0JwPRwroIgCIIgCNYOzThYTwGnSVpMtkn+2lRe\nSglbTnaCWSfQRRaJuAcYAXRK6iJLQTsntTsBOEPSQuAhYFgL+v8aqHi0dUrp+zaZk9cFXJ7XNy+a\nfk8B9kj6HA8sqSBTqX2JC8kOAlmk7Ej7b5ULSBor6Qc15lNiLnAVWTrib23flR/bdhdwA1n62iPA\nD1Lq2s5ke7+6gG8AF9l+m8xJu0rSAmAGsGHZeGcCByk75GK+pE+U1Y8E/lJH5+uAZ4FFafwJZTr/\nkezB/5a0xg+T7evDdjdwL/Dx9LumPFVsoOywjuEVqr6T7LIA2Bf4airfhp6RqEp0AAslzQc+Bfx7\nLR3IImT7pjX4KPBarq5WJLK0TkvIXkTMSPOeAfSaU425XgYcKGkpWbrhpWX1mwN9EWUN+hv96gsq\ngoYJuxWTsFshKeL3KQXFtJuK/C3dab/TlrbPqSsctEw6xOEm20+2W5e+RNlJh3+wfW+7dXmvSIdm\nLLL9/RoyZuJ7p1PQR8Sm+2ISdism64PdJkKRnxEr0Z8OSwgapz/bTRK2e2UoFd3BGk0WyXm17Luw\ngiAoQ9Jssn2Dx9t+voZccf8oBEEQBH3CsBHDWP5cpUN5gyAosU46WEEQ9D2SHH8XgiAIgiAIalPN\nwWrlu6eCIAiCfkYRc9SDsFtRCbsVk7BbMSmi3cLBCoIgCIIgCIIg6CMiRTAIgh5EimAQBEEQBEF9\nIkUwCIIgCIIgCIJgLRMOVhAEwTpAEXPUg7BbUQm7FZOwWzEpot3CwQqCIAiCIAiCIOgjYg9WEAQ9\niO/BCoJ1g2HDRrF8+e/brUYQBME6S3wPVhAEDZE5WPF3IQiKj4j/44MgCNYecchFEATBOk1nuxUI\nWqKz3QoELVDEPSFB2K2oFNFuDTlYkkZJeqJK3SxJY/pWrcaQNFLSfEnTc2XL2qFLNSTtK2lSA3LL\ncvL3VJORtEUf6lVxnFx9Tb3TfTGrjkyf3x/5Phuxt6RjJD0p6d1GdanXr6RVjWnbo813JT0h6TJJ\nW0maI2mepH0asW2Dc/2OpCWSFkj6iaTNcnW/kDRX0tbN6v7/2Lv3eKuqeu/jny+oeTuiUoGaoIdT\nPlkpopalwTbLynvmXY8e0y4vO1nZ5VhWgJfUSk+kWfnEQQsy9RzybiLK4ihGICKoIGnitQf1lBR2\nyoz9e/4YY8Hca6/b3m7ce+L3/Xrt155rzDHn/M0xJps11rgsMzMzM2tPT3qwBuI4g8OAGRHx4ULa\nQIyznZiiwXZPz9MTrc7X07j7QzvXfwD4CDC7D8/bm/v+OLBLRPwb8H5gcUTsHhF3t3m+dvLMAN4W\nEaOBR4CvrDk4YiywADiwx5FbCXT0dwDWKx39HYD1QkdHR3+HYL3geiunMtZbTxpYG0qaKmmJpGsk\nbVybQdKxkhbnnwty2iBJU3LaIkmfzemjJN2eP2m/V9KOvYh/S+C5mrTnC/GcmK+5UNKVOW2KpEmS\n5kh6VNLhOX0zSTNzLIskHZLTR+YegSmSlkmaJukD+fhlkvbI+TaVNLnQK3FwDuNvwB/buJfnC9tD\nJN0k6WFJlxXS14zxlHRG7g1ZXCjTTfNxC3P6kTl9zxzv/Tm+zYoXlnRz7glcKGmlpH9uM+7VwB/y\nOQYVemjul/Tp2sy53O7JZXx1jveDkq4p5FnTsyZp/9r8LcqtrohYFhGPFMuvDc/nGIZLmp3LZ7Gk\nvdeGqnPzvd4j6Q05cUr1mcqvV+Xf1wObAwskfRm4EDgsn3djutbt8ZJ+nff9QFJ1Xzv3OjMiOvPL\nucCbarKsIP27MTMzM7N1YIMe5N0JODki5kqaDJwGXFzdKWkb4AJgN2AlcHtupDwNbBcRu+R81SFL\n04BvRsQNkjaid/PBBgOdxYSIeFe+zs7AV4F3R8QLkopvKodHxN6S3grcAEwH/gocFhEvShpKenN6\nQ84/CvhoRCyRdC9wTD7+kHyNw4GzgDsi4hRJQ4B5kmZGxK+AX+WYdgc+GRGfqL2RatzZnsBbgSeB\n2yQdHhHTqzuVhrmdlPMNBn4tqZLjfCYiDsr5/kHShsDPgSMj4j5JmwN/qbn2gYXz/gdwXUSsqsbd\nSEQ8DRyRX34CGEnqoYma8iaX6deA/SLiL7mRcQZwPvAjSZtExF+Ao4Gf5fxn1cl/bqNyk3QzcEpE\nrGgWdzsK5z0O+GVEnJ8bOtVG3mbAPRHxNUkXknqnvlnvVPl8h0r6U0RUhzY+C+weEafn19V7+D+5\nDN4TEaslfR84Hpjai3v9GKnuizpJz0wLEwrbHfhT9jKo4Hoqowqut/KpVCql/FT9tc71Vk4Dqd4q\nlUpbc8J60sB6MiLm5u2pwGcoNLBIb/ZnRUS1R2MaMJb0hnhHSZOAW4AZ+U3+thFxA0BE/K0HcZDP\nL2DXHEs97wOujYgX8jVWFvZdl9OWau18FAHnSxpLehO6bWHf8ohYkrcfAmbm7QeAHfL2/sDBkr6U\nX28EjACWVS8aEQtIDZFW5kXEE/k+rwL2ITUCq/YBfhERf815pgPvBW4DviPpfODmiLhb0tuB30XE\nfTmGF/MxXS4o6fXAT4EjcuOqp94P/CDyklU15Q2wF7AzMCfX3YakBspqSb8kld1/kYavfYn0jqNb\n/mYBVBuKfWw+MDk3VK+PiEU5/aWIuCVvLyDdfz3t9ppVh//tB4wB5uf73hh4tlvmFvcq6Szg5Yj4\nWc2uZ2jr3dyE1lnMzMzMXkM6Ojq6NPYmTpxYN19PGli18z/qzQfp9mYyIlZK2hX4IPAp4Ejgc/Xy\ndjmRdBqpVyCAA4qf1EsaBDwGvATc3IN7qHqpTszHA68HdouITqUFBTauk7+z8LqTtWUoUi/XI72I\np1Y7Zd39oIhHci/UAcA5ku4gNSZblfUg4CpgQkQs7UW87RBpvtzxdfZdDfwr8AIwPyL+nBsXjfK/\naiLirtzoPhC4QtJFETEVeLmQbTVrn4O/k3tjCw3DnhBwZUSc1duYJf0L6Rl4X53d04FvSFoSETv3\n9ho2EHX0dwDWKx39HYD1wkD5NN16xvVWTmWst54MyxspqThs6q6a/fOAsZK2ljQYOBaYnYd6DY6I\nX5CGiI3JvShPSToUQNJGkjYpniwiLouI3SJiTO0wqIjojDHZmKEAACAASURBVIgdgHtJw6nquRM4\nUnllNklbNchXbXwMAZ7Ljat9ScPdavM0cxtw+poDpNFtHNPIu5Tmfg0i3V9tWd9Fmr+zsdJ8qo8A\nd+Vhmn/JvRbfIfWELAOG5+GJSNo810/RhcCiiLi2XjBKc7iubBHz7cAnq+euU95zgb0ljcr7N5X0\n5rxvdo7146wd0tYs/ytRnOu0raSZTTNLI0jPxWTgxznOLuep8TiwR94+lK4NrGbPUXXfHcARhTld\nW+UY2iLpQ6QewEMi4qU6WU4EbnXjyszMzGzd6EkD62Hg05KWkCbJ/zCnV4eErQDOJA0oX0jqibgR\n2A6oSFpIGoJ2Zj7uROB0SYuAOcCwXsT/G6Du0tZ5SN95pEbeQuCiYrzFrPn3NGDPHM8JwNI6eeod\nX3UOaSGQxUpL2p9dm0HS7pIub3I/VfOAS0nDEX8bEdcVrx0RC4ErSMPXfgVcnoeuvYM092sh8A3g\n3Ih4mdRIu1TS/aRV5l5Xc70vAPsrLXJxn6SDavaPAP63Rcw/Bp4CFufrH1sT8/8A/wJclcv4HtK8\nPvKiDDcBH8q/m+anQR0oLdYxvE76YZKeIg1TvElrl/Xfhq49UfV0AIsk3QccBXy3WQzA/wXG5TLY\nC/hzYV+znshqOS0lfRAxI9/3DKDePdW9V+AS0mIat+e6vKxm/1ak1QVtvVPp7wCsVyr9HYD1Qhm/\nl8dcb2VVxnpTmb/lPc93GhoRZ7bMbL2WF3H4aUQ82N+x9CWllQ6fiIib+juWV0teNGNxRPyoSZ7o\n/9X3recqeLhZGVVYd/Umyvx//EA2kCbdW/tcb+U0kOtNEhHRbYRS2RtYo0g9OS/WfBeWmdWQNJs0\nb/CEiHimST43sMzWC25gmZmtS+tlA8vM+l5qYJlZ2Q0bNpIVKx7v7zDMzNZbjRpYvfnuKTNbz0WE\nf0r2M2vWrH6PwT8Dq97cuFp3yjgnxFxvZVXGenMDy8zMzMzMrI94iKCZdSEp/HfBzMzMrDkPETQz\nMzMzM1vH3MAyM1sPlHGMurneysr1Vk6ut3IqY725gWVmZmZmZtZHPAfLzLrwHCwzMzOz1hrNwdqg\nP4Ixs4FN6va3wszMrEeGbTeMFU+v6O8wzF517sEysy4kBRP6OwrrseXAjv0dhPWY662cXG/tmZC+\nV3GgqFQqdHR09HcY1kMDud68iqCZmZmZmdk61lYDS9JISQ802DdL0pi+Das9kkZIuk/SrYW05f0R\nSyOSxkma0ka+5YX8NzbKI2nrPoyr7nUK+5vGnZ+LWS3y9PnzUTxnO/Ut6QhJD0pa3W4src4raVV7\n0XY55tuSHpB0oaTXS5oraYGkfdqp2zbvdStJMyQtk3SbpCGFff8taZ6kN/Y0disBf5peTq63cnK9\nldJA7QWx5spYbz3pwRo4fbxrHQbMiIgPF9IGYpztxBQNtnt6np5odb6ext0f2rn+A8BHgNl9eN7e\n3PfHgV0i4t+A9wOLI2L3iLi7zfO1k+dMYGZE7ATcCXxlzcERY4EFwIE9jtzMzMzM2tKTBtaGkqZK\nWiLpGkkb12aQdKykxfnngpw2SNKUnLZI0mdz+ihJt0u6X9K9knrzedCWwHM1ac8X4jkxX3OhpCtz\n2hRJkyTNkfSopMNz+maSZuZYFkk6JKePlLQ0H7dM0jRJH8jHL5O0R863qaTJhV6Jg3MYfwP+2Ma9\nPF/YHiLpJkkPS7qskL5mjKekM3JvyOJCmW6aj1uY04/M6XvmeO/P8W1WvLCkm3NP4EJJKyX9c5tx\nrwb+kM8xqNBDc7+kT9dmzuV2Ty7jq3O8H5R0TSHPmp41SfvX5m9RbnVFxLKIeKRYfm14PscwXNLs\nXD6LJe29NlSdm+/1HklvyIlTqs9Ufr0q/74e2BxYIOnLwIXAYfm8G9O1bo+X9Ou87wfSmhUnWt4r\ncChwZd6+kvQhRNEK0r8bW98MqL57a5vrrZxcb6VUxu9TsnLWW09WEdwJODki5kqaDJwGXFzdKWkb\n4AJgN2AlcHtupDwNbBcRu+R8W+RDpgHfjIgbJG1E7+aDDQY6iwkR8a58nZ2BrwLvjogXJBXfVA6P\niL0lvRW4AZgO/BU4LCJelDQUmJv3AYwCPhoRSyTdCxyTjz8kX+Nw4Czgjog4JQ/LmidpZkT8CvhV\njml34JMR8YnaG6nGne0JvBV4ErhN0uERMb26U2mY20k532Dg15IqOc5nIuKgnO8fJG0I/Bw4MiLu\nk7Q58Jeaax9YOO9/ANdFxKpq3I1ExNPAEfnlJ4CRpB6aqClvcpl+DdgvIv6SGxlnAOcDP5K0SUT8\nBTga+FnOf1ad/Oc2KjdJNwOnRMQrXrKocN7jgF9GxPm5oVNt5G0G3BMRX5N0Ial36pv1TpXPd6ik\nP0VEdWjjs8DuEXF6fl29h/+Ty+A9EbFa0veB44Gpbd7rGyPi2XzNFeo+HLCT9Mw0Vxz4uQMeDmNm\nZmaveZVKpa0GX08aWE9GxNy8PRX4DIUGFunN/qyIqPZoTAPGkt4Q7yhpEnALMCO/yd82Im4AiIi/\n9SAO8vkF7Jpjqed9wLUR8UK+xsrCvuty2tLCG1AB50saS3oTum1h3/KIWJK3HwJm5u0HSG8/AfYH\nDpb0pfx6I2AEsKx60YhYQGqItDIvIp7I93kVsA+pEVi1D/CLiPhrzjMdeC9wG/AdSecDN0fE3ZLe\nDvwuIu7LMbyYj+lyQUmvB34KHJEbVz31fuAH1S9QqilvgL2AnYE5ue42JDVQVkv6Jans/os0fO1L\nQEe9/M0CqDYU+9h8YHJuqF4fEYty+ksRcUveXkC6/3ra7TWrDv/bDxgDzM/3vTHwbLfM7d9r7bDC\nZ0hl29y+bZ7dBg43gsvJ9VZOrrdSKuNcHhtY9dbR0dElnokTJ9bN15MGVu0btXrzQbq9mYyIlZJ2\nBT4IfAo4EvhcvbxdTiSdRuoVCOCA4if1kgYBjwEvATf34B6qXqoT8/HA64HdIqJTaUGBjevk7yy8\n7mRtGYrUy/VIL+Kp1U5Zdz8o4pHcC3UAcI6kO0iNyVZlPQi4CpgQEUt7EW87RJovd3ydfVcD/wq8\nAMyPiD/nxkWj/K+aiLgrN7oPBK6QdFFETAVeLmRbzdrn4O/k3thCw7AnBFwZEWf1MuRnJQ2LiGcl\nDaf7ENrpwDckLYmInXt5DTMzMzNroCfD8kZKKg6buqtm/zxgrKStJQ0GjgVm56FegyPiF6QhYmNy\nL8pTkg4FkLSRpE2KJ4uIyyJit4gYUzsMKiI6I2IH4F7ScKp67gSOVF6ZTdJWDfJVGx9DgOdy42pf\n0nC32jzN3AacvuYAaXQbxzTyLqW5X4NI91db1neR5u9srDSf6iPAXXmY5l8i4mfAd0g9IcuA4Xl4\nIpI2z/VTdCGwKCKurReM0hyuK+vtK7gd+GT13HXKey6wt6RRef+mkt6c983OsX6cNJyxVf5XojjX\naVtJM5tmlkaQnovJwI9znF3OU+NxYI+8fShdG1jNnqPqvjuAIwpzurbKMbTrBuBf8vZJwPU1+08E\nbnXjaj3kOSHl5HorJ9dbKZVxLo+Vs9560sB6GPi0pCWkSfI/zOnVIWErSCuYVYCFpJ6IG4HtgIqk\nhaQhaGfm404ETpe0CJgDDOtF/L8B6i5tnYf0nUdq5C0ELirGW8yaf08D9szxnAAsrZOn3vFV55AW\nAlmstKT92bUZJO0u6fIm91M1D7iUNBzxtxFxXfHaEbEQuII0fO1XwOV56No7SHO/FgLfAM6NiJdJ\njbRLJd0PzABeV3O9LwD7Ky1ycZ+kg2r2jwD+t0XMPwaeAhbn6x9bE/P/kN74X5XL+B7SvD4iohO4\nCfhQ/t00Pw3qQGmxjuF10g+T9BRpmOJNWrus/zZ07YmqpwNYJOk+4Cjgu81iAP4vMC6XwV7Anwv7\nmvVEVstpKemDiBn5vmcA9e6p7r2SGssfkLSMNNzwgpr9WwF90ctqZmZmZnVoIH3Ddk/l+U5DI+LM\nlpmt1/IiDj+NiAf7O5a+pLTS4RMRcVN/x/JqyYtmLI6IHzXJE0x49WIyM7P11AQo8/tMs1YkERHd\nRiiVvYE1itST82LNd2GZWQ1Js0nzBk+IiGea5CvvHwUzMxswhm03jBVPv+KFfc0GrPWygWVmfU9S\n+O9C+VQqlQG10pK1x/VWTq63cnK9ldNArrdGDazefPeUmZmZmZmZ1eEeLDPrwj1YZmZmZq25B8vM\nzMzMzGwdcwPLzGw9UMbvCTHXW1m53srJ9VZOZaw3N7DMzMzMzMz6iOdgmVkXnoNlZmZm1lqjOVgb\n9EcwZjawSd3+VlgfGDZsJCtWPN7fYZiZmdk65CGCZlZH+Gcd/Dz77BM9qoWeKOMYdXO9lZXrrZxc\nb+VUxnpr2sCSNFLSAw32zZI0Zt2E1ZykEZLuk3RrIW15f8TSiKRxkqa0kW9AxV3UTmyt8kgaL+mM\nvouq6zklTZE0tkX+LSVNl7RI0lxJO7dxjVmSRrTY36PnX9IRkpZIuiO/vkrS/ZI+m+/j8BbHt3Ov\nx+X7XCTpbkm7FPZdJOkhSeN6EreZmZmZta+dHqyBOBnjMGBGRHy4kDYQ42wnpoEYd1XZ46/6KrAw\nInYFTgK+109xnAKcGhH7SRoO7BERoyNiUh9e4zFgbL7Xc4HLqzsi4gvA2cDH+vB6NkAM1G+5t+Zc\nb+Xkeisn11s5lbHe2mlgbShpav7k/RpJG9dmkHSspMX554KcNih/4r44f5r+2Zw+StLt+ZP7eyXt\n2Iu4twSeq0l7vhDPifmaCyVdmdOmSJokaY6kR6u9BZI2kzQzx7JI0iE5faSkpfm4ZZKmSfpAPn6Z\npD1yvk0lTc49IwskHZzD+Bvwxzbu5fl8nuGSZueeucWS9s7pqySdm8vrHklvyOkHFa45o5A+XtJP\nct5lkk7N6ePy+W+S9LCky5ScLOnfC2V3qqSLasu0VfyNyr1I0j9KulXS/BzLWyRtIenxQp5NJT0p\naXC9/HWuv5JU1s3sDNwJEBHLgB2q5dXE74HVjZ7j7ChJv87lWa2vkyRdUrifGyWNlfR1YB9gsqRv\nAbcB2+X63qemnMZIquT7vlXSsHbvNSLmRkT1uZsLbFeTZQXp34+ZmZmZrQsR0fAHGAl0Anvl15OB\nM/L2LGAMsA3wBLA1qcF2B3BI3jejcK4t8u+5wCF5eyNg42YxNIhrIvC5Bvt2Bh4Gtsqvt8y/pwBX\n5+23Ao/k7cHA5nl7aCF9JOnN7M759b3A5Lx9CDA9b58HHJe3hwDLgE1qYtoduLzFPZ0BfCVvC9gs\nb3cCB+TtC4GvVq9VOPYU4Nt5ezywMJftUOBJYDgwDvjffF8CZgCHA5sBjwKD8/FzgLf1ok4alfv4\nwjMzExiVt98J3JG3fwGMy9tHVcuqSf4156zzXBxUJ/084KLCef4G7NbmfTV6jmcVyvzDwO15+yTg\ne4X8N5J6lKrH7FZ4vhYX8k3J9bFBroOhhfKY3O691uT5Yu1zB7wXuKnFcQHhn3XyQ6wrs2bNWmfn\ntnXH9VZOrrdycr2V00Cut/z/erf3Uu2sIvhkRMzN21OBzwAXF/bvCcyKiD8ASJoGjCUNT9pR0iTg\nFmCGpM2BbSPiBlJErXoeupEkYNccSz3vA66NiBfyNVYW9l2X05ZKemP1lMD5SnNbOoFtC/uWR8SS\nvP0Q6Q0/wAPADnl7f+BgSV/KrzcCRpAaWuTrLQA+0eLW5pN6NzYEro+IRTn9pYi4JW8vAN6ft7eX\ndA2pgbshsLxwrutz2f5e0p2kRsUfgXkR8QSk+T/APhExXWlO0EGSHgY2iIiHWsRaT7NyR9JmwHuA\na3MdkuMGuAY4GpgNHAN8v0X+uiJifINdFwCTJN1HqruFwOo27+sxap7jwr7p+fcCUoOpHa2W59sJ\neDtwe77vQcDvajM1udd0EWlf4GRSr1nRM8BbJL0uIl5qfIYJhe2O/GNmZmb22lWpVNpadKOdBla0\neA113jRGxEpJuwIfBD4FHAl8rl7eLieSTgM+nq9zQESsKOwbRHrD+xJwcxux1yq+oazGcTzwelLP\nQqfSog0b18nfWXjdydqyE/DRiHikF/GsERF35UbegcAVki6KiKnAy4VsqwvXvQT4TkTcrLRoQfEN\nd7GORP06K+abTJqn9DCpJ2VdGAS8EBH1Foa4AThP0lakHqM7gc2b5O+RiFhFYd5RruPH2jy23nN8\nat5dfR6K9fJ3ug697TaktgUBD0bE3j08bu0J0sIWlwMfqjZ4qyLiMUlLgSck7de4MT2ht5e3flLG\nMermeisr11s5ud7KaSDVW0dHR5d4Jk6cWDdfO3OwRkp6V94+DrirZv88YKykrSUNBo4FZksaShp2\n9gvga8CYiHgReErSoQCSNpK0SfFkEXFZROwWEWOKjau8rzMidiAN1zu6Qbx3AkdK2jpfY6sG+aoN\nrCHAc7lxtS9deyLa+TKg24DT1xwgjW7jmO7BpBXrnouIycCPSQ2NZjFswdqejZNq9h2ay3YoaWjg\n/Jy+p9LcskGk8rsbICLmAduT6u6qBvEtbXELTcs9N3KWSzqicM5d8r4/k+p0Emn4WjTL31OShuSe\nQSR9HJidn0WU5t9t0+TYbs9xo6z59+PA6Dy/bXtS72HD09dJWwa8QdJe+fobqI1VDwvxjgD+C/jn\niPhtnf27ADuSepJ701NpZmZmZk2008B6GPi0pCWkyfE/zOlpskZqBJ0JVEhDr+ZHxI2kyfUVSQuB\nn+Y8ACcCp0taRJprUp3A3xO/Ic356iYP6TuP1MhbCFQXbGjUEzeN1PBYBJwALK2Tp97xVeeQFgJZ\nrLSk/dm1GSTtLuny7od20QEsysPYjgK+2+K6E4H/lDSf7otRLCbVxz3A2YWG6r3ApaThjr/NjYaq\na4A5sXaBhGL8Q1vE3qzci04ATlFasONB0ly2qqtJvYk/L6Qd3yR/N5ImSjqozq63Ag/mRuIHgeqC\nKwJGAX9octpGz3Hd5yki5pAaWQ+R6nBBbZ4Gr6vHvwwcAVwo6X7Sv6l39+Bev076t3GZ0mIj82r2\nbwU8HhGddY61Eivj94SY662sXG/l5HorpzLWm9L8rHLJ852GRsSZLTO/xkgaD6yKiItr0scBX4iI\nuo0USTcCF0fErDr7DgR2jIhL10XM/UXS24CTI+KL/R3Lq0XSUcBHIuLYJnmicbveXhmxrv7mViqV\nATWMwtrjeisn11s5ud7KaSDXmyQiotuIpLI2sEYBVwAvRtfvwnrN62kDS9IQ0jDPhRFxzKsXqb3a\nlJbffy9ptco7muRzA2udWXcNLDMzM3t1rVcNLDNbd1IDy9aFYcNGsmLF4/0dhpmZmfWBRg2sduZg\nmdlrTL3vdPDPK/9Zl42rMo5RN9dbWbneysn1Vk5lrDc3sMzMzMzMzPqIhwiaWReSwn8XzMzMzJrz\nEEEzMzMzM7N1zA0sM7P1QBnHqJvrraxcb+XkeiunMtabG1hmZmZmZmZ9xHOwzKwLz8EyMzMza63R\nHKwN+iMYMxvYpG5/K8z61LDthrHi6RX9HYaZmVmfcw+WmXUhKZjQ31FYjy0HduzvIHpgQvq+tde6\nSqVCR0dHf4dhPeR6KyfXWzkN5Hrr1SqCkkZKeqDBvlmSxvRVgD0haYSk+yTdWkhb3h+xNCJpnKQp\nbeQbUHEXtRNbqzySxks6o++i6npOSVMkjW2Rf0tJ0yUtkjRX0s5tXGOWpBEt9vfo+Zd0hKQlku7I\nr6+SdL+kz+b7OLzF8S3vNef7nqRH8rlHF9IvkvSQpHE9idvMzMzM2tfOIhcD8SPGw4AZEfHhQtpA\njLOdmAZi3FVlj7/qq8DCiNgVOAn4Xj/FcQpwakTsJ2k4sEdEjI6ISX11AUkfBkZFxJuBTwI/rO6L\niC8AZwMf66vr2QBSpt4rW2Ogfiprzbneysn1Vk5lrLd2GlgbSpqaP3m/RtLGtRkkHStpcf65IKcN\nyp+4L849B5/N6aMk3Z4/Xb9XUm/eFmwJPFeT9nwhnhPzNRdKujKnTZE0SdIcSY9WewskbSZpZo5l\nkaRDcvpISUvzccskTZP0gXz8Mkl75HybSpqce0YWSDo4h/E34I9t3Mvz+TzDJc3OPXOLJe2d01dJ\nOjeX1z2S3pDTDypcc0Yhfbykn+S8yySdmtPH5fPfJOlhSZcpOVnSvxfK7lRJF9WWaav4G5V7kaR/\nlHSrpPk5lrdI2kLS44U8m0p6UtLgevnrXH8lqayb2Rm4EyAilgE7VMurid8Dqxs9x9lRkn6dy7Na\nXydJuqRwPzdKGivp68A+wGRJ3wJuA7bL9b1PTTmNkVTJ932rpGE9uNdDgZ/ke/01MKRwPMAK0r8f\nMzMzM1sH2mlg7QRcGhE7A6uA04o7JW0DXAB0AKOBPXMjZTSwXUTsknsOqsPlpgGXRMRo4D3A/+tF\n3IOBzmJCRLwrx7MzqceiIyJ2A4pviIdHxN7AwcCFOe2vwGERsQfwPuCiQv5RwLcjYqdcDsfk47+U\nrwFwFnBHROyVj/+OpE0i4lcR8fkc0+6SLq93I9W4geOAX0bEGGBX4P6cvhlwTy6vu4CP5/S7ImKv\niNgduBr4cuG07yDVx3uAbyj1lgDsCXwaeCvwT8BHgGuAgyUNznlOBv6jJraG2iz3qsuBf42IPUll\n+IOI+BOwUGuHrR2Uy2F1vfx1rv/5iJibY5go6aA6110EVBvU7wRGAG9qcV9HRMQzNH6OAQbn+/88\ndJm11K1XLyLOAe4FjouILwOHAI9GxJiIuLuaT9IGwCXAR/N9TwG+2YN73Q54qvD6mZxW1Un692Pr\nmwE72NiaKeP3u5jrraxcb+VUxnprZxXBJ6tv6oCpwGeAiwv79wRmRcQfACRNA8YC5wI7SpoE3ALM\nkLQ5sG1E3AAQEa0+je9GkkgNkKkNsrwPuDYiXsjXWFnYd11OWyrpjdVTAucrzW3pBLYt7FseEUvy\n9kPAzLz9ALBD3t6f1ED5Un69EekN/LLqRSNiAfCJFrc2n9S7sSFwfUQsyukvRcQteXsB8P68vb2k\na4BtgA3p+vbq+ly2v5d0J/BOUm/avIh4AtL8H2CfiJiuNCfoIEkPAxtExEMtYq2nWbkjaTNSg+/a\nXIfkuCE18o4GZgPHAN9vkb+uiBjfYNcFwCRJ95HqbiGwus37eoya57iwb3r+vQAY2eb5Wi3PtxPw\nduD2fN+DgN/VZmpyr608A7xF0usi4qWGuWYVtnfAw8/MzMzsNa9SqbTV4GungVX7aXy9OTfd3jRG\nxEpJuwIfBD4FHAl8rl7eLieSTiP10gRwQESsKOwbRHrD+xJwcxux1yq+oazGcTzwemC3iOhUWrRh\n4zr5OwuvO1lbdiL1NjzSi3jWiIi7ciPvQOAKSRdFxFTg5UK21YXrXgJ8JyJuzr0/xTfcxToSjedJ\nVdMnk3qfHqZrD01fGgS8kHvoat0AnCdpK2AMaTjf5k3y90hErKIw7yjX8WNtHlvvOT41764+D8V6\n+Ttde4a7DaltQcCDuae0N54Bti+8flNOAyAiHpO0FHhC0n4NG9P79vLq1n/cCC6lMs4tMNdbWbne\nymkg1VtHR0eXeCZOnFg3XztDBEdKKg5ju6tm/zxgrKSt8zCzY4HZkoaShlD9AvgaMCYiXgSeknQo\ngKSNJG1SPFlEXBYRu+WhUytq9nVGxA6koVZHN4j3TuBISVvna2zVIF+1gTUEeC43rvala09EO18G\ndBtw+poDCqu29YTSinXPRcRk4MekhkazGLZgbc/GSTX7Ds1lOxQYR+odgzR8c2RuqB4N3A0QEfNI\nb8qPBa5qEN/SFrfQtNxzI2e5pCMK59wl7/szqU4nATdF0jB/T0kaknsGkfRxYHZ+FlGaf7dNk2O7\nPceNsubfjwOjlWxP6j1sePo6acuAN0jaK19/A7Wx6mHBDcCJ+di9gJUR8WzhfnYhvRXftpc9lWZm\nZmbWRDsNrIeBT0taQpocX12VLAByI+hMoEIaejU/Im4kzfuoSFoI/DTngfTm73RJi4A5QHECfrt+\nA2xdb0ce0nceqZG3kLVzqhr1xE0jNTwWAScAS+vkqXd81TmkhUAWKy1pf3ZthmZzsAo6gEV5GNtR\nwHdbXHci8J+S5tN9MYrFpPq4Bzi70FC9F7iUNNzxt7nRUHUNMCciui3MkRsZTTUp96ITgFOUFux4\nkDQPqepqUm/izwtpxzfJ302TeUlvBR7MjcQPkueH5SF4o4A/NDlto+e47vMUEXNIjayHSHW4oDZP\ng9fV418GjgAulHQ/6d/Uu9u91zycdLmkR4EfUTNnEtgKeDwiOmuPtZLzHKxSKuPcAnO9lZXrrZzK\nWG+l/KLhPN9paESc2TLza4yk8cCqiLi4Jn0c8IWIqNtIkXQjcHFEzKqz70Bgx4i4dF3E3F8kvQ04\nOSK+2N+xvFokHQV8JCKObZLHXzRcRv6i4VIayF+gaY253srJ9VZOA7ne1OCLhsvawBoFXAG8WPNd\nWK95PW1gSRpCGua5MCKOefUitVeb0vL77wW+EhF3NMnnBpatexPcwDIzs3JbrxpYZrbuSPIfBVvn\nhm03jBVPr2id0czMbIBq1MBqZw6Wmb3GRIR/SvYza9asfo+hJz9uXCVlnFtgrreycr2VUxnrzQ0s\nMzMzMzOzPuIhgmbWhaTw3wUzMzOz5jxE0MzMzMzMbB1zA8vMbD1QxjHq5norK9dbObneyqmM9eYG\nlpmZmZmZWR/xHCwz68JzsMzMzMxaazQHa4P+CMbMBjap298KMzMzs/XesGEjWbHi8Vd0DvdgmVkX\n6YuG/XehfCpARz/HYD1XwfVWRhVcb2VUwfVWRhVe3XoT7baPerWKoKSRkh5osG+WpDFtXb2PSRoh\n6T5JtxbSlvdHLI1IGidpShv5BlTcRe3E1iqPpPGSzui7qLqeU9IUSWPbOOZ7kh6RdL+k0W3knyVp\nRIv9PXr+JR0haYmkO/Lrq3I8n833cXiL41veq6Tjupj/5QAAIABJREFUJC3KP3dL2qWw7yJJD0ka\n15O4zczMzKx97QwRHIgfZR8GzIiIMwtpAzHOdmIaiHFXlT1+ACR9GBgVEW+W9C7gh8Be/RDKKcCp\nEXGPpOHAHhHx5hxjy8Z4mx4DxkbEHyV9CLicfK8R8QVJ84CPAbP76Ho2YHT0dwDWKx39HYD1Skd/\nB2C90tHfAVivdPR3AD3WziqCG0qamj95v0bSxrUZJB0raXH+uSCnDcqfuC/On6Z/NqePknR7/uT+\nXkk79iLuLYHnatKeL8RzYr7mQklX5rQpkiZJmiPp0WpvgaTNJM3MsSySdEhOHylpaT5umaRpkj6Q\nj18maY+cb1NJkyXNlbRA0sE5jL8Bf2zjXp7P5xkuaXbumVssae+cvkrSubm87pH0hpx+UOGaMwrp\n4yX9JOddJunUnD4un/8mSQ9LukzJyZL+vVB2p0q6qLZMW8XfqNyLJP2jpFslzc+xvEXSFpIeL+TZ\nVNKTkgbXy1/n+itJZd3MocBPACLi18AQScNaHPN7YHWj5zg7StKvc3lW6+skSZcU7udGSWMlfR3Y\nB5gs6VvAbcB2ub73qSmnMZIq+b5vLcTa8l4jYm5EVJ+7ucB2NVlWkP79mJmZmdm6EBENf4CRQCew\nV349GTgjb88CxgDbAE8AW5MabHcAh+R9Mwrn2iL/ngsckrc3AjZuFkODuCYCn2uwb2fgYWCr/HrL\n/HsKcHXefivwSN4eDGyet4cW0keS3szunF/fC0zO24cA0/P2ecBxeXsIsAzYpCam3YHLW9zTGcBX\n8raAzfJ2J3BA3r4Q+Gr1WoVjTwG+nbfHAwtz2Q4FngSGA+OA/833JWAGcDiwGfAoMDgfPwd4Wy/q\npFG5jy88MzNJPUkA7wTuyNu/AMbl7aOqZdUk/5pz1nkuDqqTfiPwnsLrmcCYNu+r0XM8q1DmHwZu\nz9snAd+rufbYwjG7FZ6vxYV8U3J9bJDrYGihPCa3e681eb5Y+9wB7wVuanFcQPindD+zBkAM/nG9\nvVZ+XG/l/HG9lfPn1a43ol05L7U/7QwRfDIi5ubtqcBngIsL+/cEZkXEHwAkTQPGAucCO0qaBNwC\nzJC0ObBtRNxAiqhVz0M3kgTsmmOp533AtRHxQr7GysK+63LaUklvrJ4SOF9pbksnsG1h3/KIWJK3\nHyK9MQd4ANghb+8PHCzpS/n1RsAIUkOLfL0FwCda3Np8Uu/GhsD1EbEop78UEbfk7QXA+/P29pKu\nITVwNwSWF851fS7b30u6k9Q4+SMwLyKegDT/B9gnIqYrzQk6SNLDwAYR8VCLWOtpVu5I2gx4D3Bt\nrkNy3ADXAEeThq0dA3y/Rf66ImJ8L+Ju5TFqnuPCvun59wJSg6kdrZbn2wl4O3B7vu9BwO9qM7W6\nV0n7AieTes2KngHeIul1EfFS4zNMKGx3UMbueTMzM7O+VKlU2vri497Mwap9DXXeNEbESkm7Ah8E\nPgUcCXyuXt4uJ5JOAz6er3NARKwo7BtEesP7EnBzG7HXKr6hrMZxPPB6Us9Cp9KiDRvXyd9ZeN3J\n2rIT8NGIeKQX8awREXflRt6BwBWSLoqIqcDLhWyrC9e9BPhORNystGhB8Q13sY5E/Tor5psMfJXU\nAzXlldxHE4OAFyKi3sIQNwDnSdqK1GN0J7B5k/w99QywfeH1m3JaSw2e41Pz7urzUKyXv9N16G23\nIbUtCHgwIvbu4XFrT5AWtrgc+FC1wVsVEY9JWgo8IWm/xo3pCb29vPWbjv4OwHqlo78DsF7p6O8A\nrFc6+jsA65WO/g5gjY6ODjo6Ota8njhxYt187czBGqm0MADAccBdNfvnAWMlbS1pMHAsMFvSUNKw\ns18AXyMNyXoReErSoQCSNpK0SfFkEXFZROwWEWOKjau8rzMidiAN1zu6Qbx3AkdK2jpfY6sG+aoN\nrCHAc7lxtS9deyLa+TKg24DT1xzQxgp1dYNJK9Y9FxGTgR+TGhrNYtiCtT0bJ9XsOzSX7VDS0MD5\nOX1Ppbllg0jldzdARMwjNUCOBa5qEN/SFrfQtNwjYhWwXNIRhXPukvf9mVSnk0jD16JZ/l64ATgx\nn2MvYGVEPJtfz5S0TaMD6z3HjbLm348Do/P8tu1JvYcNT18nbRnwhhwnkjaQtHOTc9TGOwL4L+Cf\nI+K3dfbvAuxI6knuTU+lmZmZmTXRTgPrYeDTkpaQJsf/MKcHQG4EnUlapH4hMD8ibiRNrq9IWgj8\nNOeB9Eb3dEmLSHNNWi02UM9vSHO+uslD+s4jNfIWAtUFGxr1xE0jNTwWAScAS+vkqXd81TmkhUAW\nKy1pf3ZtBkm7S7q8yf1Aap4vknQfad7Nd1tcdyLwn5Lm030xisWk+rgHOLvQUL0XuJQ03PG3udFQ\ndQ0wJ9YukFCMf2iL2JuVe9EJwClKC3Y8SJrLVnU1qTfx54W045vk70bSREkH1YntFlJj7VHgR8Bp\nOb+AUcAfmpy20XNc93mKiDmkRtZDpDpcUJunwevq8S8DRwAXSrqf9G/q3e3eK/B10r+Ny5QWG5lX\ns38r4PGI6KxzrJVapb8DsF6p9HcA1iuV/g7AeqXS3wFYr1T6O4AeK+UXDef5TkOj6zLtRlpFEFgV\nERfXpI8DvhARdRspkm4ELo6IWXX2HQjsGBGXrouY+4uktwEnR8QX+zuWV4uko4CPRMSxTfJE43a9\nDVwVBtIwCmtXBddbGVVwvZVRBddbGVUo2xcNl7WBNQq4AngxIj7cz+EMKD1tYEkaQhrmuTAijnn1\nIrVXm9Ly++8lrVZ5R5N8bmCZmZnZa9RrtIFlZuuOG1hmZmb22vXKG1jtrCJoZq857azvYmZmZrZ+\nGTas3W/eacwNLDPrxj3b5VOpVLosHWvl4HorJ9dbObneyqmM9eYhgmbWhaTw3wUzMzOz5hoNEWxn\nmXYzMzMzMzNrgxtYZmbrgUql0t8hWC+43srJ9VZOrrdyKmO9uYFlZmZmZmbWRzwHy8y68BwsMzMz\ns9Y8B8vMzMzMzGwd8zLtZtaN5O/BMjMzs4Fj2HbDWPH0iv4Ooy0eImhmXUgKJvR3FNZjy4Ed+zsI\n6zHXWzm53srJ9VZO1XqbMPC+p7NXQwQljZT0QIN9sySN6asAe0LSCEn3Sbq1kLa8P2JpRNI4SVPa\nyDeg4i5qJ7ZWeSSNl3RG30XV9ZySpkga28Yx35P0iKT7JY1uI/8sSSNa7O/R8y/pCElLJN2RX1+V\n4/lsvo/DWxz/iu5V0kWSHpI0ridxW0n4TUM5ud7KyfVWTq63ciphvbUzB2tgNRWTw4AZEfHhQtpA\njLOdmAZi3FVljx8ASR8GRkXEm4FPAj/sp1BOAU6NiP0kDQf2iIjRETGpry7Q7F4j4gvA2cDH+up6\nZmZmZtZVOw2sDSVNzZ+8XyNp49oMko6VtDj/XJDTBuVP3BdLWiTpszl9lKTb86fr90rqTbt0S+C5\nmrTnC/GcmK+5UNKVOW2KpEmS5kh6tNpbIGkzSTNzLIskHZLTR0pamo9bJmmapA/k45dJ2iPn21TS\nZElzJS2QdHAO42/AH9u4l+fzeYZLmp175hZL2junr5J0bi6veyS9IacfVLjmjEL6eEk/yXmXSTo1\np4/L579J0sOSLlNysqR/L5TdqZIuqi3TVvE3KvciSf8o6VZJ83Msb5G0haTHC3k2lfSkpMH18te5\n/kpSWTdzKPATgIj4NTBE0rAWx/weWN3oOc6OkvTrXJ7V+jpJ0iWF+7lR0lhJXwf2ASZL+hZwG7Bd\nru99asppjKRKvu9bC7H2xb2uIP37sfXNgO0Lt6Zcb+Xkeisn11s5lbDe2mlg7QRcGhE7A6uA04o7\nJW0DXAB0AKOBPXMjZTSwXUTsEhG7AtXhctOASyJiNPAe4P/1Iu7BQGcxISLelePZGfgq0BERuwHF\nN8TDI2Jv4GDgwpz2V+CwiNgDeB9wUSH/KODbEbFTLodj8vFfytcAOAu4IyL2ysd/R9ImEfGriPh8\njml3SZfXu5Fq3MBxwC8jYgywK3B/Tt8MuCeX113Ax3P6XRGxV0TsDlwNfLlw2neQ6uM9wDeUeksA\n9gQ+DbwV+CfgI8A1wMGSBuc8JwP/URNbQ22We9XlwL9GxJ6kMvxBRPwJWKi1w9YOyuWwul7+Otf/\nfETMzTFMlHRQnetuBzxVeP1MTmt2X0dExDM0fo4BBuf7/zx0mbXUrVcvIs4B7gWOi4gvA4cAj0bE\nmIi4u5pP0gbAJcBH831PAb7Zh/faSfr3Y2ZmZmbrQDurCD5ZfVMHTAU+A1xc2L8nMCsi/gAgaRow\nFjgX2FHSJOAWYIakzYFtI+IGgIho9Wl8N5JEaoBMbZDlfcC1EfFCvsbKwr7rctpSSW+snhI4X2lu\nSyewbWHf8ohYkrcfAmbm7QeAHfL2/qQGypfy642AEcCy6kUjYgHwiRa3Np/Uu7EhcH1ELMrpL0XE\nLXl7AfD+vL29pGuAbYAN6dq+vz6X7e8l3Qm8k9SbNi8inoA0/wfYJyKmK80JOkjSw8AGEfFQi1jr\naVbuSNqM1OC7NtchOW5IjbyjgdnAMcD3W+SvKyLG9yLuVh6j5jku7Juefy8ARrZ5vlbL8+0EvB24\nPd/3IOB3tZlewb0+A7xF0usi4qWGuWYVtneglOOfX3NcR+Xkeisn11s5ud7KaQDVW6VSoVKptMzX\nTgOr9tP4enNuur1pjIiVknYFPgh8CjgS+Fy9vF1OJJ1G6qUJ4ICIWFHYN4j0hvcl4OY2Yq9VfENZ\njeN44PXAbhHRqbRow8Z18ncWXneytuxE6m14pBfxrBERd+VG3oHAFZIuioipwMuFbKsL170E+E5E\n3Jx7f4pvuIt1JBrPk6qmTyb1Pj1M1x6avjQIeCH30NW6AThP0lbAGOBOYPMm+XvqGWD7wus35bSW\nGjzHp+bd1eehWC9/p2vPcLchtS0IeDD3lPZG03uNiMckLQWekLRfw8b0vr28upmZmdl6qqOjg46O\njjWvJ06cWDdfO0MER0oqDmO7q2b/PGCspK3zMLNjgdmShpKGUP0C+BowJiJeBJ6SdCiApI0kbVI8\nWURcFhG75aFTK2r2dUbEDqShVkc3iPdO4EhJW+drbNUgX7WBNQR4Ljeu9qVrT0Q7XwZ0G3D6mgPa\nWKGubjBpxbrnImIy8GNSQ6NZDFuwtmfjpJp9h+ayHQqMI/WOQRq+OTI3VI8G7gaIiHmkN+XHAlc1\niG9pi1toWu4RsQpYLumIwjl3yfv+TKrTScBNkTTM3ws3ACfmc+wFrIyIZ/PrmXmYa131nuNGWfPv\nx4HRSrYn9R42PH2dtGXAG3KcSNogD79sV8N7zWm7kD4L2raXPZU2UJVwjLrheisr11s5ud7KqYT1\n1k4D62Hg05KWkCbHV1clC4DcCDoTqAALgfkRcSNp3kdF0kLgpzkPpDd/p0taBMwBWi02UM9vgK3r\n7chD+s4jNfIWsnZOVaOeuGmkhsci4ARgaZ089Y6vOoe0EMhipSXtz67N0GwOVkEHsEjSfcBRwHdb\nXHci8J+S5tN9MYrFpPq4Bzi70FC9F7iUNNzxt7nRUHUNMCciui3MkRsZTTUp96ITgFOUFux4kDQP\nqepqUm/izwtpxzfJ302jeUl5iOVySY8CPyLPI8xD8EYBf2hy2kbPcd3nKSLmkBpZD5HqcEFtngav\nq8e/DBwBXCjpftK/qXe/0nst2Ap4PCI6a481MzMzs1eulF80nOc7DY2IM1tmfo2RNB5YFREX16SP\nA74QEXUbKZJuBC6OiFl19h0I7BgRl66LmPuLpLcBJ0fEF/s7lleLpKOAj0TEsU3y+IuGzczMbGCZ\nsJ580fAANh3YW4UvGrbekTRE0jLgz/UaVwARcfP61rgCiIiHXmONq4uAL5KGoJqZmZnZOlDKHiwz\nW3ck+Y+CmZmZDSjDthvGiqdXtM74KmrUg9XOKoJm9hrjD17Kp1KpdFnZyMrB9VZOrrdycr2VUxnr\nzT1YZtaFpPDfBTMzM7Pm1rc5WGZmZmZmZgOOG1hmZuuBdr5Z3gYe11s5ud7KyfVWTmWsNzewzMzM\nzMzM+ojnYJlZF56DZWZmZtaa52CZmZmZmZmtY25gmVk3kgbEz/A3De/voiiNMo5RN9dbWbneysn1\nVk5lrDd/D5aZdTehvwNInp3wbH+HYGZmZtYjnoNl/U7Sqoj4h/6OA0DSGOBKYF5EnJLTlkfEjv0Q\ny67AthFxa359ErBDRExscdytwF7AXRFxSCH9WGA88KOI+Pcmx8dAaWAxwV96bGZmZgOT52DZQDaQ\n3kGfAHy/2rjKehSfpL76dzUaOKAmrZ1YvkW6j64HRlwFjAM+/8pDMzMzM7N63MCyAUPSREkLJd0n\n6WlJkyWNlLRU0hRJyyRNk/QBSXPy6z3ysXtKukfSAkl3S3pzL8PYEniuJu35fI1xkmZLuknSw5Iu\nK8S+StJ3JC0E9pI0RlJF0nxJt0oalvOdLukhSfdL+llO2zTf69wc/8GSNgTOBo7K5XEk8L/Ai61u\nICJmNcoXEc8CQ3pcKjbglXGMurneysr1Vk6ut3IqY715DpYNGBExHhgvaQjw38Aledco4KMRsUTS\nvcAxEbG3pEOAs4CPAEuBfSKiU9J+wPnAEb0IYzDQWRPXuwov9wTeCjwJ3Cbp8IiYDmwG/Coivihp\nA2A2cEhE/F7SUcA3gVOAfyMN83tZ0hb5nGcBd0TEKfne5wEzgW8Au0fE6bVBSjo475vQi3v0Bytm\nZmZm64gbWDYQTQUuioj7JY0ElkfEkrzvIVLjA+ABYGTe3hL4Se65CnrxbOeG0dtY27CrZ15EPJHz\nXwXsA0wHVuffADsBbwdulyRSg+Z3ed8i4GeSrgOuy2n7AwdL+lJ+vREwolmsEXEjcGP7d9fFHySN\niojfNswxq7C9A/Cqz0Cznuro6OjvEKwXXG/l5HorJ9dbOQ2keqtUKm31qLmBZQOKpAnAkxHxk0Ly\nS4XtzsLrTtY+w+cAd0bE4blRVmwiVM99LnAgEBExpmbfm0g9R49GxL1NQqydA1V9/ZfCt/MKeDAi\n9q5z/IHAWOAQ4CxJ78j5PxoRj9TEtFeTOF6JScD9kj4TEVfUzbHvOrqymZmZWUl1dHR0afBNnFh/\n3TEPFbKBQLBm2Nv7gc/W29/CEOCZvH1yvQwR8bWI2K22cZX3PQ1sl8JQR5PrvDPPCxsEHA3cVSfG\nZcAbqg0kSRtI2jnvGxERs4EzgS1IQwtvA9YMA5Q0Om+uynl6QzQut68C/9SwcWWlVMYx6uZ6KyvX\nWzm53sqpjPXmBpYNBNWen88D2wLz88IOE2r2124XfQu4QNICevlc5x6oR4Gtm2S7F7iUNFTxtxFR\nHea3Jq6IeJk0/+tCSfcDC4F35yGIUyUtAhYAkyLiT6Tetw0lLZb0AGlxC0i9cDsXFrlYIy+EMaFe\ngJL+G7gaeJ+kJyV9oCbLRnmxCzMzMzPrY/4eLLMCSd8HHoiIH9bZNw74QvG7pcpG0huBRRGxTZM8\n/h4sMzMzsxb8PVhm7fkJcLKkyf0dSF/LXzQ8g9TbZ2ZmZmbrgHuwzKwLSQPmj8Kw7Yax4ukV/R1G\nKVQqlQG10pK1x/VWTq63cnK9ldNArrdGPVheRdDMuvEHL2ZmZma94x4sM+tCUvjvgpmZmVlznoNl\nZmZmZma2jrmBZWa2Hijj94SY662sXG/l5HorpzLWmxtYZmZmZmZmfcRzsMysC8/BMjMzM2vNc7DM\nzMzMzMzWMTewzKwbSa/oZ/ibhvf3LbzmlHGMurneysr1Vk6ut3IqY735e7DMrLsJr+zwZyc82ydh\nmJmZmZWN52CZ9RNJI4GbIuIdbeb/FnAw8BLwW+DkiPhTD663EzAFGAN8NSIubpAvXmkDiwn+smIz\nMzNbv3kOltnA1JNWyAzgbRExGngE+EoPr/V74DPAt3t4nJmZmZm1yQ0ss/61oaSpkpZIukbSxpJ2\nl7RQ0n2SFktaDRARMyOiMx83F3hTTy4UEf8TEQuAv/fxPdgAUMYx6uZ6KyvXWzm53sqpjPXmBpZZ\n/9oJuDQidgZWAadFxIKI2C0ixgC/pH6P08eAW1/FOM3MzMysDZ6DZdZP8hys2RGxQ369L/CZiDg8\nvz4aOBXYv/jFVJLOAsZExEd7ed3xwKqmc7DGFRJ2AHbs4UUmeA6WmZmZrV8qlUqXHrWJEyfWnYPl\nVQTN+ldtKyQAJL0d+Abw3prG1b8ABwDvq3cySecCBwKRe8B6Z99eH2lmZv+/vTsPsqws7zj+/SGF\nCwIxUQczCGjU4IYM4qCBShor4BYQNSoGRVyyQQIVxXJLdKbKqMQykaASFzIhmIFCRUXKEhBoE1QE\nHAaQTRIWlzhoBQiSKBF48sd9G05PLzPT0829p/v7qZq657xne24/dXv6ue/7niNpURobG2NsbOz+\n9dWrV0+7n0MEpeHaLcm+bfkPgIuS7ASsBY6oqtsmdkzyQuBtwCFVdfd0J6uqv+wML5zNlG9b1G99\nHKMu89ZX5q2fzFs/9TFv9mBJw3UdcHSSNcB3gZOAVwG7Ap9KEh7ojToR2A44b9DMxVV11OZeKMky\n4DJgB+C+JMcCT6uqu+bzDUmSJC1lzsGSNInPwZIkSdo0n4MlSZIkSQvMHixJkyTZ6l8Ky5YvY8MP\nN8xHONpM4+Pjkybeqh/MWz+Zt34yb/00ynmbqQfLOViSpvCLF0mSpLmxB0vSJEnK3wuSJEmzcw6W\nJEmSJC0wCyxJWgT6+JwQmbe+Mm/9ZN76qY95s8CSJEmSpHniHCxJkzgHS5IkadOcgyVJkiRJC8wC\nS9IUSRb9v5132XnYP+Z51ccx6jJvfWXe+sm89VMf8+ZzsCRNtWrYASy8W1fdOuwQJEnSIuQcLGnE\nJLkJeHZV3ZbkoqraP8nvAMdV1cFbcd6Tgd8Dbq2qPWfZr5ZCgcUqH6gsSZLmzjlYUn/c/1d/Ve0/\nXfscrQFesJXnkCRJ0iwssKQhSfLHSS5Psi7JjUnOn9jU2ednnUN2SnJ2kuuSfHxLr1dVFwG3b2XY\nGlF9HKMu89ZX5q2fzFs/9TFvFljSkFTVJ6pqBbAS+AHw4el26yw/BzgaeCrwpCQvX/goJUmStCW8\nyYU0fH8PXFBVX9nEfpdU1S0ASU4D9gfOXJCILuws7w48YUGuonk0NjY27BA0B+atn8xbP5m3fhql\nvI2Pj29Wj5oFljRESY4EHl9VR23G7hvPwZq0nmQl8InW/p6qOnvOgR0w5yMlSZIWpbGxsUkF3+rV\nq6fdzyGC0pAkeTbwVuC1s+3WWd43yW5JtgFeDVzU3bGqLqmqFVW19yzFVTY6pxaJPo5Rl3nrK/PW\nT+atn/qYNwssaXiOBh4FXNhudPHJ1t7tmeouXwJ8FLga+I+q+sKWXCzJWuCbwFOSfD/JG+YeuiRJ\nkqbjc7AkTeJzsCRJkjbN52BJkiRJ0gKzwJKkRaCPY9Rl3vrKvPWTeeunPubNuwhKmmrVsANYeMuW\nLxt2CJIkaRFyDpakSZKUvxckSZJm5xwsSZIkSVpgFliStAj0cYy6zFtfmbd+Mm/91Me8WWBJkiRJ\n0jxxDpakSZyDJUmStGnOwZIkSZKkBWaBJUmLQB/HqMu89ZV56yfz1k99zJvPwZI0RTKlt1uSJPXM\nsuXL2PDDDcMOY8lxDpakSZLUUnjQsCRJi94q8G/9heMcLGkzJLkvyYc6629N8p4hxbJbi+foTtuJ\nSY4YRjySJEnaNAssabK7gZcn+dVhB9L8BDg2icN5Nbubhh2A5sS89ZN56yfz1kt9nINlgSVNdg/w\nSeAtG29oPUrnJ1mf5Lwku7T2NUlOSPKNJP+e5OWdY45Lckk75r1ziOenwPnAkdPEs1eSb7Vzfz7J\nTq39wiQfTPLtJNcl2a+1b5Pkb1r7+iR/OId4JEmSNAsLLGmyAj4GHJ5kh422nQisqaq9gLVtfcLO\nVbUfcDBwPECSA4EnV9VKYAWwT5L95xDP8cBxmXrniVOAt7V4vgt0C7iHVNW+wF/A/TOq3gTc0dpX\nAn+UZLctjEej6gnDDkBzYt76ybz1k3nrpbGxsWGHsMUcdiRtpKruSnIKcCzw886m5wEva8un0gqp\n5ovt2GuTPLa1HQQcmGQdEGB74MnARVsYz81JLgYOn2hLsiOwU1VNnOsU4IzOYWe21+8AE0XUQcAz\nk7yyre/Y4rllykUv7Czvjv8pSZKkJW98fHyzhixaYEnTOwFYB6zptM12G567O8vpvH6gqj4100FJ\nDmXQ81TAm6tq3Qy7fgD4HDA+zXVmi+deHvicB/jzqjpvluMGDtjkHho1N2Eh3EfmrZ/MWz+Zt14a\nHx8fmV6ssbGxSbGsXr162v0cIihNFoCqup1Bj9CbOtu+CbymLb8W+LfZzgGcA7wxyfYASX49yWO6\nO1bVF6tqRVXtPUNxNRHP9cA1wCFt/U7gton5VcDrgK9vRjxHTdwwI8mTkzx8hmMkSZI0B/ZgSZN1\ne6k+DBzdaTsGWJPkOAY3n3jDNMfcv15V5yXZA/hWmz71MwaF2U/nGM9fM+hVm3Ak8A+tSLpxU/EA\nn2Yw4G9dm8/1E+DQLYhFo8xvZfvJvPWTeesn89ZLo9J7tSV80LCkSXzQsCRJi8QqHzS8kHzQsCQt\nZj7fpZ/MWz+Zt34yb73Ux+dgOURQ0lSrhh2AJEnaWsuWLxt2CFtt/fr1vRsmaIElaQqHE/TPqlWr\nWLVq1bDD0BYyb/1k3vrJvPXTHXfcMewQtphDBCVJkiRpnlhgSdIicPPNNw87BM2Beesn89ZP5q2f\n+pg37yIoaZIk/lKQJEnaDNPdRdACS5IkSZLmiUMEJUmSJGmeWGBJkiRJ0jyxwJIkSZKkeWKBJQmA\nJC9Mcl2S7yV5+7DjWeqSnJzk1iRXdtoeleTcJNcnOSfJTp1t70xyQ5JrkxzUad87yZUtrx95sN/H\nUpNklyQXJLk6yVVJjmnt5m6EJXlokm8nubxC8BmbAAAGy0lEQVTl7v2t3bz1QJJtkqxLclZbN28j\nLsnNSa5on7lLWtuiyZsFliSSbAN8FHgB8HTgNUn2GG5US94aBvnoegfwtar6TeAC4J0ASZ4GvAp4\nKvAi4ONJJu5qdBLwpqp6CvCUJBufU/PrHuAtVfV04HnA0e2zZO5GWFXdDRxQVSuAPYHnJ9kP89YX\nxwLXdNbN2+i7DxirqhVVtbK1LZq8WWBJAlgJ3FBVt1TVL4HTgZcOOaYlraouAm7fqPmlwClt+RTg\n0LZ8CHB6Vd1TVTcDNwArk+wM7FBVl7b9/rlzjBZAVW2oqvVt+S7gWmAXzN3Iq6r/bYsPZfD30e2Y\nt5GXZBfgxcCnO83mbfSFqXXIosmbBZYkgOXADzrrP2xtGi2PrapbYfCHPPDY1r5x/n7U2pYzyOUE\n8/ogSrI7sBdwMbDM3I22NszscmADMF5V12De+uDvgLcB3ecOmbfRV8B5SS5N8ubWtmjytu2wA5Ak\nzZkPMhxRSR4JfA44tqrumuYB3uZuxFTVfcCKJDsC5yQZY2qezNsISfIS4NaqWt/yNRPzNnr2q6of\nJ3kMcG6S61lEnzd7sCTB4NugXTvru7Q2jZZbkywDaEMjftLafwQ8vrPfRP5matcCSrItg+Lq1Kr6\nUms2dz1RVXcCXwH2wbyNuv2AQ5LcCJzGYO7cqcAG8zbaqurH7fWnwBcZTFVYNJ83CyxJAJcCT0qy\nW5LtgMOAs4YckwZj1NNZPws4si2/HvhSp/2wJNsleQLwJOCSNsTiv5OsbBOCj+gco4Xzj8A1VXVC\np83cjbAkj564Y1mShwMHApdj3kZaVb2rqnatqicy+H/rgqp6HfBlzNvISvKI1stPku2Bg4CrWESf\nN4cISqKq7k3yZ8C5DL54Obmqrh1yWEtakrXAGPBrSb4PvBf4IPDZJG8EbmFwVyWq6pokZzC4i9Yv\ngaOqamJoxdHAPwEPA75SVV99MN/HUtPuPHc4cFWbz1PAu4DjgTPM3ch6HHBK+yNtGwa9j+e3HJq3\n/vkg5m2ULQO+0IZObwv8S1Wdm+QyFkne8kB8kiRJkqSt4RBBSZIkSZonFliSJEmSNE8ssCRJkiRp\nnlhgSZIkSdI8scCSJEmSpHligSVJkiRJ88QCS5IkjZQkFybZe4Ztpyd5Ylu+OcnXN9q+PsmVbfn1\nSU6c5TonJXneDNsOSfJXc38XkpYqCyxJktQLSX4D2L6qbmxNBeyQZHnbvkdr65rtgZ/7AhfPsO3L\nwCuSbLsVIUtagiywJEnSrJI8IsnZSS5PcmWSV7b2m5Ic39ou7vQsPTrJ55J8u/37rc55Tm77fifJ\nIa39YUlOS3J1kjOBh80QymEMCp+uM1o7wGuAtRtt37X1iF2f5D2d97QH8L2qqiTHtGuvT7IWoKoK\n+CZw0Bx/bJKWKAssSZK0KS8EflRVK6pqT+CrnW23t7aPASe0thOAv62qfYHfBz7d2t8NnF9VzwWe\nD3woycOBPwX+p6qeDrwX2GeGOPYHLuusF/B54GVt/WCmFmDPadufBbyyM/TwRZ338XZgr6raC/iT\nzrGXAr89QyySNC0LLEmStClXAQcm+UCS/avqZ51tp7fX04DntuXfBT6a5HLgLOCRSR7BoDfoHa19\nHNgO2JVBEfMZgKq6Crhihjh2A368Udt/AbcneTVwDfDzjbafV1V3VNUvgDMZFGkAL+CBAusKYG2S\nw4F7O8f+J7D7DLFI0rQcVyxJkmZVVTe0np8XA+9L8rWqet/E5u6u7XUbYN+q+mX3PEkAXlFVN0zT\nPqlpplBm2HYGgx60I2Y4ZtJ66zXbqao2tLaXMCjyDgHeneQZVXVfu9Zsc7gkaQp7sCRJ0qySPA74\neVWtBT4EdO/w9+r2ehjwrbZ8DnBs5/hnddqP6bTv1Rb/FTi8tT0D2HOGUG4Bdu6G1l6/ABwPnDvN\nMQcm+ZVWVB0KfAM4ALiwXS/ArlX1deAdwI7AI9uxj2vXlKTNZg+WJEnalGcymC91H/B/TJ6n9Kgk\nVwC/YHCTCRgUVx9r7Q9hUEAdBbwP+Ei7jXqAmxj0Gp0ErElyNXAtk+dZdV3EYH7WurZeAFV1F4PC\nb7resEsYDA1cDpxaVevards/27Y/BPhMkh1bTCdU1Z1t20rg7E3+dCSpI4Ob5EiSJG2ZJDcBz66q\n2x6k6z0ROLGqXrKV57mMwRDGe2fZJwwKuedU1T1bcz1JS4tDBCVJ0lw9qN/Studf3TlxO/itOM8+\nsxVXzcHA5y2uJG0pe7AkSZIkaZ7YgyVJkiRJ88QCS5IkSZLmiQWWJEmSJM0TCyxJkiRJmicWWJIk\nSZI0T/4f8YECY6U7o3gAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "co_t, de_t = compression_decompression_times()\n", - "\n", - "fig = plt.figure(figsize=(12, len(compression_configs)*.3))\n", - "fig.suptitle('Decompression speed', fontsize=14, y=1.01)\n", - "\n", - "\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c == 'blosc' and o['shuffle'] == 2]\n", - "x = (nbytes / 1000000) / np.array([de_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='bit shuffle', color='b')\n", - "\n", - "y = [i for i, (c, o) in enumerate(compression_configs) if c != 'blosc' or o['shuffle'] == 0]\n", - "x = (nbytes / 1000000) / np.array([de_t[i] for i in y])\n", - "ax.barh(bottom=np.array(y)+.2, width=x.max(axis=1), height=.6, label='no shuffle', color='g')\n", - "\n", - "ax.set_yticks(np.arange(len(labels))+.5)\n", - "ax.set_yticklabels(labels, rotation=0)\n", - "\n", - "xlim = (0, np.max((nbytes / 1000000) / np.array(de_t)) + 100)\n", - "ax.set_xlim(*xlim)\n", - "ax.set_ylim(0, len(de_t))\n", - "ax.set_xlabel('speed (Mb/s)')\n", - "ax.grid(axis='x')\n", - "ax.legend(loc='upper right')\n", - "\n", - "fig.tight_layout();" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import cpuinfo" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Vendor ID: GenuineIntel\n", - "Hardware Raw: \n", - "Brand: Intel(R) Xeon(R) CPU E3-1505M v5 @ 2.80GHz\n", - "Hz Advertised: 2.8000 GHz\n", - "Hz Actual: 1.1000 GHz\n", - "Hz Advertised Raw: (2800000000, 0)\n", - "Hz Actual Raw: (1100000000, 0)\n", - "Arch: X86_64\n", - "Bits: 64\n", - "Count: 8\n", - "Raw Arch String: x86_64\n", - "L2 Cache Size: 8192 KB\n", - "L2 Cache Line Size: 0\n", - "L2 Cache Associativity: 0\n", - "Stepping: 3\n", - "Model: 94\n", - "Family: 6\n", - "Processor Type: 0\n", - "Extended Model: 0\n", - "Extended Family: 0\n", - "Flags: 3dnowprefetch, abm, acpi, adx, aes, aperfmperf, apic, arat, arch_perfmon, avx, avx2, bmi1, bmi2, bts, clflush, clflushopt, cmov, constant_tsc, cx16, cx8, de, ds_cpl, dtes64, dtherm, dts, eagerfpu, epb, ept, erms, est, f16c, flexpriority, fma, fpu, fsgsbase, fxsr, hle, ht, hwp, hwp_act_window, hwp_epp, hwp_noitfy, ida, invpcid, lahf_lm, lm, mca, mce, mmx, monitor, movbe, mpx, msr, mtrr, nonstop_tsc, nopl, nx, pae, pat, pbe, pcid, pclmulqdq, pdcm, pdpe1gb, pebs, pge, pln, pni, popcnt, pse, pse36, pts, rdrand, rdseed, rdtscp, rep_good, rtm, sep, smap, smep, smx, ss, sse, sse2, sse4_1, sse4_2, ssse3, syscall, tm, tm2, tpr_shadow, tsc, tsc_adjust, tsc_deadline_timer, vme, vmx, vnmi, vpid, x2apic, xgetbv1, xsave, xsavec, xsaveopt, xtopology, xtpr\n" - ] - } - ], - "source": [ - "cpuinfo.main()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/object_arrays.ipynb b/notebooks/object_arrays.ipynb deleted file mode 100644 index 714d024907..0000000000 --- a/notebooks/object_arrays.ipynb +++ /dev/null @@ -1,350 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Object arrays\n", - "\n", - "See [#212](https://github.com/alimanfoo/zarr/pull/212) for more information." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.2.0a2.dev82+dirty'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'0.5.0'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import numcodecs\n", - "numcodecs.__version__" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## API changes in Zarr version 2.2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Creation of an object array requires providing new ``object_codec`` argument:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object, object_codec=numcodecs.MsgPack())\n", - "z" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To maintain backwards compatibility with previously-created data, the object codec is treated as a filter and inserted as the first filter in the chain:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Typezarr.core.Array
Data typeobject
Shape(10,)
Chunk shape(5,)
OrderC
Read-onlyFalse
Filter [0]MsgPack(encoding='utf-8')
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes80
No. bytes stored396
Storage ratio0.2
Chunks initialized0/2
" - ], - "text/plain": [ - "Type : zarr.core.Array\n", - "Data type : object\n", - "Shape : (10,)\n", - "Chunk shape : (5,)\n", - "Order : C\n", - "Read-only : False\n", - "Filter [0] : MsgPack(encoding='utf-8')\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 80\n", - "No. bytes stored : 396\n", - "Storage ratio : 0.2\n", - "Chunks initialized : 0/2" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z.info" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array(['foo', 'bar', 1, list([2, 4, 6, 'baz']), {'a': 'b', 'c': 'd'}, None,\n", - " None, None, None, None], dtype=object)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z[0] = 'foo'\n", - "z[1] = b'bar' # msgpack doesn't support bytes objects correctly\n", - "z[2] = 1\n", - "z[3] = [2, 4, 6, 'baz']\n", - "z[4] = {'a': 'b', 'c': 'd'}\n", - "a = z[:]\n", - "a" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If no ``object_codec`` is provided, a ``ValueError`` is raised:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "ename": "ValueError", - "evalue": "missing object_codec for object array", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzarr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/creation.py\u001b[0m in \u001b[0;36mempty\u001b[0;34m(shape, **kwargs)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 205\u001b[0m \"\"\"\n\u001b[0;32m--> 206\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/creation.py\u001b[0m in \u001b[0;36mcreate\u001b[0;34m(shape, chunks, dtype, compressor, fill_value, order, store, synchronizer, overwrite, path, chunk_store, filters, cache_metadata, read_only, object_codec, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor,\n\u001b[1;32m 113\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfill_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverwrite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moverwrite\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m chunk_store=chunk_store, filters=filters, object_codec=object_codec)\n\u001b[0m\u001b[1;32m 115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0;31m# instantiate array\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/storage.py\u001b[0m in \u001b[0;36minit_array\u001b[0;34m(store, shape, chunks, dtype, compressor, fill_value, order, overwrite, path, chunk_store, filters, object_codec)\u001b[0m\n\u001b[1;32m 289\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverwrite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moverwrite\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 290\u001b[0m \u001b[0mchunk_store\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunk_store\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfilters\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 291\u001b[0;31m object_codec=object_codec)\n\u001b[0m\u001b[1;32m 292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 293\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/storage.py\u001b[0m in \u001b[0;36m_init_array_metadata\u001b[0;34m(store, shape, chunks, dtype, compressor, fill_value, order, overwrite, path, chunk_store, filters, object_codec)\u001b[0m\n\u001b[1;32m 346\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mfilters\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[0;31m# there are no filters so we can be sure there is no object codec\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 348\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'missing object_codec for object array'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 349\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 350\u001b[0m \u001b[0;31m# one of the filters may be an object codec, issue a warning rather\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mValueError\u001b[0m: missing object_codec for object array" - ] - } - ], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For API backward-compatibility, if object codec is provided via filters, issue a warning but don't raise an error." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/aliman/src/github/alimanfoo/zarr/zarr/storage.py:353: FutureWarning: missing object_codec for object array; this will raise a ValueError in version 3.0\n", - " 'ValueError in version 3.0', FutureWarning)\n" - ] - } - ], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object, filters=[numcodecs.MsgPack()])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If a user tries to subvert the system and create an object array with no object codec, a runtime check is added to ensure no object arrays are passed down to the compressor (which could lead to nasty errors and/or segfaults):" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "z = zarr.empty(10, chunks=5, dtype=object, object_codec=numcodecs.MsgPack())\n", - "z._filters = None # try to live dangerously, manually wipe filters" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "ename": "RuntimeError", - "evalue": "cannot write object array without object codec", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mz\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'foo'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m__setitem__\u001b[0;34m(self, selection, value)\u001b[0m\n\u001b[1;32m 1094\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1095\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpop_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1096\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1097\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1098\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mset_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36mset_basic_selection\u001b[0;34m(self, selection, value, fields)\u001b[0m\n\u001b[1;32m 1189\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1190\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1191\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_basic_selection_nd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1192\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1193\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mset_orthogonal_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_set_basic_selection_nd\u001b[0;34m(self, selection, value, fields)\u001b[0m\n\u001b[1;32m 1480\u001b[0m \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasicIndexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1481\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1482\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1484\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_set_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_set_selection\u001b[0;34m(self, indexer, value, fields)\u001b[0m\n\u001b[1;32m 1528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1529\u001b[0m \u001b[0;31m# put data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1530\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_chunk_setitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk_coords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_selection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1532\u001b[0m def _chunk_getitem(self, chunk_coords, chunk_selection, out, out_selection,\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_chunk_setitem\u001b[0;34m(self, chunk_coords, chunk_selection, value, fields)\u001b[0m\n\u001b[1;32m 1633\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1634\u001b[0m self._chunk_setitem_nosync(chunk_coords, chunk_selection, value,\n\u001b[0;32m-> 1635\u001b[0;31m fields=fields)\n\u001b[0m\u001b[1;32m 1636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1637\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_chunk_setitem_nosync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_coords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunk_selection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_chunk_setitem_nosync\u001b[0;34m(self, chunk_coords, chunk_selection, value, fields)\u001b[0m\n\u001b[1;32m 1707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1708\u001b[0m \u001b[0;31m# encode chunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1709\u001b[0;31m \u001b[0mcdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_encode_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1710\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1711\u001b[0m \u001b[0;31m# store\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_encode_chunk\u001b[0;34m(self, chunk)\u001b[0m\n\u001b[1;32m 1753\u001b[0m \u001b[0;31m# check object encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1754\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1755\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cannot write object array without object codec'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1756\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1757\u001b[0m \u001b[0;31m# compress\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mRuntimeError\u001b[0m: cannot write object array without object codec" - ] - } - ], - "source": [ - "z[0] = 'foo'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here is another way to subvert the system, wiping filters **after** storing some data. To cover this case a runtime check is added to ensure no object arrays are handled inappropriately during decoding (which could lead to nasty errors and/or segfaults)." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array(['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', 'Hei maailma!',\n", - " 'Xin chào thế giới', 'Njatjeta Botë!', 'Γεια σου κόσμε!', 'こんにちは世界',\n", - " '世界，你好！', 'Helló, világ!', 'Zdravo svete!', 'เฮลโลเวิลด์'], dtype=object)" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from numcodecs.tests.common import greetings\n", - "z = zarr.array(greetings, chunks=5, dtype=object, object_codec=numcodecs.MsgPack())\n", - "z[:]" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "ename": "RuntimeError", - "evalue": "cannot read object array without object codec", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# try to live dangerously, manually wipe filters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mz\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 551\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 552\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpop_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 553\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 554\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 555\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_basic_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mEllipsis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36mget_basic_selection\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 677\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 678\u001b[0m return self._get_basic_selection_nd(selection=selection, out=out,\n\u001b[0;32m--> 679\u001b[0;31m fields=fields)\n\u001b[0m\u001b[1;32m 680\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 681\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_get_basic_selection_zd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_get_basic_selection_nd\u001b[0;34m(self, selection, out, fields)\u001b[0m\n\u001b[1;32m 719\u001b[0m \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBasicIndexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 720\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 721\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 722\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 723\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_orthogonal_selection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfields\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_get_selection\u001b[0;34m(self, indexer, out, fields)\u001b[0m\n\u001b[1;32m 1007\u001b[0m \u001b[0;31m# load chunk selection into output array\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1008\u001b[0m self._chunk_getitem(chunk_coords, chunk_selection, out, out_selection,\n\u001b[0;32m-> 1009\u001b[0;31m drop_axes=indexer.drop_axes, fields=fields)\n\u001b[0m\u001b[1;32m 1010\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1011\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_chunk_getitem\u001b[0;34m(self, chunk_coords, chunk_selection, out, out_selection, drop_axes, fields)\u001b[0m\n\u001b[1;32m 1597\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1598\u001b[0m \u001b[0;31m# decode chunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1599\u001b[0;31m \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_decode_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1600\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1601\u001b[0m \u001b[0;31m# select data from chunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/src/github/alimanfoo/zarr/zarr/core.py\u001b[0m in \u001b[0;36m_decode_chunk\u001b[0;34m(self, cdata)\u001b[0m\n\u001b[1;32m 1733\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1734\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1735\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cannot read object array without object codec'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1736\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1737\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mRuntimeError\u001b[0m: cannot read object array without object codec" - ] - } - ], - "source": [ - "z._filters = [] # try to live dangerously, manually wipe filters\n", - "z[:]" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/repr_info.ipynb b/notebooks/repr_info.ipynb deleted file mode 100644 index 487a4175ba..0000000000 --- a/notebooks/repr_info.ipynb +++ /dev/null @@ -1,365 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import zarr" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root = zarr.group()\n", - "root" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/
Typezarr.hierarchy.Group
Read-onlyFalse
Store typezarr.storage.DictStore
No. members0
No. arrays0
No. groups0
" - ], - "text/plain": [ - "Name : /\n", - "Type : zarr.hierarchy.Group\n", - "Read-only : False\n", - "Store type : zarr.storage.DictStore\n", - "No. members : 0\n", - "No. arrays : 0\n", - "No. groups : 0" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root.info" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "z = root.zeros('foo/bar/baz', shape=1000000)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar/baz
Typezarr.core.Array
Data typefloat64
Shape(1000000,)
Chunk shape(15625,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typezarr.storage.DictStore
No. bytes8000000 (7.6M)
No. bytes stored321
Storage ratio24922.1
Chunks initialized0/64
" - ], - "text/plain": [ - "Name : /foo/bar/baz\n", - "Type : zarr.core.Array\n", - "Data type : float64\n", - "Shape : (1000000,)\n", - "Chunk shape : (15625,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : zarr.storage.DictStore\n", - "No. bytes : 8000000 (7.6M)\n", - "No. bytes stored : 321\n", - "Storage ratio : 24922.1\n", - "Chunks initialized : 0/64" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z.info" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "z[:] = 42" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar/baz
Typezarr.core.Array
Data typefloat64
Shape(1000000,)
Chunk shape(15625,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typezarr.storage.DictStore
No. bytes8000000 (7.6M)
No. bytes stored39553 (38.6K)
Storage ratio202.3
Chunks initialized64/64
" - ], - "text/plain": [ - "Name : /foo/bar/baz\n", - "Type : zarr.core.Array\n", - "Data type : float64\n", - "Shape : (1000000,)\n", - "Chunk shape : (15625,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : zarr.storage.DictStore\n", - "No. bytes : 8000000 (7.6M)\n", - "No. bytes stored : 39553 (38.6K)\n", - "Storage ratio : 202.3\n", - "Chunks initialized : 64/64" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z.info" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(1000):\n", - " root.create_group(i)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/
Typezarr.hierarchy.Group
Read-onlyFalse
Store typezarr.storage.DictStore
No. members1001
No. arrays0
No. groups1001
Groups0, 1, 10, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 11, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 12, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 13, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 14, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 15, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 16, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 17, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 18, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 19, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 2, 20, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 21, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 22, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 23, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 24, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 25, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 26, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 27, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 28, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 29, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 3, 30, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 31, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 32, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 33, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 34, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 35, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 36, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 37, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 38, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 39, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 4, 40, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 41, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 42, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 43, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 44, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 45, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 46, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 47, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 48, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 49, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 5, 50, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 51, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 52, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 53, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 54, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 55, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 56, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 57, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 58, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 59, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 6, 60, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 61, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 62, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 63, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 64, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 65, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 66, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 67, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 68, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 69, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 7, 70, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 71, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 72, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 73, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 74, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 75, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 76, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 77, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 78, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 79, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 8, 80, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 81, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819, 82, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 83, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 84, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 85, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 86, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 87, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 88, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 89, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 9, 90, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 91, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 92, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 93, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 94, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 95, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 96, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 97, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 98, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 99, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, foo
" - ], - "text/plain": [ - "Name : /\n", - "Type : zarr.hierarchy.Group\n", - "Read-only : False\n", - "Store type : zarr.storage.DictStore\n", - "No. members : 1001\n", - "No. arrays : 0\n", - "No. groups : 1001\n", - "Groups : 0, 1, 10, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 11,\n", - " : 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 12, 120, 121,\n", - " : 122, 123, 124, 125, 126, 127, 128, 129, 13, 130, 131, 132, 133,\n", - " : 134, 135, 136, 137, 138, 139, 14, 140, 141, 142, 143, 144, 145,\n", - " : 146, 147, 148, 149, 15, 150, 151, 152, 153, 154, 155, 156, 157,\n", - " : 158, 159, 16, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,\n", - " : 17, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 18, 180,\n", - " : 181, 182, 183, 184, 185, 186, 187, 188, 189, 19, 190, 191, 192,\n", - " : 193, 194, 195, 196, 197, 198, 199, 2, 20, 200, 201, 202, 203, 204,\n", - " : 205, 206, 207, 208, 209, 21, 210, 211, 212, 213, 214, 215, 216,\n", - " : 217, 218, 219, 22, 220, 221, 222, 223, 224, 225, 226, 227, 228,\n", - " : 229, 23, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 24,\n", - " : 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 25, 250, 251,\n", - " : 252, 253, 254, 255, 256, 257, 258, 259, 26, 260, 261, 262, 263,\n", - " : 264, 265, 266, 267, 268, 269, 27, 270, 271, 272, 273, 274, 275,\n", - " : 276, 277, 278, 279, 28, 280, 281, 282, 283, 284, 285, 286, 287,\n", - " : 288, 289, 29, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 3,\n", - " : 30, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 31, 310,\n", - " : 311, 312, 313, 314, 315, 316, 317, 318, 319, 32, 320, 321, 322,\n", - " : 323, 324, 325, 326, 327, 328, 329, 33, 330, 331, 332, 333, 334,\n", - " : 335, 336, 337, 338, 339, 34, 340, 341, 342, 343, 344, 345, 346,\n", - " : 347, 348, 349, 35, 350, 351, 352, 353, 354, 355, 356, 357, 358,\n", - " : 359, 36, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 37,\n", - " : 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 38, 380, 381,\n", - " : 382, 383, 384, 385, 386, 387, 388, 389, 39, 390, 391, 392, 393,\n", - " : 394, 395, 396, 397, 398, 399, 4, 40, 400, 401, 402, 403, 404, 405,\n", - " : 406, 407, 408, 409, 41, 410, 411, 412, 413, 414, 415, 416, 417,\n", - " : 418, 419, 42, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429,\n", - " : 43, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 44, 440,\n", - " : 441, 442, 443, 444, 445, 446, 447, 448, 449, 45, 450, 451, 452,\n", - " : 453, 454, 455, 456, 457, 458, 459, 46, 460, 461, 462, 463, 464,\n", - " : 465, 466, 467, 468, 469, 47, 470, 471, 472, 473, 474, 475, 476,\n", - " : 477, 478, 479, 48, 480, 481, 482, 483, 484, 485, 486, 487, 488,\n", - " : 489, 49, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 5, 50,\n", - " : 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 51, 510, 511,\n", - " : 512, 513, 514, 515, 516, 517, 518, 519, 52, 520, 521, 522, 523,\n", - " : 524, 525, 526, 527, 528, 529, 53, 530, 531, 532, 533, 534, 535,\n", - " : 536, 537, 538, 539, 54, 540, 541, 542, 543, 544, 545, 546, 547,\n", - " : 548, 549, 55, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559,\n", - " : 56, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 57, 570,\n", - " : 571, 572, 573, 574, 575, 576, 577, 578, 579, 58, 580, 581, 582,\n", - " : 583, 584, 585, 586, 587, 588, 589, 59, 590, 591, 592, 593, 594,\n", - " : 595, 596, 597, 598, 599, 6, 60, 600, 601, 602, 603, 604, 605, 606,\n", - " : 607, 608, 609, 61, 610, 611, 612, 613, 614, 615, 616, 617, 618,\n", - " : 619, 62, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 63,\n", - " : 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 64, 640, 641,\n", - " : 642, 643, 644, 645, 646, 647, 648, 649, 65, 650, 651, 652, 653,\n", - " : 654, 655, 656, 657, 658, 659, 66, 660, 661, 662, 663, 664, 665,\n", - " : 666, 667, 668, 669, 67, 670, 671, 672, 673, 674, 675, 676, 677,\n", - " : 678, 679, 68, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689,\n", - " : 69, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 7, 70, 700,\n", - " : 701, 702, 703, 704, 705, 706, 707, 708, 709, 71, 710, 711, 712,\n", - " : 713, 714, 715, 716, 717, 718, 719, 72, 720, 721, 722, 723, 724,\n", - " : 725, 726, 727, 728, 729, 73, 730, 731, 732, 733, 734, 735, 736,\n", - " : 737, 738, 739, 74, 740, 741, 742, 743, 744, 745, 746, 747, 748,\n", - " : 749, 75, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 76,\n", - " : 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 77, 770, 771,\n", - " : 772, 773, 774, 775, 776, 777, 778, 779, 78, 780, 781, 782, 783,\n", - " : 784, 785, 786, 787, 788, 789, 79, 790, 791, 792, 793, 794, 795,\n", - " : 796, 797, 798, 799, 8, 80, 800, 801, 802, 803, 804, 805, 806, 807,\n", - " : 808, 809, 81, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819,\n", - " : 82, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 83, 830,\n", - " : 831, 832, 833, 834, 835, 836, 837, 838, 839, 84, 840, 841, 842,\n", - " : 843, 844, 845, 846, 847, 848, 849, 85, 850, 851, 852, 853, 854,\n", - " : 855, 856, 857, 858, 859, 86, 860, 861, 862, 863, 864, 865, 866,\n", - " : 867, 868, 869, 87, 870, 871, 872, 873, 874, 875, 876, 877, 878,\n", - " : 879, 88, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 89,\n", - " : 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 9, 90, 900, 901,\n", - " : 902, 903, 904, 905, 906, 907, 908, 909, 91, 910, 911, 912, 913,\n", - " : 914, 915, 916, 917, 918, 919, 92, 920, 921, 922, 923, 924, 925,\n", - " : 926, 927, 928, 929, 93, 930, 931, 932, 933, 934, 935, 936, 937,\n", - " : 938, 939, 94, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949,\n", - " : 95, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 96, 960,\n", - " : 961, 962, 963, 964, 965, 966, 967, 968, 969, 97, 970, 971, 972,\n", - " : 973, 974, 975, 976, 977, 978, 979, 98, 980, 981, 982, 983, 984,\n", - " : 985, 986, 987, 988, 989, 99, 990, 991, 992, 993, 994, 995, 996,\n", - " : 997, 998, 999, foo" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root.info" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar
Typezarr.hierarchy.Group
Read-onlyFalse
Store typezarr.storage.DictStore
No. members1
No. arrays1
No. groups0
Arraysbaz
" - ], - "text/plain": [ - "Name : /foo/bar\n", - "Type : zarr.hierarchy.Group\n", - "Read-only : False\n", - "Store type : zarr.storage.DictStore\n", - "No. members : 1\n", - "No. arrays : 1\n", - "No. groups : 0\n", - "Arrays : baz" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "root['foo/bar'].info" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/repr_tree.ipynb b/notebooks/repr_tree.ipynb deleted file mode 100644 index dde8bbb9a0..0000000000 --- a/notebooks/repr_tree.ipynb +++ /dev/null @@ -1,1388 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.3.3.dev87'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "g1 = zarr.group()\n", - "g2 = g1.create_group('foo')\n", - "g3 = g1.create_group('bar')\n", - "g3.create_group('baz')\n", - "g3.create_dataset('xxx', shape=100)\n", - "g3.create_dataset('yyy', shape=(100, 100), dtype='i4')\n", - "g5 = g3.create_group('quux')\n", - "g5.create_dataset('aaa', shape=100)\n", - "g5.create_dataset('bbb', shape=(100, 100), dtype='i4')\n", - "g7 = g3.create_group('zoo')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Generate text (unicode) tree:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/\n", - " ├── bar\n", - " │ ├── baz\n", - " │ ├── quux\n", - " │ │ ├── aaa (100,) float64\n", - " │ │ └── bbb (100, 100) int32\n", - " │ ├── xxx (100,) float64\n", - " │ ├── yyy (100, 100) int32\n", - " │ └── zoo\n", - " └── foo\n" - ] - } - ], - "source": [ - "print(g1.tree())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The ``level`` parameter controls how deep the tree is." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/\n", - " ├── bar\n", - " └── foo\n" - ] - } - ], - "source": [ - "print(g1.tree(level=1))" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/\n", - " ├── bar\n", - " │ ├── baz\n", - " │ ├── quux\n", - " │ ├── xxx (100,) float64\n", - " │ ├── yyy (100, 100) int32\n", - " │ └── zoo\n", - " └── foo\n" - ] - } - ], - "source": [ - "print(g1.tree(level=2))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Alternative plain ASCII tree:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/\n", - " +-- bar\n", - " | +-- baz\n", - " | +-- quux\n", - " | | +-- aaa (100,) float64\n", - " | | +-- bbb (100, 100) int32\n", - " | +-- xxx (100,) float64\n", - " | +-- yyy (100, 100) int32\n", - " | +-- zoo\n", - " +-- foo\n" - ] - } - ], - "source": [ - "print(bytes(g1.tree()).decode())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "HTML trees:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "dc848c2195a54f1386f67cc631871449", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Tree(nodes=(Node(disabled=True, name='/', nodes=(Node(disabled=True, name='bar', nodes=(Node(disabled=True, na…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g1.tree()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Use ``expand=True`` to have all groups automatically expanded." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9459567637cf48cca67ba0145cc3a626", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Tree(nodes=(Node(disabled=True, name='/', nodes=(Node(disabled=True, name='bar', nodes=(Node(disabled=True, na…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g1.tree(expand=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1c635a91c6a444fab7bcb33a33c73097", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Tree(nodes=(Node(disabled=True, name='/', nodes=(Node(disabled=True, name='bar', nodes=(Node(disabled=True, na…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g1.tree(expand=True, level=2)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "0a32522e7c3b4f95b2258027502167f7", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Tree(nodes=(Node(disabled=True, name='/', nodes=(Node(disabled=True, name='bar'), Node(disabled=True, name='fo…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g1.tree(expand=True, level=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The ``expand`` parameter can also be an integer, giving the depth to expand to." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2c9c017eebf94d2aa785439e94cfa533", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Tree(nodes=(Node(disabled=True, name='/', nodes=(Node(disabled=True, name='bar', nodes=(Node(disabled=True, na…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g1.tree(expand=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "edcb8650759d44a995538257d637a17e", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Tree(nodes=(Node(disabled=True, name='/', nodes=(Node(disabled=True, name='bar', nodes=(Node(disabled=True, na…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g1.tree(expand=2)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ee03f431a4fa4395bb913c7107b8f82d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Tree(nodes=(Node(disabled=True, name='/', nodes=(Node(disabled=True, name='bar', nodes=(Node(disabled=True, na…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "g1.tree(expand=3)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.2" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "state": { - "00266d06794e4879b5f200a6da77a8b5": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": {} - }, - "005396454d8a4ccdbd078066066abb25": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "4a5cee4c-65aa-4837-977c-ca04e7be6608", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "foo" - } - }, - "094a129d9ef04f56bbd9e2ef440cdf9f": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "ef9fee92-e6cb-4362-b095-715fa4bff3bc", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "aaa (100,) float64", - "opened": false - } - }, - "0a32522e7c3b4f95b2258027502167f7": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "TreeModel", - "state": { - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "layout": "IPY_MODEL_24a5ef087e4546aeb18b5bd4ea4106d9", - "nodes": [ - "IPY_MODEL_a91c0ac9a6644f18a6ec657cc3c2eb95" - ] - } - }, - "120c8e8cb7664fa0aa0a6ca27a1c5f3f": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "e6f11c15-cf73-4ac9-9737-0f539be27858", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "bar", - "nodes": [ - "IPY_MODEL_741dc4fd91c346f9882b87e9c927362d", - "IPY_MODEL_a6bd734a89044570af371e3f8d4c9fc7", - "IPY_MODEL_a443244f855e40e5bf02be971f470aad", - "IPY_MODEL_9d69e6ca94ae40de8103a7aa51a79a15", - "IPY_MODEL_71ae557f54c944b08874833dbc540339" - ] - } - }, - "14b81f73c9e6404bab187635805952ad": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": {} - }, - "167b0ac3d49b4602900182c8580546eb": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "b7891bd2-6ef7-4d27-ad27-0f017548d302", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "xxx (100,) float64", - "opened": false - } - }, - "1b5ecaaed52a47d6b9846e8626dded36": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "a2ca1d10-8196-491a-898b-cb7ba4578157", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "zoo", - "opened": false - } - }, - "1c635a91c6a444fab7bcb33a33c73097": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "TreeModel", - "state": { - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "layout": "IPY_MODEL_231c730e6b154dd699e528a04e3bceac", - "nodes": [ - "IPY_MODEL_e4c46e3a01564d2986c3a0687e859251" - ] - } - }, - "2004cbc5c93142678c0443ab3260fc76": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "90ee3b0d-847e-4577-a7c3-9b21c666c292", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "/", - "nodes": [ - "IPY_MODEL_2fef457b9982499f87599dfc8404a768", - "IPY_MODEL_98eb0a6675c440ce9672156423051909" - ] - } - }, - "231c730e6b154dd699e528a04e3bceac": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": {} - }, - "2376f63399664a9cb6fcb80112ebd316": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "0120995a-5972-45eb-94a2-4b4480395aa0", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "aaa (100,) float64", - "opened": false - } - }, - "23f8520500534a16903b1d760ee3c1b0": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "d6a76614-eafd-4059-83c0-7dc725969785", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "zoo" - } - }, - "24364dd05e21498a9ee6148d7069470c": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "775d02ae-3e15-4001-9680-89cdda964afe", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "foo" - } - }, - "24a5ef087e4546aeb18b5bd4ea4106d9": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": {} - }, - "2bf13d8729f543c3b62c1dff0bc75f24": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "c994f1f5-1c0d-4a10-b056-54eb6e3654b0", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "quux", - "nodes": [ - "IPY_MODEL_859d15d7e1fb460fa9e1607835028d2f", - "IPY_MODEL_530d9c8872a34f68a5f203c490abea8c" - ] - } - }, - "2c9c017eebf94d2aa785439e94cfa533": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "TreeModel", - "state": { - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "layout": "IPY_MODEL_00266d06794e4879b5f200a6da77a8b5", - "nodes": [ - "IPY_MODEL_2004cbc5c93142678c0443ab3260fc76" - ] - } - }, - "2dd1565f8b5747f59bb0787235310fca": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "e5ac34b6-872f-41cf-84b2-0a2db45bf585", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "/", - "nodes": [ - "IPY_MODEL_c5c799513c214d878e6d9798e1418028", - "IPY_MODEL_7e51196f842148a59de5285587f7cdb3" - ] - } - }, - "2fef457b9982499f87599dfc8404a768": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "a1aad2fe-b5c5-4151-82a7-b681b524f55e", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "bar", - "nodes": [ - "IPY_MODEL_fda99f8dbbbe4636a0bab906c5267b81", - "IPY_MODEL_953f8443d806452aa6009c8d8cd6a00b", - "IPY_MODEL_167b0ac3d49b4602900182c8580546eb", - "IPY_MODEL_3aa51d78e7c14aa58818339ff8583364", - "IPY_MODEL_1b5ecaaed52a47d6b9846e8626dded36" - ], - "opened": false - } - }, - "3aa51d78e7c14aa58818339ff8583364": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "68f3fef0-f0a9-4549-80db-9c8f49e3c7c2", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "yyy (100, 100) int32", - "opened": false - } - }, - "3c31fa266b124cc7baa9572e10fff7d5": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "467cb368-3758-41ac-a037-d901001fedb8", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "foo", - "opened": false - } - }, - "3f686ef60c3742498467169845340486": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "46413eeb-ff88-4c8e-81f3-75175174511c", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "yyy (100, 100) int32", - "opened": false - } - }, - "41537120a09b49d7aef63cf97d3bcbdc": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "7bff3ed2-1676-4dc7-8865-8294df17bdf3", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "quux", - "nodes": [ - "IPY_MODEL_094a129d9ef04f56bbd9e2ef440cdf9f", - "IPY_MODEL_80da3a55f04d4ebfb5199c50553268a8" - ], - "opened": false - } - }, - "530d9c8872a34f68a5f203c490abea8c": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "ce5627ee-6b00-431f-8e0b-3e5f45e3228f", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "bbb (100, 100) int32" - } - }, - "57ce4623088643a18e8eec3ba6003fd7": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "9da1285e-bced-44b8-af47-95272401bcef", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "bar", - "nodes": [ - "IPY_MODEL_8bc5d72953c34131961f747b2cfac812", - "IPY_MODEL_a4b87dfad842423e88c5eeeba8817539", - "IPY_MODEL_81e1f7fa28b8482f9f6ec5694f99233f", - "IPY_MODEL_f394ec7d09a2410a8e19905a6b59751f", - "IPY_MODEL_9c89de1bc6f542bf8c16810dcfd5d703" - ] - } - }, - "585c788c4e60434295e208470bf2b9db": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "961d4b35-c7ad-4503-b212-18d2b5394cac", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "aaa (100,) float64", - "opened": false - } - }, - "5b06653afbb04bcd979f5390326b8115": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "9c26687d-6dab-4daa-95cc-bbbfab950e08", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "xxx (100,) float64", - "opened": false - } - }, - "60e395b510714a92b02b331dedac2c22": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "ec0906d5-3c8e-4c02-89fa-19f052ecb448", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "foo" - } - }, - "6390bb65bfd945aba4993fb82e052b13": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "23323321-dfbd-4fb1-b970-db209ff7dbdf", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "yyy (100, 100) int32" - } - }, - "6e160ea17f5c43248a8ef50ca08b3af4": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "185ee6a2-9064-4e40-a493-cd1fa2039510", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "bbb (100, 100) int32", - "opened": false - } - }, - "71ae557f54c944b08874833dbc540339": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "7442b769-5dd9-466e-bf71-d850dc4d864b", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "zoo" - } - }, - "741dc4fd91c346f9882b87e9c927362d": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "25667d21-0738-4b33-82a6-43cead9dcee3", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "baz" - } - }, - "7aed7c2c71fe4261bfee71d4e13b611a": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "b690530c-9414-47a6-ba1a-47b8582bb18e", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "baz" - } - }, - "7cc19caed25f418d8724b6d61cd40196": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "1ea2d350-3f19-4417-be0f-a8836eeb6015", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "bar", - "nodes": [ - "IPY_MODEL_d12eeab848e548869a02e76ee7deedcb", - "IPY_MODEL_41537120a09b49d7aef63cf97d3bcbdc", - "IPY_MODEL_5b06653afbb04bcd979f5390326b8115", - "IPY_MODEL_3f686ef60c3742498467169845340486", - "IPY_MODEL_d686caad9af343a6ad01cb98d7d46be9" - ], - "opened": false - } - }, - "7e51196f842148a59de5285587f7cdb3": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "66a2f671-756a-403b-b96a-78a4abca32c6", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "foo" - } - }, - "7f23d5608a9b4ae397e32c70bd7897bc": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "556be9fc-a7ff-45bb-b756-6494f58238a4", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "/", - "nodes": [ - "IPY_MODEL_7cc19caed25f418d8724b6d61cd40196", - "IPY_MODEL_3c31fa266b124cc7baa9572e10fff7d5" - ] - } - }, - "80da3a55f04d4ebfb5199c50553268a8": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "bdc7445a-2bdc-4191-bc18-d2e55227c338", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "bbb (100, 100) int32", - "opened": false - } - }, - "81e1f7fa28b8482f9f6ec5694f99233f": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "c4646b6f-fb0a-416e-817a-f52edb067149", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "xxx (100,) float64", - "opened": false - } - }, - "82f52a3353be4a368db1f71e15892a5c": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "92392f3d-ac69-48a2-be99-99d663baf567", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "bbb (100, 100) int32", - "opened": false - } - }, - "84f56fbf5c384e16b409ee918f6d7855": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "3ea55be4-a339-4798-af2e-e37ee07d0c3d", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "yyy (100, 100) int32" - } - }, - "8524ddcc608d42fcaa3ae6384ef98a25": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": {} - }, - "859d15d7e1fb460fa9e1607835028d2f": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "74776201-7656-4b01-a9a4-723b0ec56602", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "aaa (100,) float64" - } - }, - "86f72ec1e4764ff1863e7daef9476476": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "2313648b-d4b1-47d1-a2ee-b07fbda43308", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "baz" - } - }, - "8818d09311f143f28672e8f75be2df01": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "bb7b5a89-cb39-4860-8d61-9419d59514e5", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "xxx (100,) float64" - } - }, - "896a87da7c2e4cd894c0e8bcc03e80fe": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "c73b4900-c1e4-4d60-814b-524de689373a", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "quux" - } - }, - "8bc5d72953c34131961f747b2cfac812": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "db5dcd99-6e88-40d0-b068-af436c365c8d", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "baz", - "opened": false - } - }, - "9459567637cf48cca67ba0145cc3a626": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "TreeModel", - "state": { - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "layout": "IPY_MODEL_14b81f73c9e6404bab187635805952ad", - "nodes": [ - "IPY_MODEL_2dd1565f8b5747f59bb0787235310fca" - ] - } - }, - "953f8443d806452aa6009c8d8cd6a00b": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "18140127-9a50-432d-98bf-297b8677e37e", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "quux", - "nodes": [ - "IPY_MODEL_2376f63399664a9cb6fcb80112ebd316", - "IPY_MODEL_deba2782b4c1464f81652d5f03920084" - ], - "opened": false - } - }, - "98eb0a6675c440ce9672156423051909": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "c89b7b7b-c3a3-4ac5-b2f6-9b938b2f7f98", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "foo", - "opened": false - } - }, - "9c89de1bc6f542bf8c16810dcfd5d703": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "b70bf77b-2424-478d-b009-67dd51fdfa5f", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "zoo", - "opened": false - } - }, - "9cb02d82f4964b9bb80202eb1e1965f0": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": {} - }, - "9d69e6ca94ae40de8103a7aa51a79a15": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "530496d0-18ca-4d43-a6e4-b03d3816a567", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "yyy (100, 100) int32" - } - }, - "a443244f855e40e5bf02be971f470aad": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "cf32dfd6-d234-498c-821a-63424fe24b03", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "xxx (100,) float64" - } - }, - "a4b87dfad842423e88c5eeeba8817539": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "448a6ae8-e4b4-468d-8db1-a92092a5b255", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "quux", - "nodes": [ - "IPY_MODEL_585c788c4e60434295e208470bf2b9db", - "IPY_MODEL_82f52a3353be4a368db1f71e15892a5c" - ], - "opened": false - } - }, - "a6bd734a89044570af371e3f8d4c9fc7": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "36e727be-a1b7-4a0a-9941-aae4f617b6a7", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "quux", - "nodes": [ - "IPY_MODEL_eba07ec14a46420ab8f00565e998424f", - "IPY_MODEL_6e160ea17f5c43248a8ef50ca08b3af4" - ] - } - }, - "a91c0ac9a6644f18a6ec657cc3c2eb95": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "3b5d5c34-8fc1-4e50-be15-16cbae22778a", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "/", - "nodes": [ - "IPY_MODEL_ea998265f10549a1be0fea9579ab5ebd", - "IPY_MODEL_24364dd05e21498a9ee6148d7069470c" - ] - } - }, - "b612bbe40f3045cb9bce7662872d2194": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "5e418578-9312-4ce6-933a-c6d2606d0603", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "bar", - "nodes": [ - "IPY_MODEL_86f72ec1e4764ff1863e7daef9476476", - "IPY_MODEL_896a87da7c2e4cd894c0e8bcc03e80fe", - "IPY_MODEL_e7dd9b21300d4d84bf173916c6b05ccc", - "IPY_MODEL_6390bb65bfd945aba4993fb82e052b13", - "IPY_MODEL_df525d63b58649658b18437078be0b6c" - ] - } - }, - "c5c799513c214d878e6d9798e1418028": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "d8173610-bb78-45eb-9f56-a1e1f404d1e9", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "bar", - "nodes": [ - "IPY_MODEL_7aed7c2c71fe4261bfee71d4e13b611a", - "IPY_MODEL_2bf13d8729f543c3b62c1dff0bc75f24", - "IPY_MODEL_8818d09311f143f28672e8f75be2df01", - "IPY_MODEL_84f56fbf5c384e16b409ee918f6d7855", - "IPY_MODEL_23f8520500534a16903b1d760ee3c1b0" - ] - } - }, - "d00c74f3a7714efb9d89829e16ff4cd8": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": {} - }, - "d12eeab848e548869a02e76ee7deedcb": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "f23a5051-caf9-4527-951d-61bb352a1c4f", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "baz", - "opened": false - } - }, - "d686caad9af343a6ad01cb98d7d46be9": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "ce8441db-4518-4d0d-b9e3-085f662b4c77", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "zoo", - "opened": false - } - }, - "db4c5c98a1bb49afa022d0009d5f4247": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "f6d98bfd-d8bb-4cd1-b714-e4ed7574f0a0", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "/", - "nodes": [ - "IPY_MODEL_57ce4623088643a18e8eec3ba6003fd7", - "IPY_MODEL_de6fc66267e449dd988127f5fecef78d" - ] - } - }, - "dc848c2195a54f1386f67cc631871449": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "TreeModel", - "state": { - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "layout": "IPY_MODEL_8524ddcc608d42fcaa3ae6384ef98a25", - "nodes": [ - "IPY_MODEL_7f23d5608a9b4ae397e32c70bd7897bc" - ] - } - }, - "de6fc66267e449dd988127f5fecef78d": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "1d94bc9b-a7d6-430c-a159-88c7848d2554", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "foo" - } - }, - "deba2782b4c1464f81652d5f03920084": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "c830b45a-298b-42df-a3db-7e5170e612a2", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "bbb (100, 100) int32", - "opened": false - } - }, - "df525d63b58649658b18437078be0b6c": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "2d8dbe4d-b683-452b-987b-84775b0b95db", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "zoo" - } - }, - "e4c46e3a01564d2986c3a0687e859251": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "8f477a24-5aa9-4d1f-8104-947cf2cca963", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "/", - "nodes": [ - "IPY_MODEL_b612bbe40f3045cb9bce7662872d2194", - "IPY_MODEL_005396454d8a4ccdbd078066066abb25" - ] - } - }, - "e7dd9b21300d4d84bf173916c6b05ccc": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "aec6a6ff-3944-4ff5-941d-51bcf07f7bcf", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "xxx (100,) float64" - } - }, - "ea998265f10549a1be0fea9579ab5ebd": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "0bc70d5d-6730-4265-a0f0-9c14c0ea868e", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "bar" - } - }, - "eba07ec14a46420ab8f00565e998424f": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "167e56c3-594e-4629-82b9-c71a12519d04", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "aaa (100,) float64", - "opened": false - } - }, - "edcb8650759d44a995538257d637a17e": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "TreeModel", - "state": { - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "layout": "IPY_MODEL_9cb02d82f4964b9bb80202eb1e1965f0", - "nodes": [ - "IPY_MODEL_db4c5c98a1bb49afa022d0009d5f4247" - ] - } - }, - "ee03f431a4fa4395bb913c7107b8f82d": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "TreeModel", - "state": { - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "layout": "IPY_MODEL_d00c74f3a7714efb9d89829e16ff4cd8", - "nodes": [ - "IPY_MODEL_fc9683e8600b461dbd4edaa9726179ad" - ] - } - }, - "f394ec7d09a2410a8e19905a6b59751f": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "900c0ef5-07f0-45be-9d4f-05a9e08d7709", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "icon": "table", - "name": "yyy (100, 100) int32", - "opened": false - } - }, - "fc9683e8600b461dbd4edaa9726179ad": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "26731003-f7ac-45fa-955d-181a02ba2b3d", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "/", - "nodes": [ - "IPY_MODEL_120c8e8cb7664fa0aa0a6ca27a1c5f3f", - "IPY_MODEL_60e395b510714a92b02b331dedac2c22" - ] - } - }, - "fda99f8dbbbe4636a0bab906c5267b81": { - "model_module": "ipytree", - "model_module_version": "0.1.3", - "model_name": "NodeModel", - "state": { - "_id": "093bd8d7-6f8d-4017-85eb-edf3260c4a30", - "_model_module_version": "0.1.3", - "_view_module_version": "0.1.3", - "disabled": true, - "name": "baz", - "opened": false - } - } - }, - "version_major": 2, - "version_minor": 0 - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/store_benchmark.ipynb b/notebooks/store_benchmark.ipynb deleted file mode 100644 index 869e7df608..0000000000 --- a/notebooks/store_benchmark.ipynb +++ /dev/null @@ -1,1303 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There are lies, damn lies and benchmarks..." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.2.0a2.dev22+dirty'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'6.2.5'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import bsddb3\n", - "bsddb3.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'0.93'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import lmdb\n", - "lmdb.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "import dbm.gnu\n", - "import dbm.ndbm" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import shutil\n", - "bench_dir = '../data/bench'\n", - "\n", - "\n", - "def clean():\n", - " if os.path.isdir(bench_dir):\n", - " shutil.rmtree(bench_dir)\n", - " os.makedirs(bench_dir)\n", - "\n", - " \n", - "def setup(a, name='foo/bar'):\n", - " global fdict_z, hdict_z, lmdb_z, gdbm_z, ndbm_z, bdbm_btree_z, bdbm_hash_z, zip_z, dir_z\n", - " \n", - " clean()\n", - " fdict_root = zarr.group(store=dict())\n", - " hdict_root = zarr.group(store=zarr.DictStore())\n", - " lmdb_root = zarr.group(store=zarr.LMDBStore(os.path.join(bench_dir, 'lmdb')))\n", - " gdbm_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'gdbm'), open=dbm.gnu.open))\n", - " ndbm_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'ndbm'), open=dbm.ndbm.open))\n", - " bdbm_btree_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'bdbm_btree'), open=bsddb3.btopen))\n", - " bdbm_hash_root = zarr.group(store=zarr.DBMStore(os.path.join(bench_dir, 'bdbm_hash'), open=bsddb3.hashopen))\n", - " zip_root = zarr.group(store=zarr.ZipStore(os.path.join(bench_dir, 'zip'), mode='w'))\n", - " dir_root = zarr.group(store=zarr.DirectoryStore(os.path.join(bench_dir, 'dir')))\n", - "\n", - " fdict_z = fdict_root.empty_like(name, a)\n", - " hdict_z = hdict_root.empty_like(name, a)\n", - " lmdb_z = lmdb_root.empty_like(name, a)\n", - " gdbm_z = gdbm_root.empty_like(name, a)\n", - " ndbm_z = ndbm_root.empty_like(name, a)\n", - " bdbm_btree_z = bdbm_btree_root.empty_like(name, a)\n", - " bdbm_hash_z = bdbm_hash_root.empty_like(name, a)\n", - " zip_z = zip_root.empty_like(name, a)\n", - " dir_z = dir_root.empty_like(name, a)\n", - "\n", - " # check compression ratio\n", - " fdict_z[:] = a\n", - " return fdict_z.info\n", - " \n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Main benchmarks" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "def save(a, z):\n", - " if isinstance(z.store, zarr.ZipStore):\n", - " # needed for zip benchmarks to avoid duplicate entries\n", - " z.store.clear()\n", - " z[:] = a\n", - " if hasattr(z.store, 'flush'):\n", - " z.store.flush()\n", - " \n", - " \n", - "def load(z, a):\n", - " z.get_basic_selection(out=a)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## arange" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar
Typezarr.core.Array
Data typeint64
Shape(500000000,)
Chunk shape(488282,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes4000000000 (3.7G)
No. bytes stored59269657 (56.5M)
Storage ratio67.5
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Name : /foo/bar\n", - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (500000000,)\n", - "Chunk shape : (488282,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 4000000000 (3.7G)\n", - "No. bytes stored : 59269657 (56.5M)\n", - "Storage ratio : 67.5\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = np.arange(500000000)\n", - "setup(a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### save" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "324 ms ± 60.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "302 ms ± 11.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, hdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "316 ms ± 12.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, lmdb_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "938 ms ± 111 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, gdbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "406 ms ± 8.93 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, ndbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.43 s ± 156 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, bdbm_btree_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.24 s ± 260 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, bdbm_hash_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "519 ms ± 59.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, zip_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "609 ms ± 48.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit save(a, dir_z)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### load" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "454 ms ± 56.8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(fdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "428 ms ± 13.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(hdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "429 ms ± 19.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(lmdb_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "459 ms ± 10 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(gdbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "473 ms ± 5.71 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(ndbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "504 ms ± 8 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(bdbm_btree_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "519 ms ± 9.59 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(bdbm_hash_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "575 ms ± 12.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(zip_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "494 ms ± 10.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit load(dir_z, a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## randint" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Name/foo/bar
Typezarr.core.Array
Data typeint64
Shape(500000000,)
Chunk shape(488282,)
OrderC
Read-onlyFalse
CompressorBlosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)
Store typebuiltins.dict
No. bytes4000000000 (3.7G)
No. bytes stored2020785466 (1.9G)
Storage ratio2.0
Chunks initialized1024/1024
" - ], - "text/plain": [ - "Name : /foo/bar\n", - "Type : zarr.core.Array\n", - "Data type : int64\n", - "Shape : (500000000,)\n", - "Chunk shape : (488282,)\n", - "Order : C\n", - "Read-only : False\n", - "Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)\n", - "Store type : builtins.dict\n", - "No. bytes : 4000000000 (3.7G)\n", - "No. bytes stored : 2020785466 (1.9G)\n", - "Storage ratio : 2.0\n", - "Chunks initialized : 1024/1024" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.random.seed(42)\n", - "a = np.random.randint(0, 2**30, size=500000000)\n", - "setup(a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### save" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "670 ms ± 78.1 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "611 ms ± 6.11 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, hdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "846 ms ± 24 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, lmdb_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "6.35 s ± 785 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, gdbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4.62 s ± 1.09 s per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, ndbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "7.84 s ± 1.66 s per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, bdbm_btree_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "6.49 s ± 808 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, bdbm_hash_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3.68 s ± 441 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, zip_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3.55 s ± 1.24 s per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 save(a, dir_z)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### load" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "566 ms ± 72.8 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(fdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "521 ms ± 16.1 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(hdict_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "532 ms ± 16.1 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(lmdb_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.2 s ± 10.9 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(gdbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.18 s ± 13.2 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(ndbm_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.59 s ± 16.7 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(bdbm_btree_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.61 s ± 7.31 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(bdbm_hash_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2.33 s ± 19.8 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(zip_z, a)" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "790 ms ± 56 ms per loop (mean ± std. dev. of 3 runs, 1 loop each)\n" - ] - } - ], - "source": [ - "%timeit -r3 load(dir_z, a)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### dask" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [], - "source": [ - "import dask.array as da" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [], - "source": [ - "def dask_op(source, sink, chunks=None):\n", - " if isinstance(sink.store, zarr.ZipStore):\n", - " sink.store.clear()\n", - " if chunks is None:\n", - " try:\n", - " chunks = sink.chunks\n", - " except AttributeError:\n", - " chunks = source.chunks\n", - " d = da.from_array(source, chunks=chunks, asarray=False, fancy=False, lock=False)\n", - " result = (d // 2) * 2\n", - " da.store(result, sink, lock=False)\n", - " if hasattr(sink.store, 'flush'):\n", - " sink.store.flush()\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Compare sources" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.6 s, sys: 1.8 s, total: 17.4 s\n", - "Wall time: 3.07 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.5 s, sys: 104 ms, total: 16.6 s\n", - "Wall time: 2.59 s\n" - ] - } - ], - "source": [ - "%time dask_op(hdict_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.1 s, sys: 524 ms, total: 15.6 s\n", - "Wall time: 3.02 s\n" - ] - } - ], - "source": [ - "%time dask_op(lmdb_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.5 s, sys: 712 ms, total: 17.2 s\n", - "Wall time: 3.13 s\n" - ] - } - ], - "source": [ - "%time dask_op(gdbm_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.3 s, sys: 604 ms, total: 16.9 s\n", - "Wall time: 3.22 s\n" - ] - } - ], - "source": [ - "%time dask_op(ndbm_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 19.6 s, sys: 1.24 s, total: 20.9 s\n", - "Wall time: 3.27 s\n" - ] - } - ], - "source": [ - "%time dask_op(bdbm_btree_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 20.3 s, sys: 1.08 s, total: 21.4 s\n", - "Wall time: 3.53 s\n" - ] - } - ], - "source": [ - "%time dask_op(bdbm_hash_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.7 s, sys: 700 ms, total: 16.4 s\n", - "Wall time: 3.12 s\n" - ] - } - ], - "source": [ - "%time dask_op(zip_z, fdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 17.4 s, sys: 1.08 s, total: 18.5 s\n", - "Wall time: 2.91 s\n" - ] - } - ], - "source": [ - "%time dask_op(dir_z, fdict_z)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Compare sinks" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.8 s, sys: 1.4 s, total: 17.2 s\n", - "Wall time: 3.04 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, hdict_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.2 s, sys: 1.6 s, total: 17.8 s\n", - "Wall time: 2.71 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, lmdb_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.8 s, sys: 3.05 s, total: 19.8 s\n", - "Wall time: 8.01 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, gdbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 17.9 s, sys: 3.01 s, total: 20.9 s\n", - "Wall time: 5.46 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, ndbm_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 13.8 s, sys: 3.39 s, total: 17.2 s\n", - "Wall time: 7.87 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, bdbm_btree_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 13.9 s, sys: 3.27 s, total: 17.2 s\n", - "Wall time: 6.73 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, bdbm_hash_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 13.9 s, sys: 2.5 s, total: 16.4 s\n", - "Wall time: 3.8 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, zip_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.7 s, sys: 3.72 s, total: 19.4 s\n", - "Wall time: 3.1 s\n" - ] - } - ], - "source": [ - "%time dask_op(fdict_z, dir_z)" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [], - "source": [ - "lmdb_z.store.close()\n", - "gdbm_z.store.close()\n", - "ndbm_z.store.close()\n", - "bdbm_btree_z.store.close()\n", - "bdbm_hash_z.store.close()\n", - "zip_z.store.close()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/zip_benchmark.ipynb b/notebooks/zip_benchmark.ipynb deleted file mode 100644 index 6805552422..0000000000 --- a/notebooks/zip_benchmark.ipynb +++ /dev/null @@ -1,343 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.0.2.dev0+dirty'" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import sys\n", - "sys.path.insert(0, '..')\n", - "import zarr\n", - "zarr.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array(/3L/calldata/genotype, (7449486, 773, 2), int8, chunks=(13107, 40, 2), order=C)\n", - " nbytes: 10.7G; nbytes_stored: 193.5M; ratio: 56.7; initialized: 11380/11380\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: ZipStore" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "store = zarr.ZipStore('/data/coluzzi/ag1000g/data/phase1/release/AR3.1/haplotypes/main/zarr2/zstd/ag1000g.phase1.ar3.1.haplotypes.zip',\n", - " mode='r')\n", - "grp = zarr.Group(store)\n", - "z = grp['3L/calldata/genotype']\n", - "z" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 1832 function calls in 0.024 seconds\n", - "\n", - " Ordered by: cumulative time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 1 0.000 0.000 0.024 0.024 {built-in method builtins.exec}\n", - " 1 0.000 0.000 0.024 0.024 :1()\n", - " 1 0.000 0.000 0.024 0.024 core.py:292(__getitem__)\n", - " 20 0.000 0.000 0.023 0.001 core.py:539(_chunk_getitem)\n", - " 20 0.000 0.000 0.020 0.001 core.py:679(_decode_chunk)\n", - " 20 0.000 0.000 0.020 0.001 codecs.py:355(decode)\n", - " 20 0.020 0.001 0.020 0.001 {zarr.blosc.decompress}\n", - " 20 0.000 0.000 0.002 0.000 storage.py:766(__getitem__)\n", - " 20 0.000 0.000 0.001 0.000 zipfile.py:1235(open)\n", - " 20 0.000 0.000 0.001 0.000 zipfile.py:821(read)\n", - " 20 0.000 0.000 0.001 0.000 zipfile.py:901(_read1)\n", - " 80 0.000 0.000 0.001 0.000 zipfile.py:660(read)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:854(_update_crc)\n", - " 40 0.000 0.000 0.000 0.000 {built-in method zlib.crc32}\n", - " 80 0.000 0.000 0.000 0.000 {method 'read' of '_io.BufferedReader' objects}\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:937(_read2)\n", - " 80 0.000 0.000 0.000 0.000 core.py:390()\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:953(close)\n", - " 20 0.000 0.000 0.000 0.000 {method 'reshape' of 'numpy.ndarray' objects}\n", - " 20 0.000 0.000 0.000 0.000 util.py:106(is_total_slice)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:708(__init__)\n", - " 20 0.000 0.000 0.000 0.000 {method 'decode' of 'bytes' objects}\n", - " 20 0.000 0.000 0.000 0.000 core.py:676(_chunk_key)\n", - " 80 0.000 0.000 0.000 0.000 {method 'seek' of '_io.BufferedReader' objects}\n", - " 20 0.000 0.000 0.000 0.000 {built-in method numpy.core.multiarray.frombuffer}\n", - " 80 0.000 0.000 0.000 0.000 core.py:398()\n", - " 20 0.000 0.000 0.000 0.000 {method 'join' of 'str' objects}\n", - " 20 0.000 0.000 0.000 0.000 core.py:386()\n", - " 20 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n", - " 40 0.000 0.000 0.000 0.000 util.py:121()\n", - " 231 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance}\n", - " 20 0.000 0.000 0.000 0.000 cp437.py:14(decode)\n", - " 80 0.000 0.000 0.000 0.000 {method 'tell' of '_io.BufferedReader' objects}\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:667(close)\n", - " 20 0.000 0.000 0.000 0.000 {built-in method _struct.unpack}\n", - " 140 0.000 0.000 0.000 0.000 {built-in method builtins.max}\n", - " 20 0.000 0.000 0.000 0.000 {function ZipExtFile.close at 0x7f8cd5ca2048}\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:1194(getinfo)\n", - " 140 0.000 0.000 0.000 0.000 {built-in method builtins.min}\n", - " 20 0.000 0.000 0.000 0.000 threading.py:1224(current_thread)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:654(__init__)\n", - " 1 0.000 0.000 0.000 0.000 util.py:195(get_chunk_range)\n", - " 20 0.000 0.000 0.000 0.000 {built-in method _codecs.charmap_decode}\n", - " 1 0.000 0.000 0.000 0.000 util.py:166(normalize_array_selection)\n", - " 1 0.000 0.000 0.000 0.000 util.py:198()\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:1715(_fpclose)\n", - " 20 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects}\n", - " 63 0.000 0.000 0.000 0.000 {built-in method builtins.len}\n", - " 1 0.000 0.000 0.000 0.000 {built-in method numpy.core.multiarray.empty}\n", - " 2 0.000 0.000 0.000 0.000 util.py:182()\n", - " 20 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 20 0.000 0.000 0.000 0.000 {built-in method _thread.get_ident}\n", - " 1 0.000 0.000 0.000 0.000 util.py:130(normalize_axis_selection)\n", - " 20 0.000 0.000 0.000 0.000 zipfile.py:636(_get_decompressor)\n", - " 20 0.000 0.000 0.000 0.000 threading.py:1298(main_thread)\n", - " 4 0.000 0.000 0.000 0.000 core.py:373()\n", - " 3 0.000 0.000 0.000 0.000 util.py:187()\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "import cProfile\n", - "cProfile.run('z[:10]', sort='cumtime')" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'0.11.0'" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import dask\n", - "import dask.array as da\n", - "dask.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "d = da.from_array(z, chunks=z.chunks)\n", - "d" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 3min 35s, sys: 4.36 s, total: 3min 40s\n", - "Wall time: 29.5 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[3, 0],\n", - " [1, 0],\n", - " [2, 0],\n", - " ..., \n", - " [2, 8],\n", - " [8, 8],\n", - " [0, 1]])" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time d.sum(axis=1).compute()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Array(/3L/calldata/genotype, (7449486, 773, 2), int8, chunks=(13107, 40, 2), order=C)\n", - " nbytes: 10.7G; nbytes_stored: 193.5M; ratio: 56.7; initialized: 11380/11380\n", - " compressor: Blosc(cname='zstd', clevel=1, shuffle=2)\n", - " store: DirectoryStore" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# compare with same data via directory store\n", - "store_dir = zarr.DirectoryStore('/data/coluzzi/ag1000g/data/phase1/release/AR3.1/haplotypes/main/zarr2/zstd/ag1000g.phase1.ar3.1.haplotypes')\n", - "grp_dir = zarr.Group(store_dir)\n", - "z_dir = grp_dir['3L/calldata/genotype']\n", - "z_dir" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "dask.array" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "d_dir = da.from_array(z_dir, chunks=z_dir.chunks)\n", - "d_dir" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 3min 39s, sys: 4.91 s, total: 3min 44s\n", - "Wall time: 31.1 s\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[3, 0],\n", - " [1, 0],\n", - " [2, 0],\n", - " ..., \n", - " [2, 8],\n", - " [8, 8],\n", - " [0, 1]])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%time d_dir.sum(axis=1).compute()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.1" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/pyproject.toml b/pyproject.toml index 2f21011953..6c18563a1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,451 @@ [build-system] -requires = ["setuptools>=40.8.0", "wheel"] -build-backend = "setuptools.build_meta" +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.sdist] +exclude = [ + "/.github", + "/bench", + "/docs", +] + +[project] +name = "zarr" +description = "An implementation of chunked, compressed, N-dimensional arrays for Python" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "Alistair Miles", email = "alimanfoo@googlemail.com" }, +] +maintainers = [ + { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, + { name = "jakirkham" }, + { name = "Josh Moore", email = "josh@openmicroscopy.org" }, + { name = "Joe Hamman", email = "joe@earthmover.io" }, + { name = "Juan Nunez-Iglesias", email = "juan.nunez-iglesias@monash.edu" }, + { name = "Martin Durant", email = "mdurant@anaconda.com" }, + { name = "Norman Rzepka" }, + { name = "Ryan Abernathey" }, + { name = "David Stansby" }, + { name = "Tom Augspurger", email = "tom.w.augspurger@gmail.com" }, + { name = "Deepak Cherian" } +] +requires-python = ">=3.11" +# If you add a new dependency here, please also add it to .pre-commit-config.yml +dependencies = [ + 'packaging>=22.0', + 'numpy>=1.25', + 'numcodecs[crc32c]>=0.14', + 'typing_extensions>=4.9', + 'donfig>=0.8', +] + +dynamic = [ + "version", +] +classifiers = [ + 'Development Status :: 6 - Mature', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Operating System :: Unix', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', +] +license = {text = "MIT License"} +keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] + +[project.optional-dependencies] +# User extras +remote = [ + "fsspec>=2023.10.0", + "obstore>=0.5.1", +] +gpu = [ + "cupy-cuda12x", +] +# Development extras +test = [ + "coverage", + # Pin possibly due to https://github.com/pytest-dev/pytest-cov/issues/693 + "pytest<8.4", + "pytest-asyncio", + "pytest-cov", + "pytest-accept", + "rich", + "mypy", + "hypothesis", + "pytest-xdist", +] +remote_tests = [ + 'zarr[remote]', + "botocore", + "s3fs>=2023.10.0", + "moto[s3,server]", + "requests", +] +optional = ["rich", "universal-pathlib"] +docs = [ + # Doc building + 'sphinx==8.1.3', + 'sphinx-autobuild>=2021.3.14', + 'sphinx-autoapi==3.4.0', + 'sphinx_design', + 'sphinx-issues', + 'sphinx-copybutton', + 'sphinx-reredirects', + 'pydata-sphinx-theme', + 'numpydoc', + # Changelog generation + 'towncrier', + # Optional dependencies to run examples + 'numcodecs[msgpack]', + 'rich', + 's3fs>=2023.10.0', + 'astroid<4' +] + + +[project.urls] +"Bug Tracker" = "https://github.com/zarr-developers/zarr-python/issues" +Changelog = "https://zarr.readthedocs.io/en/stable/release-notes.html" +Discussions = "https://github.com/zarr-developers/zarr-python/discussions" +Documentation = "https://zarr.readthedocs.io/" +Homepage = "https://github.com/zarr-developers/zarr-python" + +[dependency-groups] +dev = [ + "ipykernel>=6.29.5", + "pip>=25.0.1", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", + "pragma: ${PY_MAJOR_VERSION} no cover", + '.*\.\.\.' # Ignore "..." lines +] + +[tool.coverage.run] +omit = [ + "bench/compress_normal.py", +] + +[tool.hatch] +version.source = "vcs" + +[tool.hatch.build] +hooks.vcs.version-file = "src/zarr/_version.py" + +[tool.hatch.envs.test] +dependencies = [ + "numpy~={matrix:numpy}", +] +features = ["test"] + +[[tool.hatch.envs.test.matrix]] +python = ["3.11", "3.12", "3.13"] +numpy = ["1.25", "2.2"] +deps = ["minimal", "optional"] + +[tool.hatch.envs.test.overrides] +matrix.deps.dependencies = [ + {value = "zarr[remote, remote_tests, test, optional]", if = ["optional"]} +] + +[tool.hatch.envs.test.scripts] +run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src" +run = "run-coverage --no-cov" +run-pytest = "run" +run-verbose = "run-coverage --verbose" +run-mypy = "mypy src" +run-hypothesis = "run-coverage -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*" +list-env = "pip list" + +[tool.hatch.envs.doctest] +features = ["test", "optional", "remote", "remote_tests"] +description = "Test environment for doctests" + +[tool.hatch.envs.doctest.scripts] +run = "rm -r data/; pytest docs/user-guide --doctest-glob='*.rst'" +fix = "rm -r data/; pytest docs/user-guide --doctest-glob='*.rst' --accept" +list-env = "pip list" + +[tool.hatch.envs.gputest] +dependencies = [ + "numpy~={matrix:numpy}", + "universal_pathlib", +] +features = ["test", "gpu"] + +[[tool.hatch.envs.gputest.matrix]] +python = ["3.11", "3.12", "3.13"] +numpy = ["1.25", "2.2"] +version = ["minimal"] + +[tool.hatch.envs.gputest.scripts] +run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run = "run-coverage --no-cov" +run-verbose = "run-coverage --verbose" +run-mypy = "mypy src" +run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*" +list-env = "pip list" + +[tool.hatch.envs.docs] +features = ['docs'] + +[tool.hatch.envs.docs.scripts] +build = "cd docs && make html" +serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0" + +[tool.hatch.envs.upstream] +python = "3.13" +dependencies = [ + 'packaging @ git+https://github.com/pypa/packaging', + 'numpy', # from scientific-python-nightly-wheels + 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', + 's3fs @ git+https://github.com/fsspec/s3fs', + 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', + 'typing_extensions @ git+https://github.com/python/typing_extensions', + 'donfig @ git+https://github.com/pytroll/donfig', + 'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore', + # test deps + 'zarr[test]', +] + +[tool.hatch.envs.upstream.env-vars] +PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" +PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/" +PIP_PRE = "1" + +[tool.hatch.envs.upstream.scripts] +run = "pytest --verbose" +run-mypy = "mypy src" +run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" +run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src" +list-env = "pip list" + +[tool.hatch.envs.min_deps] +description = """Test environment for minimum supported dependencies + +See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/ +""" +python = "3.11" +dependencies = [ + 'zarr[remote]', + 'packaging==22.*', + 'numpy==1.25.*', + 'numcodecs==0.14.*', # 0.14 needed for zarr3 codecs + 'fsspec==2023.10.0', + 's3fs==2023.10.0', + 'universal_pathlib==0.0.22', + 'typing_extensions==4.9.*', + 'donfig==0.8.*', + 'obstore==0.5.*', + # test deps + 'zarr[test]', + 'zarr[remote_tests]', +] + +[tool.hatch.envs.min_deps.scripts] +run = "pytest --verbose" +run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" +list-env = "pip list" +run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy" +run-coverage-html = "pytest --cov-config=pyproject.toml --cov=pkg --cov-report html --cov=src" + +[tool.ruff] +line-length = 100 +force-exclude = true +extend-exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".mypy_cache", + ".nox", + ".pants.d", + ".ruff_cache", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "venv", + "docs", + "tests/test_regression/scripts/", # these are scripts that use a different version of python + "src/zarr/v2/", + "tests/v2/", +] + +[tool.ruff.lint] +extend-select = [ + "ANN", # flake8-annotations + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "EXE", # flake8-executable + "FA", # flake8-future-annotations + "FLY", # flynt + "FURB", # refurb + "G", # flake8-logging-format + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PERF", # Perflint + "PIE", # flake8-pie + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RET", # flake8-return + "RSE", # flake8-raise + "RUF", + "SIM", # flake8-simplify + "SLOT", # flake8-slots + "TC", # flake8-type-checking + "TRY", # tryceratops + "UP", # pyupgrade + "W", # pycodestyle warnings +] +ignore = [ + "ANN401", + "PT011", # TODO: apply this rule + "PT012", # TODO: apply this rule + "RET505", + "RET506", + "RUF005", + "SIM108", + "TRY003", + "UP038", # https://github.com/astral-sh/ruff/issues/7871 + # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + "W191", + "E111", + "E114", + "E117", + "D206", + "D300", + "Q000", + "Q001", + "Q002", + "Q003", + "COM812", + "COM819", + "TC006", +] + +[tool.ruff.lint.extend-per-file-ignores] +"tests/**" = ["ANN001", "ANN201", "RUF029", "SIM117", "SIM300"] + +[tool.mypy] +python_version = "3.11" +ignore_missing_imports = true +namespace_packages = false + +strict = true +warn_unreachable = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] + +[[tool.mypy.overrides]] +module = [ + "tests.package_with_entrypoint.*", + "zarr.testing.stateful", + "tests.test_codecs.test_transpose", + "tests.test_config", + "tests.test_store.test_zip", + "tests.test_store.test_local", + "tests.test_store.test_fsspec", + "tests.test_store.test_memory", + "tests.test_codecs.test_codecs", +] +strict = false + +# TODO: Move the next modules up to the strict = false section +# and fix the errors +[[tool.mypy.overrides]] +module = [ + "tests.test_metadata.*", + "tests.test_store.test_core", + "tests.test_store.test_logging", + "tests.test_store.test_object", + "tests.test_store.test_stateful", + "tests.test_store.test_wrapper", + "tests.test_group", + "tests.test_indexing", + "tests.test_properties", + "tests.test_sync", + "tests.test_regression.scripts.*" +] +ignore_errors = true + +[tool.pytest.ini_options] +minversion = "7" +testpaths = ["tests", "docs/user-guide"] +log_cli_level = "INFO" +xfail_strict = true +asyncio_mode = "auto" +doctest_optionflags = [ + "NORMALIZE_WHITESPACE", + "ELLIPSIS", + "IGNORE_EXCEPTION_DETAIL", +] +addopts = [ + "--durations=10", "-ra", "--strict-config", "--strict-markers", +] +filterwarnings = [ + "error", + # TODO: explicitly filter or catch the warnings below where we expect them to be emitted in the tests + "ignore:Consolidated metadata is currently not part in the Zarr format 3 specification.*:UserWarning", + "ignore:Creating a zarr.buffer.gpu.Buffer with an array that does not support the __cuda_array_interface__.*:UserWarning", + "ignore:Automatic shard shape inference is experimental and may change without notice.*:UserWarning", + "ignore:The codec .* is currently not part in the Zarr format 3 specification.*:UserWarning", + "ignore:The dtype .* is currently not part in the Zarr format 3 specification.*:UserWarning", + "ignore:Use zarr.create_array instead.:DeprecationWarning", + "ignore:Duplicate name.*:UserWarning", + "ignore:The `compressor` argument is deprecated. Use `compressors` instead.:UserWarning", + "ignore:Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.:UserWarning", + "ignore:Unclosed client session =1.3.14 diff --git a/requirements_rtfd.txt b/requirements_rtfd.txt deleted file mode 100644 index 2cdb12377d..0000000000 --- a/requirements_rtfd.txt +++ /dev/null @@ -1,8 +0,0 @@ -asciitree -setuptools -setuptools_scm -sphinx -sphinx-issues -sphinx-rtd-theme -numpydoc -numpy!=1.21.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index a68c77a63f..0000000000 --- a/setup.py +++ /dev/null @@ -1,58 +0,0 @@ -from setuptools import setup - -DESCRIPTION = 'An implementation of chunked, compressed, ' \ - 'N-dimensional arrays for Python.' - -with open('README.md') as f: - LONG_DESCRIPTION = f.read() - -dependencies = [ - 'asciitree', - 'numpy>=1.7', - 'fasteners', - 'numcodecs>=0.6.4', -] - -setup( - name='zarr', - description=DESCRIPTION, - long_description=LONG_DESCRIPTION, - long_description_content_type='text/markdown', - use_scm_version={ - 'version_scheme': 'guess-next-dev', - 'local_scheme': 'dirty-tag', - 'write_to': 'zarr/version.py', - }, - setup_requires=[ - 'setuptools>=38.6.0', - 'setuptools-scm>1.5.4', - ], - extras_require={ - 'jupyter': [ - 'notebook', - 'ipytree', - ], - }, - python_requires='>=3.7, <4', - install_requires=dependencies, - package_dir={'': '.'}, - packages=['zarr', 'zarr._storage', 'zarr.tests'], - classifiers=[ - 'Development Status :: 6 - Mature', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Operating System :: Unix', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ], - maintainer='Alistair Miles', - maintainer_email='alimanfoo@googlemail.com', - url='https://github.com/zarr-developers/zarr-python', - license='MIT', -) diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py new file mode 100644 index 0000000000..0d58ecf8e8 --- /dev/null +++ b/src/zarr/__init__.py @@ -0,0 +1,125 @@ +from zarr._version import version as __version__ +from zarr.api.synchronous import ( + array, + consolidate_metadata, + copy, + copy_all, + copy_store, + create, + create_array, + create_group, + create_hierarchy, + empty, + empty_like, + from_array, + full, + full_like, + group, + load, + ones, + ones_like, + open, + open_array, + open_consolidated, + open_group, + open_like, + save, + save_array, + save_group, + tree, + zeros, + zeros_like, +) +from zarr.core.array import Array, AsyncArray +from zarr.core.config import config +from zarr.core.group import AsyncGroup, Group + +# in case setuptools scm screw up and find version to be 0.0.0 +assert not __version__.startswith("0.0.0") + + +def print_debug_info() -> None: + """ + Print version info for use in bug reports. + """ + import platform + from importlib.metadata import version + + def print_packages(packages: list[str]) -> None: + not_installed = [] + for package in packages: + try: + print(f"{package}: {version(package)}") + except ModuleNotFoundError: + not_installed.append(package) + if not_installed: + print("\n**Not Installed:**") + for package in not_installed: + print(package) + + required = [ + "packaging", + "numpy", + "numcodecs", + "typing_extensions", + "donfig", + ] + optional = [ + "botocore", + "cupy-cuda12x", + "fsspec", + "numcodecs", + "s3fs", + "gcsfs", + "universal-pathlib", + "rich", + "obstore", + ] + + print(f"platform: {platform.platform()}") + print(f"python: {platform.python_version()}") + print(f"zarr: {__version__}\n") + print("**Required dependencies:**") + print_packages(required) + print("\n**Optional dependencies:**") + print_packages(optional) + + +__all__ = [ + "Array", + "AsyncArray", + "AsyncGroup", + "Group", + "__version__", + "array", + "config", + "consolidate_metadata", + "copy", + "copy_all", + "copy_store", + "create", + "create_array", + "create_group", + "create_hierarchy", + "empty", + "empty_like", + "from_array", + "full", + "full_like", + "group", + "load", + "ones", + "ones_like", + "open", + "open_array", + "open_consolidated", + "open_group", + "open_like", + "print_debug_info", + "save", + "save_array", + "save_group", + "tree", + "zeros", + "zeros_like", +] diff --git a/src/zarr/_compat.py b/src/zarr/_compat.py new file mode 100644 index 0000000000..52d96005cc --- /dev/null +++ b/src/zarr/_compat.py @@ -0,0 +1,68 @@ +import warnings +from collections.abc import Callable +from functools import wraps +from inspect import Parameter, signature +from typing import Any, TypeVar + +T = TypeVar("T") + +# Based off https://github.com/scikit-learn/scikit-learn/blob/e87b32a81c70abed8f2e97483758eb64df8255e9/sklearn/utils/validation.py#L63 + + +def _deprecate_positional_args( + func: Callable[..., T] | None = None, *, version: str = "3.1.0" +) -> Callable[..., T]: + """Decorator for methods that issues warnings for positional arguments. + + Using the keyword-only argument syntax in pep 3102, arguments after the + * will issue a warning when passed as a positional argument. + + Parameters + ---------- + func : callable, default=None + Function to check arguments on. + version : callable, default="3.1.0" + The version when positional arguments will result in error. + """ + + def _inner_deprecate_positional_args(f: Callable[..., T]) -> Callable[..., T]: + sig = signature(f) + kwonly_args = [] + all_args = [] + + for name, param in sig.parameters.items(): + if param.kind == Parameter.POSITIONAL_OR_KEYWORD: + all_args.append(name) + elif param.kind == Parameter.KEYWORD_ONLY: + kwonly_args.append(name) + + @wraps(f) + def inner_f(*args: Any, **kwargs: Any) -> T: + extra_args = len(args) - len(all_args) + if extra_args <= 0: + return f(*args, **kwargs) + + # extra_args > 0 + args_msg = [ + f"{name}={arg}" + for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:], strict=False) + ] + formatted_args_msg = ", ".join(args_msg) + warnings.warn( + ( + f"Pass {formatted_args_msg} as keyword args. From version " + f"{version} passing these as positional arguments " + "will result in an error" + ), + FutureWarning, + stacklevel=2, + ) + kwargs.update(zip(sig.parameters, args, strict=False)) + return f(**kwargs) + + return inner_f + + if func is not None: + return _inner_deprecate_positional_args(func) + + return _inner_deprecate_positional_args # type: ignore[return-value] diff --git a/zarr/_storage/__init__.py b/src/zarr/abc/__init__.py similarity index 100% rename from zarr/_storage/__init__.py rename to src/zarr/abc/__init__.py diff --git a/src/zarr/abc/buffer.py b/src/zarr/abc/buffer.py new file mode 100644 index 0000000000..3d5ac07157 --- /dev/null +++ b/src/zarr/abc/buffer.py @@ -0,0 +1,9 @@ +from zarr.core.buffer.core import ArrayLike, Buffer, BufferPrototype, NDArrayLike, NDBuffer + +__all__ = [ + "ArrayLike", + "Buffer", + "BufferPrototype", + "NDArrayLike", + "NDBuffer", +] diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py new file mode 100644 index 0000000000..d9e3520d42 --- /dev/null +++ b/src/zarr/abc/codec.py @@ -0,0 +1,429 @@ +from __future__ import annotations + +from abc import abstractmethod +from typing import TYPE_CHECKING, Generic, TypeVar + +from zarr.abc.metadata import Metadata +from zarr.core.buffer import Buffer, NDBuffer +from zarr.core.common import ChunkCoords, concurrent_map +from zarr.core.config import config + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Iterable + from typing import Self + + from zarr.abc.store import ByteGetter, ByteSetter + from zarr.core.array_spec import ArraySpec + from zarr.core.chunk_grids import ChunkGrid + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + from zarr.core.indexing import SelectorTuple + +__all__ = [ + "ArrayArrayCodec", + "ArrayBytesCodec", + "ArrayBytesCodecPartialDecodeMixin", + "ArrayBytesCodecPartialEncodeMixin", + "BaseCodec", + "BytesBytesCodec", + "CodecInput", + "CodecOutput", + "CodecPipeline", +] + +CodecInput = TypeVar("CodecInput", bound=NDBuffer | Buffer) +CodecOutput = TypeVar("CodecOutput", bound=NDBuffer | Buffer) + + +class BaseCodec(Metadata, Generic[CodecInput, CodecOutput]): + """Generic base class for codecs. + + Codecs can be registered via zarr.codecs.registry. + + Warnings + -------- + This class is not intended to be directly, please use + ArrayArrayCodec, ArrayBytesCodec or BytesBytesCodec for subclassing. + """ + + is_fixed_size: bool + + @abstractmethod + def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int: + """Given an input byte length, this method returns the output byte length. + Raises a NotImplementedError for codecs with variable-sized outputs (e.g. compressors). + + Parameters + ---------- + input_byte_length : int + chunk_spec : ArraySpec + + Returns + ------- + int + """ + ... + + def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: + """Computed the spec of the chunk after it has been encoded by the codec. + This is important for codecs that change the shape, data type or fill value of a chunk. + The spec will then be used for subsequent codecs in the pipeline. + + Parameters + ---------- + chunk_spec : ArraySpec + + Returns + ------- + ArraySpec + """ + return chunk_spec + + def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: + """Fills in codec configuration parameters that can be automatically + inferred from the array metadata. + + Parameters + ---------- + array_spec : ArraySpec + + Returns + ------- + Self + """ + return self + + def validate( + self, + *, + shape: ChunkCoords, + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_grid: ChunkGrid, + ) -> None: + """Validates that the codec configuration is compatible with the array metadata. + Raises errors when the codec configuration is not compatible. + + Parameters + ---------- + shape : ChunkCoords + The array shape + dtype : np.dtype[Any] + The array data type + chunk_grid : ChunkGrid + The array chunk grid + """ + + async def _decode_single(self, chunk_data: CodecOutput, chunk_spec: ArraySpec) -> CodecInput: + raise NotImplementedError + + async def decode( + self, + chunks_and_specs: Iterable[tuple[CodecOutput | None, ArraySpec]], + ) -> Iterable[CodecInput | None]: + """Decodes a batch of chunks. + Chunks can be None in which case they are ignored by the codec. + + Parameters + ---------- + chunks_and_specs : Iterable[tuple[CodecOutput | None, ArraySpec]] + Ordered set of encoded chunks with their accompanying chunk spec. + + Returns + ------- + Iterable[CodecInput | None] + """ + return await _batching_helper(self._decode_single, chunks_and_specs) + + async def _encode_single( + self, chunk_data: CodecInput, chunk_spec: ArraySpec + ) -> CodecOutput | None: + raise NotImplementedError + + async def encode( + self, + chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]], + ) -> Iterable[CodecOutput | None]: + """Encodes a batch of chunks. + Chunks can be None in which case they are ignored by the codec. + + Parameters + ---------- + chunks_and_specs : Iterable[tuple[CodecInput | None, ArraySpec]] + Ordered set of to-be-encoded chunks with their accompanying chunk spec. + + Returns + ------- + Iterable[CodecOutput | None] + """ + return await _batching_helper(self._encode_single, chunks_and_specs) + + +class ArrayArrayCodec(BaseCodec[NDBuffer, NDBuffer]): + """Base class for array-to-array codecs.""" + + +class ArrayBytesCodec(BaseCodec[NDBuffer, Buffer]): + """Base class for array-to-bytes codecs.""" + + +class BytesBytesCodec(BaseCodec[Buffer, Buffer]): + """Base class for bytes-to-bytes codecs.""" + + +Codec = ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec + + +class ArrayBytesCodecPartialDecodeMixin: + """Mixin for array-to-bytes codecs that implement partial decoding.""" + + async def _decode_partial_single( + self, byte_getter: ByteGetter, selection: SelectorTuple, chunk_spec: ArraySpec + ) -> NDBuffer | None: + raise NotImplementedError + + async def decode_partial( + self, + batch_info: Iterable[tuple[ByteGetter, SelectorTuple, ArraySpec]], + ) -> Iterable[NDBuffer | None]: + """Partially decodes a batch of chunks. + This method determines parts of a chunk from the slice selection, + fetches these parts from the store (via ByteGetter) and decodes them. + + Parameters + ---------- + batch_info : Iterable[tuple[ByteGetter, SelectorTuple, ArraySpec]] + Ordered set of information about slices of encoded chunks. + The slice selection determines which parts of the chunk will be fetched. + The ByteGetter is used to fetch the necessary bytes. + The chunk spec contains information about the construction of an array from the bytes. + + Returns + ------- + Iterable[NDBuffer | None] + """ + return await concurrent_map( + list(batch_info), + self._decode_partial_single, + config.get("async.concurrency"), + ) + + +class ArrayBytesCodecPartialEncodeMixin: + """Mixin for array-to-bytes codecs that implement partial encoding.""" + + async def _encode_partial_single( + self, + byte_setter: ByteSetter, + chunk_array: NDBuffer, + selection: SelectorTuple, + chunk_spec: ArraySpec, + ) -> None: + raise NotImplementedError + + async def encode_partial( + self, + batch_info: Iterable[tuple[ByteSetter, NDBuffer, SelectorTuple, ArraySpec]], + ) -> None: + """Partially encodes a batch of chunks. + This method determines parts of a chunk from the slice selection, encodes them and + writes these parts to the store (via ByteSetter). + If merging with existing chunk data in the store is necessary, this method will + read from the store first and perform the merge. + + Parameters + ---------- + batch_info : Iterable[tuple[ByteSetter, NDBuffer, SelectorTuple, ArraySpec]] + Ordered set of information about slices of to-be-encoded chunks. + The slice selection determines which parts of the chunk will be encoded. + The ByteSetter is used to write the necessary bytes and fetch bytes for existing chunk data. + The chunk spec contains information about the chunk. + """ + await concurrent_map( + list(batch_info), + self._encode_partial_single, + config.get("async.concurrency"), + ) + + +class CodecPipeline: + """Base class for implementing CodecPipeline. + A CodecPipeline implements the read and write paths for chunk data. + On the read path, it is responsible for fetching chunks from a store (via ByteGetter), + decoding them and assembling an output array. On the write path, it encodes the chunks + and writes them to a store (via ByteSetter).""" + + @abstractmethod + def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: + """Fills in codec configuration parameters that can be automatically + inferred from the array metadata. + + Parameters + ---------- + array_spec : ArraySpec + + Returns + ------- + Self + """ + ... + + @classmethod + @abstractmethod + def from_codecs(cls, codecs: Iterable[Codec]) -> Self: + """Creates a codec pipeline from an iterable of codecs. + + Parameters + ---------- + codecs : Iterable[Codec] + + Returns + ------- + Self + """ + ... + + @property + @abstractmethod + def supports_partial_decode(self) -> bool: ... + + @property + @abstractmethod + def supports_partial_encode(self) -> bool: ... + + @abstractmethod + def validate( + self, *, shape: ChunkCoords, dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGrid + ) -> None: + """Validates that all codec configurations are compatible with the array metadata. + Raises errors when a codec configuration is not compatible. + + Parameters + ---------- + shape : ChunkCoords + The array shape + dtype : np.dtype[Any] + The array data type + chunk_grid : ChunkGrid + The array chunk grid + """ + ... + + @abstractmethod + def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int: + """Given an input byte length, this method returns the output byte length. + Raises a NotImplementedError for codecs with variable-sized outputs (e.g. compressors). + + Parameters + ---------- + byte_length : int + array_spec : ArraySpec + + Returns + ------- + int + """ + ... + + @abstractmethod + async def decode( + self, + chunk_bytes_and_specs: Iterable[tuple[Buffer | None, ArraySpec]], + ) -> Iterable[NDBuffer | None]: + """Decodes a batch of chunks. + Chunks can be None in which case they are ignored by the codec. + + Parameters + ---------- + chunk_bytes_and_specs : Iterable[tuple[Buffer | None, ArraySpec]] + Ordered set of encoded chunks with their accompanying chunk spec. + + Returns + ------- + Iterable[NDBuffer | None] + """ + ... + + @abstractmethod + async def encode( + self, + chunk_arrays_and_specs: Iterable[tuple[NDBuffer | None, ArraySpec]], + ) -> Iterable[Buffer | None]: + """Encodes a batch of chunks. + Chunks can be None in which case they are ignored by the codec. + + Parameters + ---------- + chunk_arrays_and_specs : Iterable[tuple[NDBuffer | None, ArraySpec]] + Ordered set of to-be-encoded chunks with their accompanying chunk spec. + + Returns + ------- + Iterable[Buffer | None] + """ + ... + + @abstractmethod + async def read( + self, + batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], + out: NDBuffer, + drop_axes: tuple[int, ...] = (), + ) -> None: + """Reads chunk data from the store, decodes it and writes it into an output array. + Partial decoding may be utilized if the codecs and stores support it. + + Parameters + ---------- + batch_info : Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]] + Ordered set of information about the chunks. + The first slice selection determines which parts of the chunk will be fetched. + The second slice selection determines where in the output array the chunk data will be written. + The ByteGetter is used to fetch the necessary bytes. + The chunk spec contains information about the construction of an array from the bytes. + out : NDBuffer + """ + ... + + @abstractmethod + async def write( + self, + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], + value: NDBuffer, + drop_axes: tuple[int, ...] = (), + ) -> None: + """Encodes chunk data and writes it to the store. + Merges with existing chunk data by reading first, if necessary. + Partial encoding may be utilized if the codecs and stores support it. + + Parameters + ---------- + batch_info : Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple]] + Ordered set of information about the chunks. + The first slice selection determines which parts of the chunk will be encoded. + The second slice selection determines where in the value array the chunk data is located. + The ByteSetter is used to fetch and write the necessary bytes. + The chunk spec contains information about the chunk. + value : NDBuffer + """ + ... + + +async def _batching_helper( + func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput | None]], + batch_info: Iterable[tuple[CodecInput | None, ArraySpec]], +) -> list[CodecOutput | None]: + return await concurrent_map( + list(batch_info), + _noop_for_none(func), + config.get("async.concurrency"), + ) + + +def _noop_for_none( + func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput | None]], +) -> Callable[[CodecInput | None, ArraySpec], Awaitable[CodecOutput | None]]: + async def wrap(chunk: CodecInput | None, chunk_spec: ArraySpec) -> CodecOutput | None: + if chunk is None: + return None + return await func(chunk, chunk_spec) + + return wrap diff --git a/src/zarr/abc/metadata.py b/src/zarr/abc/metadata.py new file mode 100644 index 0000000000..a56f986645 --- /dev/null +++ b/src/zarr/abc/metadata.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Self + + from zarr.core.common import JSON + +from dataclasses import dataclass, fields + +__all__ = ["Metadata"] + + +@dataclass(frozen=True) +class Metadata: + def to_dict(self) -> dict[str, JSON]: + """ + Recursively serialize this model to a dictionary. + This method inspects the fields of self and calls `x.to_dict()` for any fields that + are instances of `Metadata`. Sequences of `Metadata` are similarly recursed into, and + the output of that recursion is collected in a list. + """ + out_dict = {} + for field in fields(self): + key = field.name + value = getattr(self, key) + if isinstance(value, Metadata): + out_dict[field.name] = getattr(self, field.name).to_dict() + elif isinstance(value, str): + out_dict[key] = value + elif isinstance(value, Sequence): + out_dict[key] = tuple(v.to_dict() if isinstance(v, Metadata) else v for v in value) + else: + out_dict[key] = value + + return out_dict + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> Self: + """ + Create an instance of the model from a dictionary + """ + + return cls(**data) diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py new file mode 100644 index 0000000000..1fbdb3146c --- /dev/null +++ b/src/zarr/abc/store.py @@ -0,0 +1,520 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from asyncio import gather +from dataclasses import dataclass +from itertools import starmap +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.common import concurrent_map +from zarr.core.config import config + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, AsyncIterator, Iterable + from types import TracebackType + from typing import Any, Self, TypeAlias + + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.common import BytesLike + +__all__ = ["ByteGetter", "ByteSetter", "Store", "set_or_delete"] + + +@dataclass +class RangeByteRequest: + """Request a specific byte range""" + + start: int + """The start of the byte range request (inclusive).""" + end: int + """The end of the byte range request (exclusive).""" + + +@dataclass +class OffsetByteRequest: + """Request all bytes starting from a given byte offset""" + + offset: int + """The byte offset for the offset range request.""" + + +@dataclass +class SuffixByteRequest: + """Request up to the last `n` bytes""" + + suffix: int + """The number of bytes from the suffix to request.""" + + +ByteRequest: TypeAlias = RangeByteRequest | OffsetByteRequest | SuffixByteRequest + + +class Store(ABC): + """ + Abstract base class for Zarr stores. + """ + + _read_only: bool + _is_open: bool + + def __init__(self, *, read_only: bool = False) -> None: + self._is_open = False + self._read_only = read_only + + @classmethod + async def open(cls, *args: Any, **kwargs: Any) -> Self: + """ + Create and open the store. + + Parameters + ---------- + *args : Any + Positional arguments to pass to the store constructor. + **kwargs : Any + Keyword arguments to pass to the store constructor. + + Returns + ------- + Store + The opened store instance. + """ + store = cls(*args, **kwargs) + await store._open() + return store + + def with_read_only(self, read_only: bool = False) -> Store: + """ + Return a new store with a new read_only setting. + + The new store points to the same location with the specified new read_only state. + The returned Store is not automatically opened, and this store is + not automatically closed. + + Parameters + ---------- + read_only + If True, the store will be created in read-only mode. Defaults to False. + + Returns + ------- + A new store of the same type with the new read only attribute. + """ + raise NotImplementedError( + f"with_read_only is not implemented for the {type(self)} store type." + ) + + def __enter__(self) -> Self: + """Enter a context manager that will close the store upon exiting.""" + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """Close the store.""" + self.close() + + async def _open(self) -> None: + """ + Open the store. + + Raises + ------ + ValueError + If the store is already open. + """ + if self._is_open: + raise ValueError("store is already open") + self._is_open = True + + async def _ensure_open(self) -> None: + """Open the store if it is not already open.""" + if not self._is_open: + await self._open() + + async def is_empty(self, prefix: str) -> bool: + """ + Check if the directory is empty. + + Parameters + ---------- + prefix : str + Prefix of keys to check. + + Returns + ------- + bool + True if the store is empty, False otherwise. + """ + if not self.supports_listing: + raise NotImplementedError + if prefix != "" and not prefix.endswith("/"): + prefix += "/" + async for _ in self.list_prefix(prefix): + return False + return True + + async def clear(self) -> None: + """ + Clear the store. + + Remove all keys and values from the store. + """ + if not self.supports_deletes: + raise NotImplementedError + if not self.supports_listing: + raise NotImplementedError + self._check_writable() + await self.delete_dir("") + + @property + def read_only(self) -> bool: + """Is the store read-only?""" + return self._read_only + + def _check_writable(self) -> None: + """Raise an exception if the store is not writable.""" + if self.read_only: + raise ValueError("store was opened in read-only mode and does not support writing") + + @abstractmethod + def __eq__(self, value: object) -> bool: + """Equality comparison.""" + ... + + @abstractmethod + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + """Retrieve the value associated with a given key. + + Parameters + ---------- + key : str + prototype : BufferPrototype + The prototype of the output buffer. Stores may support a default buffer prototype. + byte_range : ByteRequest, optional + ByteRequest may be one of the following. If not provided, all data associated with the key is retrieved. + - RangeByteRequest(int, int): Request a specific range of bytes in the form (start, end). The end is exclusive. If the given range is zero-length or starts after the end of the object, an error will be returned. Additionally, if the range ends after the end of the object, the entire remainder of the object will be returned. Otherwise, the exact requested range will be returned. + - OffsetByteRequest(int): Request all bytes starting from a given byte offset. This is equivalent to bytes={int}- as an HTTP header. + - SuffixByteRequest(int): Request the last int bytes. Note that here, int is the size of the request, not the byte offset. This is equivalent to bytes=-{int} as an HTTP header. + + Returns + ------- + Buffer + """ + ... + + @abstractmethod + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + """Retrieve possibly partial values from given key_ranges. + + Parameters + ---------- + prototype : BufferPrototype + The prototype of the output buffer. Stores may support a default buffer prototype. + key_ranges : Iterable[tuple[str, tuple[int | None, int | None]]] + Ordered set of key, range pairs, a key may occur multiple times with different ranges + + Returns + ------- + list of values, in the order of the key_ranges, may contain null/none for missing keys + """ + ... + + @abstractmethod + async def exists(self, key: str) -> bool: + """Check if a key exists in the store. + + Parameters + ---------- + key : str + + Returns + ------- + bool + """ + ... + + @property + @abstractmethod + def supports_writes(self) -> bool: + """Does the store support writes?""" + ... + + @abstractmethod + async def set(self, key: str, value: Buffer) -> None: + """Store a (key, value) pair. + + Parameters + ---------- + key : str + value : Buffer + """ + ... + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + """ + Store a key to ``value`` if the key is not already present. + + Parameters + ---------- + key : str + value : Buffer + """ + # Note for implementers: the default implementation provided here + # is not safe for concurrent writers. There's a race condition between + # the `exists` check and the `set` where another writer could set some + # value at `key` or delete `key`. + if not await self.exists(key): + await self.set(key, value) + + async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None: + """ + Insert multiple (key, value) pairs into storage. + """ + await gather(*starmap(self.set, values)) + + @property + def supports_consolidated_metadata(self) -> bool: + """ + Does the store support consolidated metadata?. + + If it doesn't an error will be raised on requests to consolidate the metadata. + Returning `False` can be useful for stores which implement their own + consolidation mechanism outside of the zarr-python implementation. + """ + + return True + + @property + @abstractmethod + def supports_deletes(self) -> bool: + """Does the store support deletes?""" + ... + + @abstractmethod + async def delete(self, key: str) -> None: + """Remove a key from the store + + Parameters + ---------- + key : str + """ + ... + + @property + @abstractmethod + def supports_partial_writes(self) -> bool: + """Does the store support partial writes?""" + ... + + @abstractmethod + async def set_partial_values( + self, key_start_values: Iterable[tuple[str, int, BytesLike]] + ) -> None: + """Store values at a given key, starting at byte range_start. + + Parameters + ---------- + key_start_values : list[tuple[str, int, BytesLike]] + set of key, range_start, values triples, a key may occur multiple times with different + range_starts, range_starts (considering the length of the respective values) must not + specify overlapping ranges for the same key + """ + ... + + @property + @abstractmethod + def supports_listing(self) -> bool: + """Does the store support listing?""" + ... + + @abstractmethod + def list(self) -> AsyncIterator[str]: + """Retrieve all keys in the store. + + Returns + ------- + AsyncIterator[str] + """ + # This method should be async, like overridden methods in child classes. + # However, that's not straightforward: + # https://stackoverflow.com/questions/68905848 + + @abstractmethod + def list_prefix(self, prefix: str) -> AsyncIterator[str]: + """ + Retrieve all keys in the store that begin with a given prefix. Keys are returned relative + to the root of the store. + + Parameters + ---------- + prefix : str + + Returns + ------- + AsyncIterator[str] + """ + # This method should be async, like overridden methods in child classes. + # However, that's not straightforward: + # https://stackoverflow.com/questions/68905848 + + @abstractmethod + def list_dir(self, prefix: str) -> AsyncIterator[str]: + """ + Retrieve all keys and prefixes with a given prefix and which do not contain the character + “/” after the given prefix. + + Parameters + ---------- + prefix : str + + Returns + ------- + AsyncIterator[str] + """ + # This method should be async, like overridden methods in child classes. + # However, that's not straightforward: + # https://stackoverflow.com/questions/68905848 + + async def delete_dir(self, prefix: str) -> None: + """ + Remove all keys and prefixes in the store that begin with a given prefix. + """ + if not self.supports_deletes: + raise NotImplementedError + if not self.supports_listing: + raise NotImplementedError + self._check_writable() + if prefix != "" and not prefix.endswith("/"): + prefix += "/" + async for key in self.list_prefix(prefix): + await self.delete(key) + + def close(self) -> None: + """Close the store.""" + self._is_open = False + + async def _get_many( + self, requests: Iterable[tuple[str, BufferPrototype, ByteRequest | None]] + ) -> AsyncGenerator[tuple[str, Buffer | None], None]: + """ + Retrieve a collection of objects from storage. In general this method does not guarantee + that objects will be retrieved in the order in which they were requested, so this method + yields tuple[str, Buffer | None] instead of just Buffer | None + """ + for req in requests: + yield (req[0], await self.get(*req)) + + async def getsize(self, key: str) -> int: + """ + Return the size, in bytes, of a value in a Store. + + Parameters + ---------- + key : str + + Returns + ------- + nbytes : int + The size of the value (in bytes). + + Raises + ------ + FileNotFoundError + When the given key does not exist in the store. + """ + # Note to implementers: this default implementation is very inefficient since + # it requires reading the entire object. Many systems will have ways to get the + # size of an object without reading it. + value = await self.get(key, prototype=default_buffer_prototype()) + if value is None: + raise FileNotFoundError(key) + return len(value) + + async def getsize_prefix(self, prefix: str) -> int: + """ + Return the size, in bytes, of all values under a prefix. + + Parameters + ---------- + prefix : str + The prefix of the directory to measure. + + Returns + ------- + nbytes : int + The sum of the sizes of the values in the directory (in bytes). + + See Also + -------- + zarr.Array.nbytes_stored + Store.getsize + + Notes + ----- + ``getsize_prefix`` is just provided as a potentially faster alternative to + listing all the keys under a prefix calling :meth:`Store.getsize` on each. + + In general, ``prefix`` should be the path of an Array or Group in the Store. + Implementations may differ on the behavior when some other ``prefix`` + is provided. + """ + # TODO: Overlap listing keys with getsize calls. + # Currently, we load the list of keys into memory and only then move + # on to getting sizes. Ideally we would overlap those two, which should + # improve tail latency and might reduce memory pressure (since not all keys + # would be in memory at once). + keys = [(x,) async for x in self.list_prefix(prefix)] + limit = config.get("async.concurrency") + sizes = await concurrent_map(keys, self.getsize, limit=limit) + return sum(sizes) + + +@runtime_checkable +class ByteGetter(Protocol): + async def get( + self, prototype: BufferPrototype, byte_range: ByteRequest | None = None + ) -> Buffer | None: ... + + +@runtime_checkable +class ByteSetter(Protocol): + async def get( + self, prototype: BufferPrototype, byte_range: ByteRequest | None = None + ) -> Buffer | None: ... + + async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None: ... + + async def delete(self) -> None: ... + + async def set_if_not_exists(self, default: Buffer) -> None: ... + + +async def set_or_delete(byte_setter: ByteSetter, value: Buffer | None) -> None: + """Set or delete a value in a byte setter + + Parameters + ---------- + byte_setter : ByteSetter + value : Buffer | None + + Notes + ----- + If value is None, the key will be deleted. + """ + if value is None: + await byte_setter.delete() + else: + await byte_setter.set(value) diff --git a/zarr/tests/__init__.py b/src/zarr/api/__init__.py similarity index 100% rename from zarr/tests/__init__.py rename to src/zarr/api/__init__.py diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py new file mode 100644 index 0000000000..3b53095636 --- /dev/null +++ b/src/zarr/api/asynchronous.py @@ -0,0 +1,1350 @@ +from __future__ import annotations + +import asyncio +import dataclasses +import warnings +from typing import TYPE_CHECKING, Any, Literal, cast + +import numpy as np +import numpy.typing as npt +from typing_extensions import deprecated + +from zarr.abc.store import Store +from zarr.core.array import ( + Array, + AsyncArray, + CompressorLike, + _get_default_chunk_encoding_v2, + create_array, + from_array, + get_array_metadata, +) +from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, ArrayConfigParams +from zarr.core.buffer import NDArrayLike +from zarr.core.common import ( + JSON, + AccessModeLiteral, + ChunkCoords, + DimensionNames, + MemoryOrder, + ZarrFormat, + _default_zarr_format, + _warn_order_kwarg, + _warn_write_empty_chunks_kwarg, +) +from zarr.core.dtype import ZDTypeLike, get_data_type_from_native_dtype, parse_data_type +from zarr.core.group import ( + AsyncGroup, + ConsolidatedMetadata, + GroupMetadata, + create_hierarchy, +) +from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata +from zarr.errors import GroupNotFoundError, NodeTypeValidationError +from zarr.storage import StorePath +from zarr.storage._common import make_store_path + +if TYPE_CHECKING: + from collections.abc import Iterable + + from zarr.abc.codec import Codec + from zarr.core.buffer import NDArrayLikeOrScalar + from zarr.core.chunk_key_encodings import ChunkKeyEncoding + from zarr.storage import StoreLike + + # TODO: this type could use some more thought + ArrayLike = AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | Array | npt.NDArray[Any] + PathLike = str + +__all__ = [ + "array", + "consolidate_metadata", + "copy", + "copy_all", + "copy_store", + "create", + "create_array", + "create_hierarchy", + "empty", + "empty_like", + "from_array", + "full", + "full_like", + "group", + "load", + "ones", + "ones_like", + "open", + "open_array", + "open_consolidated", + "open_group", + "open_like", + "save", + "save_array", + "save_group", + "tree", + "zeros", + "zeros_like", +] + + +_READ_MODES: tuple[AccessModeLiteral, ...] = ("r", "r+", "a") +_CREATE_MODES: tuple[AccessModeLiteral, ...] = ("a", "w", "w-") +_OVERWRITE_MODES: tuple[AccessModeLiteral, ...] = ("w",) + + +def _infer_overwrite(mode: AccessModeLiteral) -> bool: + """ + Check that an ``AccessModeLiteral`` is compatible with overwriting an existing Zarr node. + """ + return mode in _OVERWRITE_MODES + + +def _get_shape_chunks(a: ArrayLike | Any) -> tuple[ChunkCoords | None, ChunkCoords | None]: + """Helper function to get the shape and chunks from an array-like object""" + shape = None + chunks = None + + if hasattr(a, "shape") and isinstance(a.shape, tuple): + shape = a.shape + + if hasattr(a, "chunks") and isinstance(a.chunks, tuple) and (len(a.chunks) == len(a.shape)): + chunks = a.chunks + + elif hasattr(a, "chunklen"): + # bcolz carray + chunks = (a.chunklen,) + a.shape[1:] + + return shape, chunks + + +def _like_args(a: ArrayLike, kwargs: dict[str, Any]) -> dict[str, Any]: + """Set default values for shape and chunks if they are not present in the array-like object""" + + new = kwargs.copy() + + shape, chunks = _get_shape_chunks(a) + if shape is not None: + new["shape"] = shape + if chunks is not None: + new["chunks"] = chunks + + if hasattr(a, "dtype"): + new["dtype"] = a.dtype + + if isinstance(a, AsyncArray): + new["order"] = a.order + if isinstance(a.metadata, ArrayV2Metadata): + new["compressor"] = a.metadata.compressor + new["filters"] = a.metadata.filters + else: + # TODO: Remove type: ignore statement when type inference improves. + # mypy cannot correctly infer the type of a.metadata here for some reason. + new["codecs"] = a.metadata.codecs # type: ignore[unreachable] + + else: + # TODO: set default values compressor/codecs + # to do this, we may need to evaluate if this is a v2 or v3 array + # new["compressor"] = "default" + pass + + return new + + +def _handle_zarr_version_or_format( + *, zarr_version: ZarrFormat | None, zarr_format: ZarrFormat | None +) -> ZarrFormat | None: + """Handle the deprecated zarr_version kwarg and return zarr_format""" + if zarr_format is not None and zarr_version is not None and zarr_format != zarr_version: + raise ValueError( + f"zarr_format {zarr_format} does not match zarr_version {zarr_version}, please only set one" + ) + if zarr_version is not None: + warnings.warn( + "zarr_version is deprecated, use zarr_format", DeprecationWarning, stacklevel=2 + ) + return zarr_version + return zarr_format + + +async def consolidate_metadata( + store: StoreLike, + path: str | None = None, + zarr_format: ZarrFormat | None = None, +) -> AsyncGroup: + """ + Consolidate the metadata of all nodes in a hierarchy. + + Upon completion, the metadata of the root node in the Zarr hierarchy will be + updated to include all the metadata of child nodes. For Stores that do + not support consolidated metadata, this operation raises a ``TypeError``. + + Parameters + ---------- + store : StoreLike + The store-like object whose metadata you wish to consolidate. + path : str, optional + A path to a group in the store to consolidate at. Only children + below that group will be consolidated. + + By default, the root node is used so all the metadata in the + store is consolidated. + zarr_format : {2, 3, None}, optional + The zarr format of the hierarchy. By default the zarr format + is inferred. + + Returns + ------- + group: AsyncGroup + The group, with the ``consolidated_metadata`` field set to include + the metadata of each child node. If the Store doesn't support + consolidated metadata, this function raises a `TypeError`. + See ``Store.supports_consolidated_metadata``. + """ + store_path = await make_store_path(store, path=path) + + if not store_path.store.supports_consolidated_metadata: + store_name = type(store_path.store).__name__ + raise TypeError( + f"The Zarr Store in use ({store_name}) doesn't support consolidated metadata", + ) + + group = await AsyncGroup.open(store_path, zarr_format=zarr_format, use_consolidated=False) + group.store_path.store._check_writable() + + members_metadata = { + k: v.metadata + async for k, v in group.members(max_depth=None, use_consolidated_for_children=False) + } + # While consolidating, we want to be explicit about when child groups + # are empty by inserting an empty dict for consolidated_metadata.metadata + for k, v in members_metadata.items(): + if isinstance(v, GroupMetadata) and v.consolidated_metadata is None: + v = dataclasses.replace(v, consolidated_metadata=ConsolidatedMetadata(metadata={})) + members_metadata[k] = v + + if any(m.zarr_format == 3 for m in members_metadata.values()): + warnings.warn( + "Consolidated metadata is currently not part in the Zarr format 3 specification. It " + "may not be supported by other zarr implementations and may change in the future.", + category=UserWarning, + stacklevel=1, + ) + + ConsolidatedMetadata._flat_to_nested(members_metadata) + + consolidated_metadata = ConsolidatedMetadata(metadata=members_metadata) + metadata = dataclasses.replace(group.metadata, consolidated_metadata=consolidated_metadata) + group = dataclasses.replace( + group, + metadata=metadata, + ) + await group._save_metadata() + return group + + +async def copy(*args: Any, **kwargs: Any) -> tuple[int, int, int]: + raise NotImplementedError + + +async def copy_all(*args: Any, **kwargs: Any) -> tuple[int, int, int]: + raise NotImplementedError + + +async def copy_store(*args: Any, **kwargs: Any) -> tuple[int, int, int]: + raise NotImplementedError + + +async def load( + *, + store: StoreLike, + path: str | None = None, + zarr_format: ZarrFormat | None = None, + zarr_version: ZarrFormat | None = None, +) -> NDArrayLikeOrScalar | dict[str, NDArrayLikeOrScalar]: + """Load data from an array or group into memory. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system or name of zip file. + path : str or None, optional + The path within the store from which to load. + + Returns + ------- + out + If the path contains an array, out will be a numpy array. If the path contains + a group, out will be a dict-like object where keys are array names and values + are numpy arrays. + + See Also + -------- + save, savez + + Notes + ----- + If loading data from a group of arrays, data will not be immediately loaded into + memory. Rather, arrays will be loaded into memory as they are requested. + """ + zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) + + obj = await open(store=store, path=path, zarr_format=zarr_format) + if isinstance(obj, AsyncArray): + return await obj.getitem(slice(None)) + else: + raise NotImplementedError("loading groups not yet supported") + + +async def open( + *, + store: StoreLike | None = None, + mode: AccessModeLiteral | None = None, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + path: str | None = None, + storage_options: dict[str, Any] | None = None, + **kwargs: Any, # TODO: type kwargs as valid args to open_array +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup: + """Convenience function to open a group or array using file-mode-like semantics. + + Parameters + ---------- + store : Store or str, optional + Store or path to directory in file system or name of zip file. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Persistence mode: 'r' means read only (must exist); 'r+' means + read/write (must exist); 'a' means read/write (create if doesn't + exist); 'w' means create (overwrite if exists); 'w-' means create + (fail if exists). + If the store is read-only, the default is 'r'; otherwise, it is 'a'. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + path : str or None, optional + The path within the store to open. + storage_options : dict + If the store is backed by an fsspec-based implementation, then this dict will be passed to + the Store constructor for that implementation. Ignored otherwise. + **kwargs + Additional parameters are passed through to :func:`zarr.creation.open_array` or + :func:`zarr.hierarchy.open_group`. + + Returns + ------- + z : array or group + Return type depends on what exists in the given store. + """ + zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) + if mode is None: + if isinstance(store, (Store, StorePath)) and store.read_only: + mode = "r" + else: + mode = "a" + store_path = await make_store_path(store, mode=mode, path=path, storage_options=storage_options) + + # TODO: the mode check below seems wrong! + if "shape" not in kwargs and mode in {"a", "r", "r+", "w"}: + try: + metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) + # TODO: remove this cast when we fix typing for array metadata dicts + _metadata_dict = cast("ArrayMetadataDict", metadata_dict) + # for v2, the above would already have raised an exception if not an array + zarr_format = _metadata_dict["zarr_format"] + is_v3_array = zarr_format == 3 and _metadata_dict.get("node_type") == "array" + if is_v3_array or zarr_format == 2: + return AsyncArray(store_path=store_path, metadata=_metadata_dict) + except (AssertionError, FileNotFoundError, NodeTypeValidationError): + pass + return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs) + + try: + return await open_array(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs) + except (KeyError, NodeTypeValidationError): + # KeyError for a missing key + # NodeTypeValidationError for failing to parse node metadata as an array when it's + # actually a group + return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs) + + +async def open_consolidated( + *args: Any, use_consolidated: Literal[True] = True, **kwargs: Any +) -> AsyncGroup: + """ + Alias for :func:`open_group` with ``use_consolidated=True``. + """ + if use_consolidated is not True: + raise TypeError( + "'use_consolidated' must be 'True' in 'open_consolidated'. Use 'open' with " + "'use_consolidated=False' to bypass consolidated metadata." + ) + return await open_group(*args, use_consolidated=use_consolidated, **kwargs) + + +async def save( + store: StoreLike, + *args: NDArrayLike, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + path: str | None = None, + **kwargs: Any, # TODO: type kwargs as valid args to save +) -> None: + """Convenience function to save an array or group of arrays to the local file system. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system or name of zip file. + *args : ndarray + NumPy arrays with data to save. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + path : str or None, optional + The path within the group where the arrays will be saved. + **kwargs + NumPy arrays with data to save. + """ + zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) + + if len(args) == 0 and len(kwargs) == 0: + raise ValueError("at least one array must be provided") + if len(args) == 1 and len(kwargs) == 0: + await save_array(store, args[0], zarr_format=zarr_format, path=path) + else: + await save_group(store, *args, zarr_format=zarr_format, path=path, **kwargs) + + +async def save_array( + store: StoreLike, + arr: NDArrayLike, + *, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + path: str | None = None, + storage_options: dict[str, Any] | None = None, + **kwargs: Any, # TODO: type kwargs as valid args to create +) -> None: + """Convenience function to save a NumPy array to the local file system, following a + similar API to the NumPy save() function. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system or name of zip file. + arr : ndarray + NumPy array with data to save. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving (default is 3 if not specified). + path : str or None, optional + The path within the store where the array will be saved. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + **kwargs + Passed through to :func:`create`, e.g., compressor. + """ + zarr_format = ( + _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) + or _default_zarr_format() + ) + if not isinstance(arr, NDArrayLike): + raise TypeError("arr argument must be numpy or other NDArrayLike array") + + mode = kwargs.pop("mode", "a") + store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + if np.isscalar(arr): + arr = np.array(arr) + shape = arr.shape + chunks = getattr(arr, "chunks", None) # for array-likes with chunks attribute + overwrite = kwargs.pop("overwrite", None) or _infer_overwrite(mode) + zarr_dtype = get_data_type_from_native_dtype(arr.dtype) + new = await AsyncArray._create( + store_path, + zarr_format=zarr_format, + shape=shape, + dtype=zarr_dtype, + chunks=chunks, + overwrite=overwrite, + **kwargs, + ) + await new.setitem(slice(None), arr) + + +async def save_group( + store: StoreLike, + *args: NDArrayLike, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + path: str | None = None, + storage_options: dict[str, Any] | None = None, + **kwargs: NDArrayLike, +) -> None: + """Convenience function to save several NumPy arrays to the local file system, following a + similar API to the NumPy savez()/savez_compressed() functions. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system or name of zip file. + *args : ndarray + NumPy arrays with data to save. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + path : str or None, optional + Path within the store where the group will be saved. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + **kwargs + NumPy arrays with data to save. + """ + + store_path = await make_store_path(store, path=path, mode="w", storage_options=storage_options) + + zarr_format = ( + _handle_zarr_version_or_format( + zarr_version=zarr_version, + zarr_format=zarr_format, + ) + or _default_zarr_format() + ) + + for arg in args: + if not isinstance(arg, NDArrayLike): + raise TypeError( + "All arguments must be numpy or other NDArrayLike arrays (except store, path, storage_options, and zarr_format)" + ) + for k, v in kwargs.items(): + if not isinstance(v, NDArrayLike): + raise TypeError(f"Keyword argument '{k}' must be a numpy or other NDArrayLike array") + + if len(args) == 0 and len(kwargs) == 0: + raise ValueError("at least one array must be provided") + aws = [] + for i, arr in enumerate(args): + aws.append( + save_array( + store_path, + arr, + zarr_format=zarr_format, + path=f"arr_{i}", + storage_options=storage_options, + ) + ) + for k, arr in kwargs.items(): + aws.append(save_array(store_path, arr, zarr_format=zarr_format, path=k)) + await asyncio.gather(*aws) + + +@deprecated("Use AsyncGroup.tree instead.") +async def tree(grp: AsyncGroup, expand: bool | None = None, level: int | None = None) -> Any: + """Provide a rich display of the hierarchy. + + .. deprecated:: 3.0.0 + `zarr.tree()` is deprecated and will be removed in a future release. + Use `group.tree()` instead. + + Parameters + ---------- + grp : Group + Zarr or h5py group. + expand : bool, optional + Only relevant for HTML representation. If True, tree will be fully expanded. + level : int, optional + Maximum depth to descend into hierarchy. + + Returns + ------- + TreeRepr + A pretty-printable object displaying the hierarchy. + """ + return await grp.tree(expand=expand, level=level) + + +async def array( + data: npt.ArrayLike | Array, **kwargs: Any +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array filled with `data`. + + Parameters + ---------- + data : array_like + The data to fill the array with. + **kwargs + Passed through to :func:`create`. + + Returns + ------- + array : array + The new array. + """ + + if isinstance(data, Array): + return await from_array(data=data, **kwargs) + + # ensure data is array-like + if not hasattr(data, "shape") or not hasattr(data, "dtype"): + data = np.asanyarray(data) + + # setup dtype + kw_dtype = kwargs.get("dtype") + if kw_dtype is None and hasattr(data, "dtype"): + kwargs["dtype"] = data.dtype + else: + kwargs["dtype"] = kw_dtype + + # setup shape and chunks + data_shape, data_chunks = _get_shape_chunks(data) + kwargs["shape"] = data_shape + kw_chunks = kwargs.get("chunks") + if kw_chunks is None: + kwargs["chunks"] = data_chunks + else: + kwargs["chunks"] = kw_chunks + + read_only = kwargs.pop("read_only", False) + if read_only: + raise ValueError("read_only=True is no longer supported when creating new arrays") + + # instantiate array + z = await create(**kwargs) + + # fill with data + await z.setitem(Ellipsis, data) + + return z + + +async def group( + *, # Note: this is a change from v2 + store: StoreLike | None = None, + overwrite: bool = False, + chunk_store: StoreLike | None = None, # not used + cache_attrs: bool | None = None, # not used, default changed + synchronizer: Any | None = None, # not used + path: str | None = None, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, # not used + attributes: dict[str, JSON] | None = None, + storage_options: dict[str, Any] | None = None, +) -> AsyncGroup: + """Create a group. + + Parameters + ---------- + store : Store or str, optional + Store or path to directory in file system. + overwrite : bool, optional + If True, delete any pre-existing data in `store` at `path` before + creating the group. + chunk_store : Store, optional + Separate storage for chunks. If not provided, `store` will be used + for storage of both chunks and metadata. + cache_attrs : bool, optional + If True (default), user attributes will be cached for attribute read + operations. If False, user attributes are reloaded from the store prior + to all attribute read operations. + synchronizer : object, optional + Array synchronizer. + path : str, optional + Group path within store. + meta_array : array-like, optional + An array instance to use for determining arrays to create and return + to users. Use `numpy.empty(())` by default. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + + Returns + ------- + g : group + The new group. + """ + + zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) + + mode: AccessModeLiteral + if overwrite: + mode = "w" + else: + mode = "r+" + store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + + if chunk_store is not None: + warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2) + if cache_attrs is not None: + warnings.warn("cache_attrs is not yet implemented", RuntimeWarning, stacklevel=2) + if synchronizer is not None: + warnings.warn("synchronizer is not yet implemented", RuntimeWarning, stacklevel=2) + if meta_array is not None: + warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2) + + if attributes is None: + attributes = {} + + try: + return await AsyncGroup.open(store=store_path, zarr_format=zarr_format) + except (KeyError, FileNotFoundError): + _zarr_format = zarr_format or _default_zarr_format() + return await AsyncGroup.from_store( + store=store_path, + zarr_format=_zarr_format, + overwrite=overwrite, + attributes=attributes, + ) + + +async def create_group( + *, + store: StoreLike, + path: str | None = None, + overwrite: bool = False, + zarr_format: ZarrFormat | None = None, + attributes: dict[str, Any] | None = None, + storage_options: dict[str, Any] | None = None, +) -> AsyncGroup: + """Create a group. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system. + path : str, optional + Group path within store. + overwrite : bool, optional + If True, pre-existing data at ``path`` will be deleted before + creating the group. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + If no ``zarr_format`` is provided, the default format will be used. + This default can be changed by modifying the value of ``default_zarr_format`` + in :mod:`zarr.core.config`. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + + Returns + ------- + AsyncGroup + The new group. + """ + + if zarr_format is None: + zarr_format = _default_zarr_format() + + mode: Literal["a"] = "a" + + store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + + return await AsyncGroup.from_store( + store=store_path, + zarr_format=zarr_format, + overwrite=overwrite, + attributes=attributes, + ) + + +async def open_group( + store: StoreLike | None = None, + *, # Note: this is a change from v2 + mode: AccessModeLiteral = "a", + cache_attrs: bool | None = None, # not used, default changed + synchronizer: Any = None, # not used + path: str | None = None, + chunk_store: StoreLike | None = None, # not used + storage_options: dict[str, Any] | None = None, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, # not used + attributes: dict[str, JSON] | None = None, + use_consolidated: bool | str | None = None, +) -> AsyncGroup: + """Open a group using file-mode-like semantics. + + Parameters + ---------- + store : Store, str, or mapping, optional + Store or path to directory in file system or name of zip file. + + Strings are interpreted as paths on the local file system + and used as the ``root`` argument to :class:`zarr.storage.LocalStore`. + + Dictionaries are used as the ``store_dict`` argument in + :class:`zarr.storage.MemoryStore``. + + By default (``store=None``) a new :class:`zarr.storage.MemoryStore` + is created. + + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Persistence mode: 'r' means read only (must exist); 'r+' means + read/write (must exist); 'a' means read/write (create if doesn't + exist); 'w' means create (overwrite if exists); 'w-' means create + (fail if exists). + cache_attrs : bool, optional + If True (default), user attributes will be cached for attribute read + operations. If False, user attributes are reloaded from the store prior + to all attribute read operations. + synchronizer : object, optional + Array synchronizer. + path : str, optional + Group path within store. + chunk_store : Store or str, optional + Store or path to directory in file system or name of zip file. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + meta_array : array-like, optional + An array instance to use for determining arrays to create and return + to users. Use `numpy.empty(())` by default. + attributes : dict + A dictionary of JSON-serializable values with user-defined attributes. + use_consolidated : bool or str, default None + Whether to use consolidated metadata. + + By default, consolidated metadata is used if it's present in the + store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file + for Zarr format 2). + + To explicitly require consolidated metadata, set ``use_consolidated=True``, + which will raise an exception if consolidated metadata is not found. + + To explicitly *not* use consolidated metadata, set ``use_consolidated=False``, + which will fall back to using the regular, non consolidated metadata. + + Zarr format 2 allowed configuring the key storing the consolidated metadata + (``.zmetadata`` by default). Specify the custom key as ``use_consolidated`` + to load consolidated metadata from a non-default key. + + Returns + ------- + g : group + The new group. + """ + + zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) + + if cache_attrs is not None: + warnings.warn("cache_attrs is not yet implemented", RuntimeWarning, stacklevel=2) + if synchronizer is not None: + warnings.warn("synchronizer is not yet implemented", RuntimeWarning, stacklevel=2) + if meta_array is not None: + warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2) + if chunk_store is not None: + warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2) + + store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path) + if attributes is None: + attributes = {} + + try: + if mode in _READ_MODES: + return await AsyncGroup.open( + store_path, zarr_format=zarr_format, use_consolidated=use_consolidated + ) + except (KeyError, FileNotFoundError): + pass + if mode in _CREATE_MODES: + overwrite = _infer_overwrite(mode) + _zarr_format = zarr_format or _default_zarr_format() + return await AsyncGroup.from_store( + store_path, + zarr_format=_zarr_format, + overwrite=overwrite, + attributes=attributes, + ) + raise GroupNotFoundError(store, store_path.path) + + +async def create( + shape: ChunkCoords | int, + *, # Note: this is a change from v2 + chunks: ChunkCoords | int | None = None, # TODO: v2 allowed chunks=True + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + fill_value: Any | None = 0, # TODO: need type + order: MemoryOrder | None = None, + store: str | StoreLike | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + path: PathLike | None = None, + chunk_store: StoreLike | None = None, + filters: list[dict[str, JSON]] | None = None, # TODO: type has changed + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, # TODO: type has changed + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, # TODO: need type + attributes: dict[str, JSON] | None = None, + # v3 only + chunk_shape: ChunkCoords | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + storage_options: dict[str, Any] | None = None, + config: ArrayConfigLike | None = None, + **kwargs: Any, +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array. + + Parameters + ---------- + shape : int or tuple of ints + Array shape. + chunks : int or tuple of ints, optional + The shape of the array's chunks. + Zarr format 2 only. Zarr format 3 arrays should use `chunk_shape` instead. + If not specified, default values are guessed based on the shape and dtype. + dtype : str or dtype, optional + NumPy dtype. + chunk_shape : int or tuple of ints, optional + The shape of the Array's chunks (default is None). + Zarr format 3 only. Zarr format 2 arrays should use `chunks` instead. + chunk_key_encoding : ChunkKeyEncoding, optional + A specification of how the chunk keys are represented in storage. + Zarr format 3 only. Zarr format 2 arrays should use `dimension_separator` instead. + Default is ``("default", "/")``. + codecs : Sequence of Codecs or dicts, optional + An iterable of Codec or dict serializations of Codecs. The elements of + this collection specify the transformation from array values to stored bytes. + Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead. + + If no codecs are provided, default codecs will be used: + + - For numeric arrays, the default is ``BytesCodec`` and ``ZstdCodec``. + - For Unicode strings, the default is ``VLenUTF8Codec`` and ``ZstdCodec``. + - For bytes or objects, the default is ``VLenBytesCodec`` and ``ZstdCodec``. + + These defaults can be changed by modifying the value of ``array.v3_default_filters``, + ``array.v3_default_serializer`` and ``array.v3_default_compressors`` in :mod:`zarr.core.config`. + compressor : Codec, optional + Primary compressor to compress chunk data. + Zarr format 2 only. Zarr format 3 arrays should use ``codecs`` instead. + + If neither ``compressor`` nor ``filters`` are provided, a default compressor will be used: + + - For numeric arrays, the default is ``ZstdCodec``. + - For Unicode strings, the default is ``VLenUTF8Codec``. + - For bytes or objects, the default is ``VLenBytesCodec``. + + These defaults can be changed by modifying the value of ``array.v2_default_compressor`` in :mod:`zarr.core.config`. + fill_value : object + Default value to use for uninitialized portions of the array. + order : {'C', 'F'}, optional + Deprecated in favor of the ``config`` keyword argument. + Pass ``{'order': }`` to ``create`` instead of using this parameter. + Memory layout to be used within each chunk. + If not specified, the ``array.order`` parameter in the global config will be used. + store : Store or str + Store or path to directory in file system or name of zip file. + synchronizer : object, optional + Array synchronizer. + overwrite : bool, optional + If True, delete all pre-existing data in `store` at `path` before + creating the array. + path : str, optional + Path under which array is stored. + chunk_store : MutableMapping, optional + Separate storage for chunks. If not provided, `store` will be used + for storage of both chunks and metadata. + filters : sequence of Codecs, optional + Sequence of filters to use to encode chunk data prior to compression. + Zarr format 2 only. If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v2_default_filters`` in :mod:`zarr.core.config`. + cache_metadata : bool, optional + If True, array configuration metadata will be cached for the + lifetime of the object. If False, array metadata will be reloaded + prior to all data access and modification operations (may incur + overhead depending on storage and data access pattern). + cache_attrs : bool, optional + If True (default), user attributes will be cached for attribute read + operations. If False, user attributes are reloaded from the store prior + to all attribute read operations. + read_only : bool, optional + True if array should be protected against modification. + object_codec : Codec, optional + A codec to encode object arrays, only needed if dtype=object. + dimension_separator : {'.', '/'}, optional + Separator placed between the dimensions of a chunk. + Zarr format 2 only. Zarr format 3 arrays should use ``chunk_key_encoding`` instead. + Default is ".". + write_empty_chunks : bool, optional + Deprecated in favor of the ``config`` keyword argument. + Pass ``{'write_empty_chunks': }`` to ``create`` instead of using this parameter. + If True, all chunks will be stored regardless of their + contents. If False, each chunk is compared to the array's fill value + prior to storing. If a chunk is uniformly equal to the fill value, then + that chunk is not be stored, and the store entry for that chunk's key + is deleted. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + Default is 3. + meta_array : array-like, optional + An array instance to use for determining arrays to create and return + to users. Use `numpy.empty(())` by default. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + config : ArrayConfig or ArrayConfigLike, optional + Runtime configuration of the array. If provided, will override the + default values from `zarr.config.array`. + + Returns + ------- + z : array + The array. + """ + zarr_format = ( + _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) + or _default_zarr_format() + ) + zdtype = parse_data_type(dtype, zarr_format=zarr_format) + if zarr_format == 2: + default_filters, default_compressor = _get_default_chunk_encoding_v2(zdtype) + if not filters: + filters = default_filters # type: ignore[assignment] + if compressor == "auto": + compressor = default_compressor + + if synchronizer is not None: + warnings.warn("synchronizer is not yet implemented", RuntimeWarning, stacklevel=2) + if chunk_store is not None: + warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2) + if cache_metadata is not None: + warnings.warn("cache_metadata is not yet implemented", RuntimeWarning, stacklevel=2) + if cache_attrs is not None: + warnings.warn("cache_attrs is not yet implemented", RuntimeWarning, stacklevel=2) + if object_codec is not None: + warnings.warn("object_codec is not yet implemented", RuntimeWarning, stacklevel=2) + if read_only is not None: + warnings.warn("read_only is not yet implemented", RuntimeWarning, stacklevel=2) + if order is not None: + _warn_order_kwarg() + if write_empty_chunks is not None: + _warn_write_empty_chunks_kwarg() + + if meta_array is not None: + warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2) + + mode = kwargs.pop("mode", None) + if mode is None: + mode = "a" + store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + + config_dict: ArrayConfigParams = {} + + if write_empty_chunks is not None: + if config is not None: + msg = ( + "Both write_empty_chunks and config keyword arguments are set. " + "This is redundant. When both are set, write_empty_chunks will be ignored and " + "config will be used." + ) + warnings.warn(UserWarning(msg), stacklevel=1) + config_dict["write_empty_chunks"] = write_empty_chunks + if order is not None and config is not None: + msg = ( + "Both order and config keyword arguments are set. " + "This is redundant. When both are set, order will be ignored and " + "config will be used." + ) + warnings.warn(UserWarning(msg), stacklevel=1) + + config_parsed = ArrayConfig.from_dict(config_dict) + + return await AsyncArray._create( + store_path, + shape=shape, + chunks=chunks, + dtype=zdtype, + compressor=compressor, + fill_value=fill_value, + overwrite=overwrite, + filters=filters, + dimension_separator=dimension_separator, + order=order, + zarr_format=zarr_format, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + attributes=attributes, + config=config_parsed, + **kwargs, + ) + + +async def empty( + shape: ChunkCoords, **kwargs: Any +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an empty array with the specified shape. The contents will be filled with the + array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + + return await create(shape=shape, fill_value=None, **kwargs) + + +async def empty_like( + a: ArrayLike, **kwargs: Any +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an empty array like `a`. The contents will be filled with the + array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + a : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + like_kwargs = _like_args(a, kwargs) + return await empty(**like_kwargs) + + +# TODO: add type annotations for fill_value and kwargs +async def full( + shape: ChunkCoords, fill_value: Any, **kwargs: Any +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array, with `fill_value` being used as the default value for + uninitialized portions of the array. + + Parameters + ---------- + shape : int or tuple of int + Shape of the empty array. + fill_value : scalar + Fill value. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return await create(shape=shape, fill_value=fill_value, **kwargs) + + +# TODO: add type annotations for kwargs +async def full_like( + a: ArrayLike, **kwargs: Any +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create a filled array like `a`. + + Parameters + ---------- + a : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + like_kwargs = _like_args(a, kwargs) + if isinstance(a, AsyncArray): + like_kwargs.setdefault("fill_value", a.metadata.fill_value) + return await full(**like_kwargs) + + +async def ones( + shape: ChunkCoords, **kwargs: Any +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array, with one being used as the default value for + uninitialized portions of the array. + + Parameters + ---------- + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return await create(shape=shape, fill_value=1, **kwargs) + + +async def ones_like( + a: ArrayLike, **kwargs: Any +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array of ones like `a`. + + Parameters + ---------- + a : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + like_kwargs = _like_args(a, kwargs) + return await ones(**like_kwargs) + + +async def open_array( + *, # note: this is a change from v2 + store: StoreLike | None = None, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + path: PathLike = "", + storage_options: dict[str, Any] | None = None, + **kwargs: Any, # TODO: type kwargs as valid args to save +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Open an array using file-mode-like semantics. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system or name of zip file. + zarr_version : {2, 3, None}, optional + The zarr format to use when saving. Deprecated in favor of zarr_format. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + path : str, optional + Path in store to array. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + **kwargs + Any keyword arguments to pass to :func:`create`. + + Returns + ------- + AsyncArray + The opened array. + """ + + mode = kwargs.pop("mode", None) + store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) + + zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) + + if "order" in kwargs: + _warn_order_kwarg() + if "write_empty_chunks" in kwargs: + _warn_write_empty_chunks_kwarg() + + try: + return await AsyncArray.open(store_path, zarr_format=zarr_format) + except FileNotFoundError: + if not store_path.read_only and mode in _CREATE_MODES: + overwrite = _infer_overwrite(mode) + _zarr_format = zarr_format or _default_zarr_format() + return await create( + store=store_path, + zarr_format=_zarr_format, + overwrite=overwrite, + **kwargs, + ) + raise + + +async def open_like( + a: ArrayLike, path: str, **kwargs: Any +) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]: + """Open a persistent array like `a`. + + Parameters + ---------- + a : Array + The shape and data-type of a define these same attributes of the returned array. + path : str + The path to the new array. + **kwargs + Any keyword arguments to pass to the array constructor. + + Returns + ------- + AsyncArray + The opened array. + """ + like_kwargs = _like_args(a, kwargs) + if isinstance(a, (AsyncArray | Array)): + kwargs.setdefault("fill_value", a.metadata.fill_value) + return await open_array(path=path, **like_kwargs) + + +async def zeros( + shape: ChunkCoords, **kwargs: Any +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array, with zero being used as the default value for + uninitialized portions of the array. + + Parameters + ---------- + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return await create(shape=shape, fill_value=0, **kwargs) + + +async def zeros_like( + a: ArrayLike, **kwargs: Any +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array of zeros like `a`. + + Parameters + ---------- + a : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + like_kwargs = _like_args(a, kwargs) + return await zeros(**like_kwargs) diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py new file mode 100644 index 0000000000..f2dc8757d6 --- /dev/null +++ b/src/zarr/api/synchronous.py @@ -0,0 +1,1365 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Literal + +from typing_extensions import deprecated + +import zarr.api.asynchronous as async_api +import zarr.core.array +from zarr._compat import _deprecate_positional_args +from zarr.core.array import Array, AsyncArray, CompressorLike +from zarr.core.group import Group +from zarr.core.sync import sync +from zarr.core.sync_group import create_hierarchy + +if TYPE_CHECKING: + from collections.abc import Iterable + + import numpy as np + import numpy.typing as npt + + from zarr.abc.codec import Codec + from zarr.api.asynchronous import ArrayLike, PathLike + from zarr.core.array import ( + CompressorsLike, + FiltersLike, + SerializerLike, + ShardsLike, + ) + from zarr.core.array_spec import ArrayConfigLike + from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar + from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike + from zarr.core.common import ( + JSON, + AccessModeLiteral, + ChunkCoords, + DimensionNames, + MemoryOrder, + ShapeLike, + ZarrFormat, + ) + from zarr.core.dtype import ZDTypeLike + from zarr.storage import StoreLike + +__all__ = [ + "array", + "consolidate_metadata", + "copy", + "copy_all", + "copy_store", + "create", + "create_array", + "create_hierarchy", + "empty", + "empty_like", + "from_array", + "full", + "full_like", + "group", + "load", + "ones", + "ones_like", + "open", + "open_array", + "open_consolidated", + "open_group", + "open_like", + "save", + "save_array", + "save_group", + "tree", + "zeros", + "zeros_like", +] + + +def consolidate_metadata( + store: StoreLike, + path: str | None = None, + zarr_format: ZarrFormat | None = None, +) -> Group: + """ + Consolidate the metadata of all nodes in a hierarchy. + + Upon completion, the metadata of the root node in the Zarr hierarchy will be + updated to include all the metadata of child nodes. For Stores that do + not use consolidated metadata, this operation raises a `TypeError`. + + Parameters + ---------- + store : StoreLike + The store-like object whose metadata you wish to consolidate. + path : str, optional + A path to a group in the store to consolidate at. Only children + below that group will be consolidated. + + By default, the root node is used so all the metadata in the + store is consolidated. + zarr_format : {2, 3, None}, optional + The zarr format of the hierarchy. By default the zarr format + is inferred. + + Returns + ------- + group: Group + The group, with the ``consolidated_metadata`` field set to include + the metadata of each child node. If the Store doesn't support + consolidated metadata, this function raises a `TypeError`. + See ``Store.supports_consolidated_metadata``. + + """ + return Group(sync(async_api.consolidate_metadata(store, path=path, zarr_format=zarr_format))) + + +def copy(*args: Any, **kwargs: Any) -> tuple[int, int, int]: + return sync(async_api.copy(*args, **kwargs)) + + +def copy_all(*args: Any, **kwargs: Any) -> tuple[int, int, int]: + return sync(async_api.copy_all(*args, **kwargs)) + + +def copy_store(*args: Any, **kwargs: Any) -> tuple[int, int, int]: + return sync(async_api.copy_store(*args, **kwargs)) + + +def load( + store: StoreLike, + path: str | None = None, + zarr_format: ZarrFormat | None = None, + zarr_version: ZarrFormat | None = None, +) -> NDArrayLikeOrScalar | dict[str, NDArrayLikeOrScalar]: + """Load data from an array or group into memory. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system or name of zip file. + path : str or None, optional + The path within the store from which to load. + + Returns + ------- + out + If the path contains an array, out will be a numpy array. If the path contains + a group, out will be a dict-like object where keys are array names and values + are numpy arrays. + + See Also + -------- + save, savez + + Notes + ----- + If loading data from a group of arrays, data will not be immediately loaded into + memory. Rather, arrays will be loaded into memory as they are requested. + """ + return sync( + async_api.load(store=store, zarr_version=zarr_version, zarr_format=zarr_format, path=path) + ) + + +@_deprecate_positional_args +def open( + store: StoreLike | None = None, + *, + mode: AccessModeLiteral | None = None, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + path: str | None = None, + storage_options: dict[str, Any] | None = None, + **kwargs: Any, # TODO: type kwargs as valid args to async_api.open +) -> Array | Group: + """Open a group or array using file-mode-like semantics. + + Parameters + ---------- + store : Store or str, optional + Store or path to directory in file system or name of zip file. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Persistence mode: 'r' means read only (must exist); 'r+' means + read/write (must exist); 'a' means read/write (create if doesn't + exist); 'w' means create (overwrite if exists); 'w-' means create + (fail if exists). + If the store is read-only, the default is 'r'; otherwise, it is 'a'. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + path : str or None, optional + The path within the store to open. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + **kwargs + Additional parameters are passed through to :func:`zarr.api.asynchronous.open_array` or + :func:`zarr.api.asynchronous.open_group`. + + Returns + ------- + z : array or group + Return type depends on what exists in the given store. + """ + obj = sync( + async_api.open( + store=store, + mode=mode, + zarr_version=zarr_version, + zarr_format=zarr_format, + path=path, + storage_options=storage_options, + **kwargs, + ) + ) + if isinstance(obj, AsyncArray): + return Array(obj) + else: + return Group(obj) + + +def open_consolidated(*args: Any, use_consolidated: Literal[True] = True, **kwargs: Any) -> Group: + """ + Alias for :func:`open_group` with ``use_consolidated=True``. + """ + return Group( + sync(async_api.open_consolidated(*args, use_consolidated=use_consolidated, **kwargs)) + ) + + +def save( + store: StoreLike, + *args: NDArrayLike, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + path: str | None = None, + **kwargs: Any, # TODO: type kwargs as valid args to async_api.save +) -> None: + """Save an array or group of arrays to the local file system. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system or name of zip file. + *args : ndarray + NumPy arrays with data to save. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + path : str or None, optional + The path within the group where the arrays will be saved. + **kwargs + NumPy arrays with data to save. + """ + return sync( + async_api.save( + store, *args, zarr_version=zarr_version, zarr_format=zarr_format, path=path, **kwargs + ) + ) + + +@_deprecate_positional_args +def save_array( + store: StoreLike, + arr: NDArrayLike, + *, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + path: str | None = None, + storage_options: dict[str, Any] | None = None, + **kwargs: Any, # TODO: type kwargs as valid args to async_api.save_array +) -> None: + """Save a NumPy array to the local file system. + + Follows a similar API to the NumPy save() function. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system or name of zip file. + arr : ndarray + NumPy array with data to save. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + path : str or None, optional + The path within the store where the array will be saved. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + **kwargs + Passed through to :func:`create`, e.g., compressor. + """ + return sync( + async_api.save_array( + store=store, + arr=arr, + zarr_version=zarr_version, + zarr_format=zarr_format, + path=path, + storage_options=storage_options, + **kwargs, + ) + ) + + +def save_group( + store: StoreLike, + *args: NDArrayLike, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + path: str | None = None, + storage_options: dict[str, Any] | None = None, + **kwargs: NDArrayLike, +) -> None: + """Save several NumPy arrays to the local file system. + + Follows a similar API to the NumPy savez()/savez_compressed() functions. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system or name of zip file. + *args : ndarray + NumPy arrays with data to save. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + path : str or None, optional + Path within the store where the group will be saved. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + **kwargs + NumPy arrays with data to save. + """ + + return sync( + async_api.save_group( + store, + *args, + zarr_version=zarr_version, + zarr_format=zarr_format, + path=path, + storage_options=storage_options, + **kwargs, + ) + ) + + +@deprecated("Use Group.tree instead.") +def tree(grp: Group, expand: bool | None = None, level: int | None = None) -> Any: + """Provide a rich display of the hierarchy. + + .. deprecated:: 3.0.0 + `zarr.tree()` is deprecated and will be removed in a future release. + Use `group.tree()` instead. + + Parameters + ---------- + grp : Group + Zarr or h5py group. + expand : bool, optional + Only relevant for HTML representation. If True, tree will be fully expanded. + level : int, optional + Maximum depth to descend into hierarchy. + + Returns + ------- + TreeRepr + A pretty-printable object displaying the hierarchy. + """ + return sync(async_api.tree(grp._async_group, expand=expand, level=level)) + + +# TODO: add type annotations for kwargs +def array(data: npt.ArrayLike | Array, **kwargs: Any) -> Array: + """Create an array filled with `data`. + + Parameters + ---------- + data : array_like + The data to fill the array with. + **kwargs + Passed through to :func:`create`. + + Returns + ------- + array : Array + The new array. + """ + + return Array(sync(async_api.array(data=data, **kwargs))) + + +@_deprecate_positional_args +def group( + store: StoreLike | None = None, + *, + overwrite: bool = False, + chunk_store: StoreLike | None = None, # not used + cache_attrs: bool | None = None, # not used, default changed + synchronizer: Any | None = None, # not used + path: str | None = None, + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, # not used + attributes: dict[str, JSON] | None = None, + storage_options: dict[str, Any] | None = None, +) -> Group: + """Create a group. + + Parameters + ---------- + store : Store or str, optional + Store or path to directory in file system. + overwrite : bool, optional + If True, delete any pre-existing data in `store` at `path` before + creating the group. + chunk_store : Store, optional + Separate storage for chunks. If not provided, `store` will be used + for storage of both chunks and metadata. + cache_attrs : bool, optional + If True (default), user attributes will be cached for attribute read + operations. If False, user attributes are reloaded from the store prior + to all attribute read operations. + synchronizer : object, optional + Array synchronizer. + path : str, optional + Group path within store. + meta_array : array-like, optional + An array instance to use for determining arrays to create and return + to users. Use `numpy.empty(())` by default. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + + Returns + ------- + g : Group + The new group. + """ + return Group( + sync( + async_api.group( + store=store, + overwrite=overwrite, + chunk_store=chunk_store, + cache_attrs=cache_attrs, + synchronizer=synchronizer, + path=path, + zarr_version=zarr_version, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + storage_options=storage_options, + ) + ) + ) + + +@_deprecate_positional_args +def open_group( + store: StoreLike | None = None, + *, + mode: AccessModeLiteral = "a", + cache_attrs: bool | None = None, # default changed, not used in async api + synchronizer: Any = None, # not used in async api + path: str | None = None, + chunk_store: StoreLike | None = None, # not used in async api + storage_options: dict[str, Any] | None = None, # not used in async api + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, # not used in async api + attributes: dict[str, JSON] | None = None, + use_consolidated: bool | str | None = None, +) -> Group: + """Open a group using file-mode-like semantics. + + Parameters + ---------- + store : Store, str, or mapping, optional + Store or path to directory in file system or name of zip file. + + Strings are interpreted as paths on the local file system + and used as the ``root`` argument to :class:`zarr.storage.LocalStore`. + + Dictionaries are used as the ``store_dict`` argument in + :class:`zarr.storage.MemoryStore``. + + By default (``store=None``) a new :class:`zarr.storage.MemoryStore` + is created. + + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Persistence mode: 'r' means read only (must exist); 'r+' means + read/write (must exist); 'a' means read/write (create if doesn't + exist); 'w' means create (overwrite if exists); 'w-' means create + (fail if exists). + cache_attrs : bool, optional + If True (default), user attributes will be cached for attribute read + operations. If False, user attributes are reloaded from the store prior + to all attribute read operations. + synchronizer : object, optional + Array synchronizer. + path : str, optional + Group path within store. + chunk_store : Store or str, optional + Store or path to directory in file system or name of zip file. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + meta_array : array-like, optional + An array instance to use for determining arrays to create and return + to users. Use `numpy.empty(())` by default. + attributes : dict + A dictionary of JSON-serializable values with user-defined attributes. + use_consolidated : bool or str, default None + Whether to use consolidated metadata. + + By default, consolidated metadata is used if it's present in the + store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file + for Zarr format 2). + + To explicitly require consolidated metadata, set ``use_consolidated=True``, + which will raise an exception if consolidated metadata is not found. + + To explicitly *not* use consolidated metadata, set ``use_consolidated=False``, + which will fall back to using the regular, non consolidated metadata. + + Zarr format 2 allows configuring the key storing the consolidated metadata + (``.zmetadata`` by default). Specify the custom key as ``use_consolidated`` + to load consolidated metadata from a non-default key. + + Returns + ------- + g : Group + The new group. + """ + return Group( + sync( + async_api.open_group( + store=store, + mode=mode, + cache_attrs=cache_attrs, + synchronizer=synchronizer, + path=path, + chunk_store=chunk_store, + storage_options=storage_options, + zarr_version=zarr_version, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + use_consolidated=use_consolidated, + ) + ) + ) + + +def create_group( + store: StoreLike, + *, + path: str | None = None, + zarr_format: ZarrFormat | None = None, + overwrite: bool = False, + attributes: dict[str, Any] | None = None, + storage_options: dict[str, Any] | None = None, +) -> Group: + """Create a group. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system. + path : str, optional + Group path within store. + overwrite : bool, optional + If True, pre-existing data at ``path`` will be deleted before + creating the group. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + If no ``zarr_format`` is provided, the default format will be used. + This default can be changed by modifying the value of ``default_zarr_format`` + in :mod:`zarr.core.config`. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + + Returns + ------- + Group + The new group. + """ + return Group( + sync( + async_api.create_group( + store=store, + path=path, + overwrite=overwrite, + storage_options=storage_options, + zarr_format=zarr_format, + attributes=attributes, + ) + ) + ) + + +# TODO: add type annotations for kwargs +def create( + shape: ChunkCoords | int, + *, # Note: this is a change from v2 + chunks: ChunkCoords | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + fill_value: Any | None = None, # TODO: need type + order: MemoryOrder | None = None, + store: str | StoreLike | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + path: PathLike | None = None, + chunk_store: StoreLike | None = None, + filters: list[dict[str, JSON]] | None = None, # TODO: type has changed + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, # TODO: type has changed + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, # TODO: default has changed + zarr_version: ZarrFormat | None = None, # deprecated + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, # TODO: need type + attributes: dict[str, JSON] | None = None, + # v3 only + chunk_shape: ChunkCoords | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + storage_options: dict[str, Any] | None = None, + config: ArrayConfigLike | None = None, + **kwargs: Any, +) -> Array: + """Create an array. + + Parameters + ---------- + shape : int or tuple of ints + Array shape. + chunks : int or tuple of ints, optional + Chunk shape. If True, will be guessed from `shape` and `dtype`. If + False, will be set to `shape`, i.e., single chunk for the whole array. + If an int, the chunk size in each dimension will be given by the value + of `chunks`. Default is True. + dtype : str or dtype, optional + NumPy dtype. + compressor : Codec, optional + Primary compressor. + fill_value : object + Default value to use for uninitialized portions of the array. + order : {'C', 'F'}, optional + Deprecated in favor of the ``config`` keyword argument. + Pass ``{'order': }`` to ``create`` instead of using this parameter. + Memory layout to be used within each chunk. + If not specified, the ``array.order`` parameter in the global config will be used. + store : Store or str + Store or path to directory in file system or name of zip file. + synchronizer : object, optional + Array synchronizer. + overwrite : bool, optional + If True, delete all pre-existing data in `store` at `path` before + creating the array. + path : str, optional + Path under which array is stored. + chunk_store : MutableMapping, optional + Separate storage for chunks. If not provided, `store` will be used + for storage of both chunks and metadata. + filters : sequence of Codecs, optional + Sequence of filters to use to encode chunk data prior to compression. + cache_metadata : bool, optional + If True, array configuration metadata will be cached for the + lifetime of the object. If False, array metadata will be reloaded + prior to all data access and modification operations (may incur + overhead depending on storage and data access pattern). + cache_attrs : bool, optional + If True (default), user attributes will be cached for attribute read + operations. If False, user attributes are reloaded from the store prior + to all attribute read operations. + read_only : bool, optional + True if array should be protected against modification. + object_codec : Codec, optional + A codec to encode object arrays, only needed if dtype=object. + dimension_separator : {'.', '/'}, optional + Separator placed between the dimensions of a chunk. + write_empty_chunks : bool, optional + Deprecated in favor of the ``config`` keyword argument. + Pass ``{'write_empty_chunks': }`` to ``create`` instead of using this parameter. + If True, all chunks will be stored regardless of their + contents. If False, each chunk is compared to the array's fill value + prior to storing. If a chunk is uniformly equal to the fill value, then + that chunk is not be stored, and the store entry for that chunk's key + is deleted. + zarr_format : {2, 3, None}, optional + The zarr format to use when saving. + meta_array : array-like, optional + An array instance to use for determining arrays to create and return + to users. Use `numpy.empty(())` by default. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + config : ArrayConfigLike, optional + Runtime configuration of the array. If provided, will override the + default values from `zarr.config.array`. + + Returns + ------- + z : Array + The array. + """ + return Array( + sync( + async_api.create( + shape=shape, + chunks=chunks, + dtype=dtype, + compressor=compressor, + fill_value=fill_value, + order=order, + store=store, + synchronizer=synchronizer, + overwrite=overwrite, + path=path, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_version=zarr_version, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + storage_options=storage_options, + config=config, + **kwargs, + ) + ) + ) + + +def create_array( + store: str | StoreLike, + *, + name: str | None = None, + shape: ShapeLike | None = None, + dtype: ZDTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat | None = 3, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, + write_data: bool = True, +) -> Array: + """Create an array. + + This function wraps :func:`zarr.core.array.create_array`. + + Parameters + ---------- + store : str or Store + Store or path to directory in file system or name of zip file. + name : str or None, optional + The name of the array within the store. If ``name`` is ``None``, the array will be located + at the root of the store. + shape : ChunkCoords, optional + Shape of the array. Can be ``None`` if ``data`` is provided. + dtype : ZDTypeLike, optional + Data type of the array. Can be ``None`` if ``data`` is provided. + data : np.ndarray, optional + Array-like data to use for initializing the array. If this parameter is provided, the + ``shape`` and ``dtype`` parameters must be identical to ``data.shape`` and ``data.dtype``, + or ``None``. + chunks : ChunkCoords, optional + Chunk shape of the array. + If not specified, default are guessed based on the shape and dtype. + shards : ChunkCoords, optional + Shard shape of the array. The default value of ``None`` results in no sharding at all. + filters : Iterable[Codec], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v3_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v2_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no ``compressor`` is provided, a default compressor will be used. + in :mod:`zarr.core.config`. + Use ``None`` to omit the default compressor. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` + in :mod:`zarr.core.config`. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If no ``order`` is provided, a default order will be used. + This default can be changed by modifying the value of ``array.order`` in :mod:`zarr.core.config`. + zarr_format : {2, 3}, optional + The zarr format to use when saving. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncoding, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + If `True`, all existing paths in the store will be deleted. + config : ArrayConfigLike, optional + Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the ``data`` parameter + then ``write_data`` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If ``write_data`` is ``False``, then the + array will be left empty. + + Returns + ------- + Array + The array. + + Examples + -------- + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> arr = await zarr.create_array( + >>> store=store, + >>> shape=(100,100), + >>> chunks=(10,10), + >>> dtype='i4', + >>> fill_value=0) + + """ + return Array( + sync( + zarr.core.array.create_array( + store, + name=name, + shape=shape, + dtype=dtype, + data=data, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, + overwrite=overwrite, + config=config, + write_data=write_data, + ) + ) + ) + + +def from_array( + store: str | StoreLike, + *, + data: Array | npt.ArrayLike, + write_data: bool = True, + name: str | None = None, + chunks: Literal["auto", "keep"] | ChunkCoords = "keep", + shards: ShardsLike | None | Literal["keep"] = "keep", + filters: FiltersLike | Literal["keep"] = "keep", + compressors: CompressorsLike | Literal["keep"] = "keep", + serializer: SerializerLike | Literal["keep"] = "keep", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, +) -> Array: + """Create an array from an existing array or array-like. + + Parameters + ---------- + store : str or Store + Store or path to directory in file system or name of zip file for the new array. + data : Array | array-like + The array to copy. + write_data : bool, default True + Whether to copy the data from the input array to the new array. + If ``write_data`` is ``False``, the new array will be created with the same metadata as the + input array, but without any data. + name : str or None, optional + The name of the array within the store. If ``name`` is ``None``, the array will be located + at the root of the store. + chunks : ChunkCoords or "auto" or "keep", optional + Chunk shape of the array. + Following values are supported: + + - "auto": Automatically determine the chunk shape based on the array's shape and dtype. + - "keep": Retain the chunk shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the chunk shape. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise "auto". + shards : ChunkCoords, optional + Shard shape of the array. + Following values are supported: + + - "auto": Automatically determine the shard shape based on the array's shape and chunk shape. + - "keep": Retain the shard shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the shard shape. + - None: No sharding. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise None. + filters : Iterable[Codec] or "auto" or "keep", optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + + Following values are supported: + + - Iterable[Codec]: List of filters to apply to the array. + - "auto": Automatically determine the filters based on the array's dtype. + - "keep": Retain the filters of the data array if it is a zarr Array. + + If no ``filters`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + compressors : Iterable[Codec] or "auto" or "keep", optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + + Following values are supported: + + - Iterable[Codec]: List of compressors to apply to the array. + - "auto": Automatically determine the compressors based on the array's dtype. + - "keep": Retain the compressors of the input array if it is a zarr Array. + + If no ``compressors`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + serializer : dict[str, JSON] | ArrayBytesCodec or "auto" or "keep", optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + + Following values are supported: + + - dict[str, JSON]: A dict representation of an ``ArrayBytesCodec``. + - ArrayBytesCodec: An instance of ``ArrayBytesCodec``. + - "auto": a default serializer will be used. These defaults can be changed by modifying the value of + ``array.v3_default_serializer`` in :mod:`zarr.core.config`. + - "keep": Retain the serializer of the input array if it is a zarr Array. + + fill_value : Any, optional + Fill value for the array. + If not specified, defaults to the fill value of the data array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If not specified, defaults to the memory order of the data array. + zarr_format : {2, 3}, optional + The zarr format to use when saving. + If not specified, defaults to the zarr format of the data array. + attributes : dict, optional + Attributes for the array. + If not specified, defaults to the attributes of the data array. + chunk_key_encoding : ChunkKeyEncoding, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + If not specified and the data array has the same zarr format as the target array, + the chunk key encoding of the data array is used. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + If not specified, defaults to the dimension names of the data array. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfig or ArrayConfigLike, optional + Runtime configuration for the array. + + Returns + ------- + Array + The array. + + Examples + -------- + Create an array from an existing Array:: + + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> store2 = zarr.storage.LocalStore('example.zarr') + >>> arr = zarr.create_array( + >>> store=store, + >>> shape=(100,100), + >>> chunks=(10,10), + >>> dtype='int32', + >>> fill_value=0) + >>> arr2 = zarr.from_array(store2, data=arr) + + + Create an array from an existing NumPy array:: + + >>> import numpy as np + >>> arr3 = zarr.from_array( + zarr.storage.MemoryStore(), + >>> data=np.arange(10000, dtype='i4').reshape(100, 100), + >>> ) + + + Create an array from any array-like object:: + + >>> arr4 = zarr.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=[[1, 2], [3, 4]], + >>> ) + + >>> arr4[...] + array([[1, 2],[3, 4]]) + + Create an array from an existing Array without copying the data:: + + >>> arr5 = zarr.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=arr4, + >>> write_data=False, + >>> ) + + >>> arr5[...] + array([[0, 0],[0, 0]]) + """ + return Array( + sync( + zarr.core.array.from_array( + store, + data=data, + write_data=write_data, + name=name, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, + overwrite=overwrite, + config=config, + ) + ) + ) + + +# TODO: add type annotations for kwargs +def empty(shape: ChunkCoords, **kwargs: Any) -> Array: + """Create an empty array with the specified shape. The contents will be filled with the + array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + return Array(sync(async_api.empty(shape, **kwargs))) + + +# TODO: move ArrayLike to common module +# TODO: add type annotations for kwargs +def empty_like(a: ArrayLike, **kwargs: Any) -> Array: + """Create an empty array like another array. The contents will be filled with the + array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + a : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + return Array(sync(async_api.empty_like(a, **kwargs))) + + +# TODO: add type annotations for kwargs and fill_value +def full(shape: ChunkCoords, fill_value: Any, **kwargs: Any) -> Array: + """Create an array with a default fill value. + + Parameters + ---------- + shape : int or tuple of int + Shape of the empty array. + fill_value : scalar + Fill value. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return Array(sync(async_api.full(shape=shape, fill_value=fill_value, **kwargs))) + + +# TODO: move ArrayLike to common module +# TODO: add type annotations for kwargs +def full_like(a: ArrayLike, **kwargs: Any) -> Array: + """Create a filled array like another array. + + Parameters + ---------- + a : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return Array(sync(async_api.full_like(a, **kwargs))) + + +# TODO: add type annotations for kwargs +def ones(shape: ChunkCoords, **kwargs: Any) -> Array: + """Create an array with a fill value of one. + + Parameters + ---------- + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return Array(sync(async_api.ones(shape, **kwargs))) + + +# TODO: add type annotations for kwargs +def ones_like(a: ArrayLike, **kwargs: Any) -> Array: + """Create an array of ones like another array. + + Parameters + ---------- + a : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return Array(sync(async_api.ones_like(a, **kwargs))) + + +# TODO: update this once async_api.open_array is fully implemented +def open_array( + store: StoreLike | None = None, + *, + zarr_version: ZarrFormat | None = None, + path: PathLike = "", + storage_options: dict[str, Any] | None = None, + **kwargs: Any, +) -> Array: + """Open an array using file-mode-like semantics. + + Parameters + ---------- + store : Store or str + Store or path to directory in file system or name of zip file. + zarr_version : {2, 3, None}, optional + The zarr format to use when saving. + path : str, optional + Path in store to array. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. + **kwargs + Any keyword arguments to pass to ``create``. + + Returns + ------- + AsyncArray + The opened array. + """ + return Array( + sync( + async_api.open_array( + store=store, + zarr_version=zarr_version, + path=path, + storage_options=storage_options, + **kwargs, + ) + ) + ) + + +# TODO: add type annotations for kwargs +def open_like(a: ArrayLike, path: str, **kwargs: Any) -> Array: + """Open a persistent array like another array. + + Parameters + ---------- + a : Array + The shape and data-type of a define these same attributes of the returned array. + path : str + The path to the new array. + **kwargs + Any keyword arguments to pass to the array constructor. + + Returns + ------- + AsyncArray + The opened array. + """ + return Array(sync(async_api.open_like(a, path=path, **kwargs))) + + +# TODO: add type annotations for kwargs +def zeros(shape: ChunkCoords, **kwargs: Any) -> Array: + """Create an array with a fill value of zero. + + Parameters + ---------- + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return Array(sync(async_api.zeros(shape=shape, **kwargs))) + + +# TODO: add type annotations for kwargs +def zeros_like(a: ArrayLike, **kwargs: Any) -> Array: + """Create an array of zeros like another array. + + Parameters + ---------- + a : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return Array(sync(async_api.zeros_like(a, **kwargs))) diff --git a/src/zarr/buffer/__init__.py b/src/zarr/buffer/__init__.py new file mode 100644 index 0000000000..db393f66c7 --- /dev/null +++ b/src/zarr/buffer/__init__.py @@ -0,0 +1,12 @@ +""" +Implementations of the Zarr Buffer interface. + +See Also +======== +zarr.abc.buffer: Abstract base class for the Zarr Buffer interface. +""" + +from zarr.buffer import cpu, gpu +from zarr.core.buffer import default_buffer_prototype + +__all__ = ["cpu", "default_buffer_prototype", "gpu"] diff --git a/src/zarr/buffer/cpu.py b/src/zarr/buffer/cpu.py new file mode 100644 index 0000000000..5307927c06 --- /dev/null +++ b/src/zarr/buffer/cpu.py @@ -0,0 +1,15 @@ +from zarr.core.buffer.cpu import ( + Buffer, + NDBuffer, + as_numpy_array_wrapper, + buffer_prototype, + numpy_buffer_prototype, +) + +__all__ = [ + "Buffer", + "NDBuffer", + "as_numpy_array_wrapper", + "buffer_prototype", + "numpy_buffer_prototype", +] diff --git a/src/zarr/buffer/gpu.py b/src/zarr/buffer/gpu.py new file mode 100644 index 0000000000..dbdc1b1357 --- /dev/null +++ b/src/zarr/buffer/gpu.py @@ -0,0 +1,7 @@ +from zarr.core.buffer.gpu import Buffer, NDBuffer, buffer_prototype + +__all__ = [ + "Buffer", + "NDBuffer", + "buffer_prototype", +] diff --git a/src/zarr/codecs/__init__.py b/src/zarr/codecs/__init__.py new file mode 100644 index 0000000000..165dbe476d --- /dev/null +++ b/src/zarr/codecs/__init__.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from zarr.codecs.blosc import BloscCname, BloscCodec, BloscShuffle +from zarr.codecs.bytes import BytesCodec, Endian +from zarr.codecs.crc32c_ import Crc32cCodec +from zarr.codecs.gzip import GzipCodec +from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation +from zarr.codecs.transpose import TransposeCodec +from zarr.codecs.vlen_utf8 import VLenBytesCodec, VLenUTF8Codec +from zarr.codecs.zstd import ZstdCodec + +__all__ = [ + "BloscCname", + "BloscCodec", + "BloscShuffle", + "BytesCodec", + "Crc32cCodec", + "Endian", + "GzipCodec", + "ShardingCodec", + "ShardingCodecIndexLocation", + "TransposeCodec", + "VLenBytesCodec", + "VLenUTF8Codec", + "ZstdCodec", +] diff --git a/src/zarr/codecs/_v2.py b/src/zarr/codecs/_v2.py new file mode 100644 index 0000000000..08853f27f1 --- /dev/null +++ b/src/zarr/codecs/_v2.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import numcodecs +import numpy as np +from numcodecs.compat import ensure_bytes, ensure_ndarray_like + +from zarr.abc.codec import ArrayBytesCodec +from zarr.registry import get_ndbuffer_class + +if TYPE_CHECKING: + import numcodecs.abc + + from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer, NDBuffer + + +@dataclass(frozen=True) +class V2Codec(ArrayBytesCodec): + filters: tuple[numcodecs.abc.Codec, ...] | None + compressor: numcodecs.abc.Codec | None + + is_fixed_size = False + + async def _decode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> NDBuffer: + cdata = chunk_bytes.as_array_like() + # decompress + if self.compressor: + chunk = await asyncio.to_thread(self.compressor.decode, cdata) + else: + chunk = cdata + + # apply filters + if self.filters: + for f in reversed(self.filters): + chunk = await asyncio.to_thread(f.decode, chunk) + + # view as numpy array with correct dtype + chunk = ensure_ndarray_like(chunk) + # special case object dtype, because incorrect handling can lead to + # segfaults and other bad things happening + if chunk_spec.dtype.dtype_cls is not np.dtypes.ObjectDType: + try: + chunk = chunk.view(chunk_spec.dtype.to_native_dtype()) + except TypeError: + # this will happen if the dtype of the chunk + # does not match the dtype of the array spec i.g. if + # the dtype of the chunk_spec is a string dtype, but the chunk + # is an object array. In this case, we need to convert the object + # array to the correct dtype. + + chunk = np.array(chunk).astype(chunk_spec.dtype.to_native_dtype()) + + elif chunk.dtype != object: + # If we end up here, someone must have hacked around with the filters. + # We cannot deal with object arrays unless there is an object + # codec in the filter chain, i.e., a filter that converts from object + # array to something else during encoding, and converts back to object + # array during decoding. + raise RuntimeError("cannot read object array without object codec") + + # ensure correct chunk shape + chunk = chunk.reshape(-1, order="A") + chunk = chunk.reshape(chunk_spec.shape, order=chunk_spec.order) + + return get_ndbuffer_class().from_ndarray_like(chunk) + + async def _encode_single( + self, + chunk_array: NDBuffer, + chunk_spec: ArraySpec, + ) -> Buffer | None: + chunk = chunk_array.as_ndarray_like() + + # ensure contiguous and correct order + chunk = chunk.astype(chunk_spec.dtype.to_native_dtype(), order=chunk_spec.order, copy=False) + + # apply filters + if self.filters: + for f in self.filters: + chunk = await asyncio.to_thread(f.encode, chunk) + + # check object encoding + if ensure_ndarray_like(chunk).dtype == object: + raise RuntimeError("cannot write object array without object codec") + + # compress + if self.compressor: + cdata = await asyncio.to_thread(self.compressor.encode, chunk) + else: + cdata = chunk + + cdata = ensure_bytes(cdata) + return chunk_spec.prototype.buffer.from_bytes(cdata) + + def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) -> int: + raise NotImplementedError diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py new file mode 100644 index 0000000000..1c5e52e9a4 --- /dev/null +++ b/src/zarr/codecs/blosc.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, replace +from enum import Enum +from functools import cached_property +from typing import TYPE_CHECKING + +import numcodecs +from numcodecs.blosc import Blosc +from packaging.version import Version + +from zarr.abc.codec import BytesBytesCodec +from zarr.core.buffer.cpu import as_numpy_array_wrapper +from zarr.core.common import JSON, parse_enum, parse_named_configuration +from zarr.core.dtype.common import HasItemSize +from zarr.registry import register_codec + +if TYPE_CHECKING: + from typing import Self + + from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer + + +class BloscShuffle(Enum): + """ + Enum for shuffle filter used by blosc. + """ + + noshuffle = "noshuffle" + shuffle = "shuffle" + bitshuffle = "bitshuffle" + + @classmethod + def from_int(cls, num: int) -> BloscShuffle: + blosc_shuffle_int_to_str = { + 0: "noshuffle", + 1: "shuffle", + 2: "bitshuffle", + } + if num not in blosc_shuffle_int_to_str: + raise ValueError(f"Value must be between 0 and 2. Got {num}.") + return BloscShuffle[blosc_shuffle_int_to_str[num]] + + +class BloscCname(Enum): + """ + Enum for compression library used by blosc. + """ + + lz4 = "lz4" + lz4hc = "lz4hc" + blosclz = "blosclz" + zstd = "zstd" + snappy = "snappy" + zlib = "zlib" + + +# See https://zarr.readthedocs.io/en/stable/user-guide/performance.html#configuring-blosc +numcodecs.blosc.use_threads = False + + +def parse_typesize(data: JSON) -> int: + if isinstance(data, int): + if data > 0: + return data + else: + raise ValueError( + f"Value must be greater than 0. Got {data}, which is less or equal to 0." + ) + raise TypeError(f"Value must be an int. Got {type(data)} instead.") + + +# todo: real validation +def parse_clevel(data: JSON) -> int: + if isinstance(data, int): + return data + raise TypeError(f"Value should be an int. Got {type(data)} instead.") + + +def parse_blocksize(data: JSON) -> int: + if isinstance(data, int): + return data + raise TypeError(f"Value should be an int. Got {type(data)} instead.") + + +@dataclass(frozen=True) +class BloscCodec(BytesBytesCodec): + is_fixed_size = False + + typesize: int | None + cname: BloscCname = BloscCname.zstd + clevel: int = 5 + shuffle: BloscShuffle | None = BloscShuffle.noshuffle + blocksize: int = 0 + + def __init__( + self, + *, + typesize: int | None = None, + cname: BloscCname | str = BloscCname.zstd, + clevel: int = 5, + shuffle: BloscShuffle | str | None = None, + blocksize: int = 0, + ) -> None: + typesize_parsed = parse_typesize(typesize) if typesize is not None else None + cname_parsed = parse_enum(cname, BloscCname) + clevel_parsed = parse_clevel(clevel) + shuffle_parsed = parse_enum(shuffle, BloscShuffle) if shuffle is not None else None + blocksize_parsed = parse_blocksize(blocksize) + + object.__setattr__(self, "typesize", typesize_parsed) + object.__setattr__(self, "cname", cname_parsed) + object.__setattr__(self, "clevel", clevel_parsed) + object.__setattr__(self, "shuffle", shuffle_parsed) + object.__setattr__(self, "blocksize", blocksize_parsed) + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> Self: + _, configuration_parsed = parse_named_configuration(data, "blosc") + return cls(**configuration_parsed) # type: ignore[arg-type] + + def to_dict(self) -> dict[str, JSON]: + if self.typesize is None: + raise ValueError("`typesize` needs to be set for serialization.") + if self.shuffle is None: + raise ValueError("`shuffle` needs to be set for serialization.") + return { + "name": "blosc", + "configuration": { + "typesize": self.typesize, + "cname": self.cname.value, + "clevel": self.clevel, + "shuffle": self.shuffle.value, + "blocksize": self.blocksize, + }, + } + + def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: + item_size = 1 + if isinstance(array_spec.dtype, HasItemSize): + item_size = array_spec.dtype.item_size + new_codec = self + if new_codec.typesize is None: + new_codec = replace(new_codec, typesize=item_size) + if new_codec.shuffle is None: + new_codec = replace( + new_codec, + shuffle=(BloscShuffle.bitshuffle if item_size == 1 else BloscShuffle.shuffle), + ) + + return new_codec + + @cached_property + def _blosc_codec(self) -> Blosc: + if self.shuffle is None: + raise ValueError("`shuffle` needs to be set for decoding and encoding.") + map_shuffle_str_to_int = { + BloscShuffle.noshuffle: 0, + BloscShuffle.shuffle: 1, + BloscShuffle.bitshuffle: 2, + } + config_dict = { + "cname": self.cname.name, + "clevel": self.clevel, + "shuffle": map_shuffle_str_to_int[self.shuffle], + "blocksize": self.blocksize, + } + # See https://github.com/zarr-developers/numcodecs/pull/713 + if Version(numcodecs.__version__) >= Version("0.16.0"): + config_dict["typesize"] = self.typesize + return Blosc.from_config(config_dict) + + async def _decode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> Buffer: + return await asyncio.to_thread( + as_numpy_array_wrapper, self._blosc_codec.decode, chunk_bytes, chunk_spec.prototype + ) + + async def _encode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> Buffer | None: + # Since blosc only support host memory, we convert the input and output of the encoding + # between numpy array and buffer + return await asyncio.to_thread( + lambda chunk: chunk_spec.prototype.buffer.from_bytes( + self._blosc_codec.encode(chunk.as_numpy_array()) + ), + chunk_bytes, + ) + + def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) -> int: + raise NotImplementedError + + +register_codec("blosc", BloscCodec) diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py new file mode 100644 index 0000000000..d663a3b2cc --- /dev/null +++ b/src/zarr/codecs/bytes.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import sys +from dataclasses import dataclass, replace +from enum import Enum +from typing import TYPE_CHECKING + +import numpy as np + +from zarr.abc.codec import ArrayBytesCodec +from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer +from zarr.core.common import JSON, parse_enum, parse_named_configuration +from zarr.core.dtype.common import HasEndianness +from zarr.registry import register_codec + +if TYPE_CHECKING: + from typing import Self + + from zarr.core.array_spec import ArraySpec + + +class Endian(Enum): + """ + Enum for endian type used by bytes codec. + """ + + big = "big" + little = "little" + + +default_system_endian = Endian(sys.byteorder) + + +@dataclass(frozen=True) +class BytesCodec(ArrayBytesCodec): + is_fixed_size = True + + endian: Endian | None + + def __init__(self, *, endian: Endian | str | None = default_system_endian) -> None: + endian_parsed = None if endian is None else parse_enum(endian, Endian) + + object.__setattr__(self, "endian", endian_parsed) + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> Self: + _, configuration_parsed = parse_named_configuration( + data, "bytes", require_configuration=False + ) + configuration_parsed = configuration_parsed or {} + return cls(**configuration_parsed) # type: ignore[arg-type] + + def to_dict(self) -> dict[str, JSON]: + if self.endian is None: + return {"name": "bytes"} + else: + return {"name": "bytes", "configuration": {"endian": self.endian.value}} + + def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: + if not isinstance(array_spec.dtype, HasEndianness): + if self.endian is not None: + return replace(self, endian=None) + elif self.endian is None: + raise ValueError( + "The `endian` configuration needs to be specified for multi-byte data types." + ) + return self + + async def _decode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> NDBuffer: + assert isinstance(chunk_bytes, Buffer) + # TODO: remove endianness enum in favor of literal union + endian_str = self.endian.value if self.endian is not None else None + if isinstance(chunk_spec.dtype, HasEndianness): + dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg] + else: + dtype = chunk_spec.dtype.to_native_dtype() + as_array_like = chunk_bytes.as_array_like() + if isinstance(as_array_like, NDArrayLike): + as_nd_array_like = as_array_like + else: + as_nd_array_like = np.asanyarray(as_array_like) + chunk_array = chunk_spec.prototype.nd_buffer.from_ndarray_like( + as_nd_array_like.view(dtype=dtype) + ) + + # ensure correct chunk shape + if chunk_array.shape != chunk_spec.shape: + chunk_array = chunk_array.reshape( + chunk_spec.shape, + ) + return chunk_array + + async def _encode_single( + self, + chunk_array: NDBuffer, + chunk_spec: ArraySpec, + ) -> Buffer | None: + assert isinstance(chunk_array, NDBuffer) + if ( + chunk_array.dtype.itemsize > 1 + and self.endian is not None + and self.endian != chunk_array.byteorder + ): + # type-ignore is a numpy bug + # see https://github.com/numpy/numpy/issues/26473 + new_dtype = chunk_array.dtype.newbyteorder(self.endian.name) # type: ignore[arg-type] + chunk_array = chunk_array.astype(new_dtype) + + nd_array = chunk_array.as_ndarray_like() + # Flatten the nd-array (only copy if needed) and reinterpret as bytes + nd_array = nd_array.ravel().view(dtype="B") + return chunk_spec.prototype.buffer.from_array_like(nd_array) + + def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: + return input_byte_length + + +register_codec("bytes", BytesCodec) + +# compatibility with earlier versions of ZEP1 +register_codec("endian", BytesCodec) diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py new file mode 100644 index 0000000000..6da673ceac --- /dev/null +++ b/src/zarr/codecs/crc32c_.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, cast + +import numpy as np +import typing_extensions +from crc32c import crc32c + +from zarr.abc.codec import BytesBytesCodec +from zarr.core.common import JSON, parse_named_configuration +from zarr.registry import register_codec + +if TYPE_CHECKING: + from typing import Self + + from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer + + +@dataclass(frozen=True) +class Crc32cCodec(BytesBytesCodec): + is_fixed_size = True + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> Self: + parse_named_configuration(data, "crc32c", require_configuration=False) + return cls() + + def to_dict(self) -> dict[str, JSON]: + return {"name": "crc32c"} + + async def _decode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> Buffer: + data = chunk_bytes.as_numpy_array() + crc32_bytes = data[-4:] + inner_bytes = data[:-4] + + # Need to do a manual cast until https://github.com/numpy/numpy/issues/26783 is resolved + computed_checksum = np.uint32( + crc32c(cast("typing_extensions.Buffer", inner_bytes)) + ).tobytes() + stored_checksum = bytes(crc32_bytes) + if computed_checksum != stored_checksum: + raise ValueError( + f"Stored and computed checksum do not match. Stored: {stored_checksum!r}. Computed: {computed_checksum!r}." + ) + return chunk_spec.prototype.buffer.from_array_like(inner_bytes) + + async def _encode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> Buffer | None: + data = chunk_bytes.as_numpy_array() + # Calculate the checksum and "cast" it to a numpy array + checksum = np.array([crc32c(cast("typing_extensions.Buffer", data))], dtype=np.uint32) + # Append the checksum (as bytes) to the data + return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("B"))) + + def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: + return input_byte_length + 4 + + +register_codec("crc32c", Crc32cCodec) diff --git a/src/zarr/codecs/gzip.py b/src/zarr/codecs/gzip.py new file mode 100644 index 0000000000..b6e693148e --- /dev/null +++ b/src/zarr/codecs/gzip.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from numcodecs.gzip import GZip + +from zarr.abc.codec import BytesBytesCodec +from zarr.core.buffer.cpu import as_numpy_array_wrapper +from zarr.core.common import JSON, parse_named_configuration +from zarr.registry import register_codec + +if TYPE_CHECKING: + from typing import Self + + from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer + + +def parse_gzip_level(data: JSON) -> int: + if not isinstance(data, (int)): + raise TypeError(f"Expected int, got {type(data)}") + if data not in range(10): + raise ValueError( + f"Expected an integer from the inclusive range (0, 9). Got {data} instead." + ) + return data + + +@dataclass(frozen=True) +class GzipCodec(BytesBytesCodec): + is_fixed_size = False + + level: int = 5 + + def __init__(self, *, level: int = 5) -> None: + level_parsed = parse_gzip_level(level) + + object.__setattr__(self, "level", level_parsed) + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> Self: + _, configuration_parsed = parse_named_configuration(data, "gzip") + return cls(**configuration_parsed) # type: ignore[arg-type] + + def to_dict(self) -> dict[str, JSON]: + return {"name": "gzip", "configuration": {"level": self.level}} + + async def _decode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> Buffer: + return await asyncio.to_thread( + as_numpy_array_wrapper, GZip(self.level).decode, chunk_bytes, chunk_spec.prototype + ) + + async def _encode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> Buffer | None: + return await asyncio.to_thread( + as_numpy_array_wrapper, GZip(self.level).encode, chunk_bytes, chunk_spec.prototype + ) + + def compute_encoded_size( + self, + _input_byte_length: int, + _chunk_spec: ArraySpec, + ) -> int: + raise NotImplementedError + + +register_codec("gzip", GzipCodec) diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py new file mode 100644 index 0000000000..cd8676b4d1 --- /dev/null +++ b/src/zarr/codecs/sharding.py @@ -0,0 +1,766 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping, MutableMapping +from dataclasses import dataclass, field, replace +from enum import Enum +from functools import lru_cache +from operator import itemgetter +from typing import TYPE_CHECKING, Any, NamedTuple, cast + +import numpy as np +import numpy.typing as npt + +from zarr.abc.codec import ( + ArrayBytesCodec, + ArrayBytesCodecPartialDecodeMixin, + ArrayBytesCodecPartialEncodeMixin, + Codec, + CodecPipeline, +) +from zarr.abc.store import ( + ByteGetter, + ByteRequest, + ByteSetter, + RangeByteRequest, + SuffixByteRequest, +) +from zarr.codecs.bytes import BytesCodec +from zarr.codecs.crc32c_ import Crc32cCodec +from zarr.core.array_spec import ArrayConfig, ArraySpec +from zarr.core.buffer import ( + Buffer, + BufferPrototype, + NDBuffer, + default_buffer_prototype, + numpy_buffer_prototype, +) +from zarr.core.chunk_grids import ChunkGrid, RegularChunkGrid +from zarr.core.common import ( + ChunkCoords, + ChunkCoordsLike, + parse_enum, + parse_named_configuration, + parse_shapelike, + product, +) +from zarr.core.dtype.npy.int import UInt64 +from zarr.core.indexing import ( + BasicIndexer, + SelectorTuple, + c_order_iter, + get_indexer, + morton_order_iter, +) +from zarr.core.metadata.v3 import parse_codecs +from zarr.registry import get_ndbuffer_class, get_pipeline_class, register_codec + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Iterator + from typing import Self + + from zarr.core.common import JSON + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + +MAX_UINT_64 = 2**64 - 1 +ShardMapping = Mapping[ChunkCoords, Buffer] +ShardMutableMapping = MutableMapping[ChunkCoords, Buffer] + + +class ShardingCodecIndexLocation(Enum): + """ + Enum for index location used by the sharding codec. + """ + + start = "start" + end = "end" + + +def parse_index_location(data: object) -> ShardingCodecIndexLocation: + return parse_enum(data, ShardingCodecIndexLocation) + + +@dataclass(frozen=True) +class _ShardingByteGetter(ByteGetter): + shard_dict: ShardMapping + chunk_coords: ChunkCoords + + async def get( + self, prototype: BufferPrototype, byte_range: ByteRequest | None = None + ) -> Buffer | None: + assert byte_range is None, "byte_range is not supported within shards" + assert prototype == default_buffer_prototype(), ( + f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}" + ) + return self.shard_dict.get(self.chunk_coords) + + +@dataclass(frozen=True) +class _ShardingByteSetter(_ShardingByteGetter, ByteSetter): + shard_dict: ShardMutableMapping + + async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None: + assert byte_range is None, "byte_range is not supported within shards" + self.shard_dict[self.chunk_coords] = value + + async def delete(self) -> None: + del self.shard_dict[self.chunk_coords] + + async def set_if_not_exists(self, default: Buffer) -> None: + self.shard_dict.setdefault(self.chunk_coords, default) + + +class _ShardIndex(NamedTuple): + # dtype uint64, shape (chunks_per_shard_0, chunks_per_shard_1, ..., 2) + offsets_and_lengths: npt.NDArray[np.uint64] + + @property + def chunks_per_shard(self) -> ChunkCoords: + result = tuple(self.offsets_and_lengths.shape[0:-1]) + # The cast is required until https://github.com/numpy/numpy/pull/27211 is merged + return cast("ChunkCoords", result) + + def _localize_chunk(self, chunk_coords: ChunkCoords) -> ChunkCoords: + return tuple( + chunk_i % shard_i + for chunk_i, shard_i in zip(chunk_coords, self.offsets_and_lengths.shape, strict=False) + ) + + def is_all_empty(self) -> bool: + return bool(np.array_equiv(self.offsets_and_lengths, MAX_UINT_64)) + + def get_full_chunk_map(self) -> npt.NDArray[np.bool_]: + return np.not_equal(self.offsets_and_lengths[..., 0], MAX_UINT_64) + + def get_chunk_slice(self, chunk_coords: ChunkCoords) -> tuple[int, int] | None: + localized_chunk = self._localize_chunk(chunk_coords) + chunk_start, chunk_len = self.offsets_and_lengths[localized_chunk] + if (chunk_start, chunk_len) == (MAX_UINT_64, MAX_UINT_64): + return None + else: + return (int(chunk_start), int(chunk_start + chunk_len)) + + def set_chunk_slice(self, chunk_coords: ChunkCoords, chunk_slice: slice | None) -> None: + localized_chunk = self._localize_chunk(chunk_coords) + if chunk_slice is None: + self.offsets_and_lengths[localized_chunk] = (MAX_UINT_64, MAX_UINT_64) + else: + self.offsets_and_lengths[localized_chunk] = ( + chunk_slice.start, + chunk_slice.stop - chunk_slice.start, + ) + + def is_dense(self, chunk_byte_length: int) -> bool: + sorted_offsets_and_lengths = sorted( + [ + (offset, length) + for offset, length in self.offsets_and_lengths + if offset != MAX_UINT_64 + ], + key=itemgetter(0), + ) + + # Are all non-empty offsets unique? + if len( + {offset for offset, _ in sorted_offsets_and_lengths if offset != MAX_UINT_64} + ) != len(sorted_offsets_and_lengths): + return False + + return all( + offset % chunk_byte_length == 0 and length == chunk_byte_length + for offset, length in sorted_offsets_and_lengths + ) + + @classmethod + def create_empty(cls, chunks_per_shard: ChunkCoords) -> _ShardIndex: + offsets_and_lengths = np.zeros(chunks_per_shard + (2,), dtype=" _ShardReader: + shard_index_size = codec._shard_index_size(chunks_per_shard) + obj = cls() + obj.buf = buf + if codec.index_location == ShardingCodecIndexLocation.start: + shard_index_bytes = obj.buf[:shard_index_size] + else: + shard_index_bytes = obj.buf[-shard_index_size:] + + obj.index = await codec._decode_shard_index(shard_index_bytes, chunks_per_shard) + return obj + + @classmethod + def create_empty( + cls, chunks_per_shard: ChunkCoords, buffer_prototype: BufferPrototype | None = None + ) -> _ShardReader: + if buffer_prototype is None: + buffer_prototype = default_buffer_prototype() + index = _ShardIndex.create_empty(chunks_per_shard) + obj = cls() + obj.buf = buffer_prototype.buffer.create_zero_length() + obj.index = index + return obj + + def __getitem__(self, chunk_coords: ChunkCoords) -> Buffer: + chunk_byte_slice = self.index.get_chunk_slice(chunk_coords) + if chunk_byte_slice: + return self.buf[chunk_byte_slice[0] : chunk_byte_slice[1]] + raise KeyError + + def __len__(self) -> int: + return int(self.index.offsets_and_lengths.size / 2) + + def __iter__(self) -> Iterator[ChunkCoords]: + return c_order_iter(self.index.offsets_and_lengths.shape[:-1]) + + def is_empty(self) -> bool: + return self.index.is_all_empty() + + +class _ShardBuilder(_ShardReader, ShardMutableMapping): + buf: Buffer + index: _ShardIndex + + @classmethod + def merge_with_morton_order( + cls, + chunks_per_shard: ChunkCoords, + tombstones: set[ChunkCoords], + *shard_dicts: ShardMapping, + ) -> _ShardBuilder: + obj = cls.create_empty(chunks_per_shard) + for chunk_coords in morton_order_iter(chunks_per_shard): + if chunk_coords in tombstones: + continue + for shard_dict in shard_dicts: + maybe_value = shard_dict.get(chunk_coords, None) + if maybe_value is not None: + obj[chunk_coords] = maybe_value + break + return obj + + @classmethod + def create_empty( + cls, chunks_per_shard: ChunkCoords, buffer_prototype: BufferPrototype | None = None + ) -> _ShardBuilder: + if buffer_prototype is None: + buffer_prototype = default_buffer_prototype() + obj = cls() + obj.buf = buffer_prototype.buffer.create_zero_length() + obj.index = _ShardIndex.create_empty(chunks_per_shard) + return obj + + def __setitem__(self, chunk_coords: ChunkCoords, value: Buffer) -> None: + chunk_start = len(self.buf) + chunk_length = len(value) + self.buf += value + self.index.set_chunk_slice(chunk_coords, slice(chunk_start, chunk_start + chunk_length)) + + def __delitem__(self, chunk_coords: ChunkCoords) -> None: + raise NotImplementedError + + async def finalize( + self, + index_location: ShardingCodecIndexLocation, + index_encoder: Callable[[_ShardIndex], Awaitable[Buffer]], + ) -> Buffer: + index_bytes = await index_encoder(self.index) + if index_location == ShardingCodecIndexLocation.start: + empty_chunks_mask = self.index.offsets_and_lengths[..., 0] == MAX_UINT_64 + self.index.offsets_and_lengths[~empty_chunks_mask, 0] += len(index_bytes) + index_bytes = await index_encoder(self.index) # encode again with corrected offsets + out_buf = index_bytes + self.buf + else: + out_buf = self.buf + index_bytes + return out_buf + + +@dataclass(frozen=True) +class _MergingShardBuilder(ShardMutableMapping): + old_dict: _ShardReader + new_dict: _ShardBuilder + tombstones: set[ChunkCoords] = field(default_factory=set) + + def __getitem__(self, chunk_coords: ChunkCoords) -> Buffer: + chunk_bytes_maybe = self.new_dict.get(chunk_coords) + if chunk_bytes_maybe is not None: + return chunk_bytes_maybe + return self.old_dict[chunk_coords] + + def __setitem__(self, chunk_coords: ChunkCoords, value: Buffer) -> None: + self.new_dict[chunk_coords] = value + + def __delitem__(self, chunk_coords: ChunkCoords) -> None: + self.tombstones.add(chunk_coords) + + def __len__(self) -> int: + return self.old_dict.__len__() + + def __iter__(self) -> Iterator[ChunkCoords]: + return self.old_dict.__iter__() + + def is_empty(self) -> bool: + full_chunk_coords_map = self.old_dict.index.get_full_chunk_map() + full_chunk_coords_map = np.logical_or( + full_chunk_coords_map, self.new_dict.index.get_full_chunk_map() + ) + for tombstone in self.tombstones: + full_chunk_coords_map[tombstone] = False + return bool(np.array_equiv(full_chunk_coords_map, False)) + + async def finalize( + self, + index_location: ShardingCodecIndexLocation, + index_encoder: Callable[[_ShardIndex], Awaitable[Buffer]], + ) -> Buffer: + shard_builder = _ShardBuilder.merge_with_morton_order( + self.new_dict.index.chunks_per_shard, + self.tombstones, + self.new_dict, + self.old_dict, + ) + return await shard_builder.finalize(index_location, index_encoder) + + +@dataclass(frozen=True) +class ShardingCodec( + ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin +): + chunk_shape: ChunkCoords + codecs: tuple[Codec, ...] + index_codecs: tuple[Codec, ...] + index_location: ShardingCodecIndexLocation = ShardingCodecIndexLocation.end + + def __init__( + self, + *, + chunk_shape: ChunkCoordsLike, + codecs: Iterable[Codec | dict[str, JSON]] = (BytesCodec(),), + index_codecs: Iterable[Codec | dict[str, JSON]] = (BytesCodec(), Crc32cCodec()), + index_location: ShardingCodecIndexLocation | str = ShardingCodecIndexLocation.end, + ) -> None: + chunk_shape_parsed = parse_shapelike(chunk_shape) + codecs_parsed = parse_codecs(codecs) + index_codecs_parsed = parse_codecs(index_codecs) + index_location_parsed = parse_index_location(index_location) + + object.__setattr__(self, "chunk_shape", chunk_shape_parsed) + object.__setattr__(self, "codecs", codecs_parsed) + object.__setattr__(self, "index_codecs", index_codecs_parsed) + object.__setattr__(self, "index_location", index_location_parsed) + + # Use instance-local lru_cache to avoid memory leaks + + # numpy void scalars are not hashable, which means an array spec with a fill value that is + # a numpy void scalar will break the lru_cache. This is commented for now but should be + # fixed. See https://github.com/zarr-developers/zarr-python/issues/3054 + # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) + object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec)) + object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard)) + + # todo: typedict return type + def __getstate__(self) -> dict[str, Any]: + return self.to_dict() + + def __setstate__(self, state: dict[str, Any]) -> None: + config = state["configuration"] + object.__setattr__(self, "chunk_shape", parse_shapelike(config["chunk_shape"])) + object.__setattr__(self, "codecs", parse_codecs(config["codecs"])) + object.__setattr__(self, "index_codecs", parse_codecs(config["index_codecs"])) + object.__setattr__(self, "index_location", parse_index_location(config["index_location"])) + + # Use instance-local lru_cache to avoid memory leaks + # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec)) + object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec)) + object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard)) + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> Self: + _, configuration_parsed = parse_named_configuration(data, "sharding_indexed") + return cls(**configuration_parsed) # type: ignore[arg-type] + + @property + def codec_pipeline(self) -> CodecPipeline: + return get_pipeline_class().from_codecs(self.codecs) + + def to_dict(self) -> dict[str, JSON]: + return { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": self.chunk_shape, + "codecs": tuple(s.to_dict() for s in self.codecs), + "index_codecs": tuple(s.to_dict() for s in self.index_codecs), + "index_location": self.index_location.value, + }, + } + + def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: + shard_spec = self._get_chunk_spec(array_spec) + evolved_codecs = tuple(c.evolve_from_array_spec(array_spec=shard_spec) for c in self.codecs) + if evolved_codecs != self.codecs: + return replace(self, codecs=evolved_codecs) + return self + + def validate( + self, + *, + shape: ChunkCoords, + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_grid: ChunkGrid, + ) -> None: + if len(self.chunk_shape) != len(shape): + raise ValueError( + "The shard's `chunk_shape` and array's `shape` need to have the same number of dimensions." + ) + if not isinstance(chunk_grid, RegularChunkGrid): + raise TypeError("Sharding is only compatible with regular chunk grids.") + if not all( + s % c == 0 + for s, c in zip( + chunk_grid.chunk_shape, + self.chunk_shape, + strict=False, + ) + ): + raise ValueError( + "The array's `chunk_shape` needs to be divisible by the shard's inner `chunk_shape`." + ) + + async def _decode_single( + self, + shard_bytes: Buffer, + shard_spec: ArraySpec, + ) -> NDBuffer: + shard_shape = shard_spec.shape + chunk_shape = self.chunk_shape + chunks_per_shard = self._get_chunks_per_shard(shard_spec) + chunk_spec = self._get_chunk_spec(shard_spec) + + indexer = BasicIndexer( + tuple(slice(0, s) for s in shard_shape), + shape=shard_shape, + chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape), + ) + + # setup output array + out = chunk_spec.prototype.nd_buffer.create( + shape=shard_shape, + dtype=shard_spec.dtype.to_native_dtype(), + order=shard_spec.order, + fill_value=0, + ) + shard_dict = await _ShardReader.from_bytes(shard_bytes, self, chunks_per_shard) + + if shard_dict.index.is_all_empty(): + out.fill(shard_spec.fill_value) + return out + + # decoding chunks and writing them into the output buffer + await self.codec_pipeline.read( + [ + ( + _ShardingByteGetter(shard_dict, chunk_coords), + chunk_spec, + chunk_selection, + out_selection, + is_complete_shard, + ) + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer + ], + out, + ) + + return out + + async def _decode_partial_single( + self, + byte_getter: ByteGetter, + selection: SelectorTuple, + shard_spec: ArraySpec, + ) -> NDBuffer | None: + shard_shape = shard_spec.shape + chunk_shape = self.chunk_shape + chunks_per_shard = self._get_chunks_per_shard(shard_spec) + chunk_spec = self._get_chunk_spec(shard_spec) + + indexer = get_indexer( + selection, + shape=shard_shape, + chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape), + ) + + # setup output array + out = shard_spec.prototype.nd_buffer.create( + shape=indexer.shape, + dtype=shard_spec.dtype.to_native_dtype(), + order=shard_spec.order, + fill_value=0, + ) + + indexed_chunks = list(indexer) + all_chunk_coords = {chunk_coords for chunk_coords, *_ in indexed_chunks} + + # reading bytes of all requested chunks + shard_dict: ShardMapping = {} + if self._is_total_shard(all_chunk_coords, chunks_per_shard): + # read entire shard + shard_dict_maybe = await self._load_full_shard_maybe( + byte_getter=byte_getter, + prototype=chunk_spec.prototype, + chunks_per_shard=chunks_per_shard, + ) + if shard_dict_maybe is None: + return None + shard_dict = shard_dict_maybe + else: + # read some chunks within the shard + shard_index = await self._load_shard_index_maybe(byte_getter, chunks_per_shard) + if shard_index is None: + return None + shard_dict = {} + for chunk_coords in all_chunk_coords: + chunk_byte_slice = shard_index.get_chunk_slice(chunk_coords) + if chunk_byte_slice: + chunk_bytes = await byte_getter.get( + prototype=chunk_spec.prototype, + byte_range=RangeByteRequest(chunk_byte_slice[0], chunk_byte_slice[1]), + ) + if chunk_bytes: + shard_dict[chunk_coords] = chunk_bytes + + # decoding chunks and writing them into the output buffer + await self.codec_pipeline.read( + [ + ( + _ShardingByteGetter(shard_dict, chunk_coords), + chunk_spec, + chunk_selection, + out_selection, + is_complete_shard, + ) + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer + ], + out, + ) + + if hasattr(indexer, "sel_shape"): + return out.reshape(indexer.sel_shape) + else: + return out + + async def _encode_single( + self, + shard_array: NDBuffer, + shard_spec: ArraySpec, + ) -> Buffer | None: + shard_shape = shard_spec.shape + chunk_shape = self.chunk_shape + chunks_per_shard = self._get_chunks_per_shard(shard_spec) + chunk_spec = self._get_chunk_spec(shard_spec) + + indexer = list( + BasicIndexer( + tuple(slice(0, s) for s in shard_shape), + shape=shard_shape, + chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape), + ) + ) + + shard_builder = _ShardBuilder.create_empty(chunks_per_shard) + + await self.codec_pipeline.write( + [ + ( + _ShardingByteSetter(shard_builder, chunk_coords), + chunk_spec, + chunk_selection, + out_selection, + is_complete_shard, + ) + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer + ], + shard_array, + ) + + return await shard_builder.finalize(self.index_location, self._encode_shard_index) + + async def _encode_partial_single( + self, + byte_setter: ByteSetter, + shard_array: NDBuffer, + selection: SelectorTuple, + shard_spec: ArraySpec, + ) -> None: + shard_shape = shard_spec.shape + chunk_shape = self.chunk_shape + chunks_per_shard = self._get_chunks_per_shard(shard_spec) + chunk_spec = self._get_chunk_spec(shard_spec) + + shard_dict = _MergingShardBuilder( + await self._load_full_shard_maybe( + byte_getter=byte_setter, + prototype=chunk_spec.prototype, + chunks_per_shard=chunks_per_shard, + ) + or _ShardReader.create_empty(chunks_per_shard), + _ShardBuilder.create_empty(chunks_per_shard), + ) + + indexer = list( + get_indexer( + selection, shape=shard_shape, chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape) + ) + ) + + await self.codec_pipeline.write( + [ + ( + _ShardingByteSetter(shard_dict, chunk_coords), + chunk_spec, + chunk_selection, + out_selection, + is_complete_shard, + ) + for chunk_coords, chunk_selection, out_selection, is_complete_shard in indexer + ], + shard_array, + ) + + if shard_dict.is_empty(): + await byte_setter.delete() + else: + await byte_setter.set( + await shard_dict.finalize( + self.index_location, + self._encode_shard_index, + ) + ) + + def _is_total_shard( + self, all_chunk_coords: set[ChunkCoords], chunks_per_shard: ChunkCoords + ) -> bool: + return len(all_chunk_coords) == product(chunks_per_shard) and all( + chunk_coords in all_chunk_coords for chunk_coords in c_order_iter(chunks_per_shard) + ) + + async def _decode_shard_index( + self, index_bytes: Buffer, chunks_per_shard: ChunkCoords + ) -> _ShardIndex: + index_array = next( + iter( + await get_pipeline_class() + .from_codecs(self.index_codecs) + .decode( + [(index_bytes, self._get_index_chunk_spec(chunks_per_shard))], + ) + ) + ) + assert index_array is not None + return _ShardIndex(index_array.as_numpy_array()) + + async def _encode_shard_index(self, index: _ShardIndex) -> Buffer: + index_bytes = next( + iter( + await get_pipeline_class() + .from_codecs(self.index_codecs) + .encode( + [ + ( + get_ndbuffer_class().from_numpy_array(index.offsets_and_lengths), + self._get_index_chunk_spec(index.chunks_per_shard), + ) + ], + ) + ) + ) + assert index_bytes is not None + assert isinstance(index_bytes, Buffer) + return index_bytes + + def _shard_index_size(self, chunks_per_shard: ChunkCoords) -> int: + return ( + get_pipeline_class() + .from_codecs(self.index_codecs) + .compute_encoded_size( + 16 * product(chunks_per_shard), self._get_index_chunk_spec(chunks_per_shard) + ) + ) + + def _get_index_chunk_spec(self, chunks_per_shard: ChunkCoords) -> ArraySpec: + return ArraySpec( + shape=chunks_per_shard + (2,), + dtype=UInt64(endianness="little"), + fill_value=MAX_UINT_64, + config=ArrayConfig( + order="C", write_empty_chunks=False + ), # Note: this is hard-coded for simplicity -- it is not surfaced into user code, + prototype=default_buffer_prototype(), + ) + + def _get_chunk_spec(self, shard_spec: ArraySpec) -> ArraySpec: + return ArraySpec( + shape=self.chunk_shape, + dtype=shard_spec.dtype, + fill_value=shard_spec.fill_value, + config=shard_spec.config, + prototype=shard_spec.prototype, + ) + + def _get_chunks_per_shard(self, shard_spec: ArraySpec) -> ChunkCoords: + return tuple( + s // c + for s, c in zip( + shard_spec.shape, + self.chunk_shape, + strict=False, + ) + ) + + async def _load_shard_index_maybe( + self, byte_getter: ByteGetter, chunks_per_shard: ChunkCoords + ) -> _ShardIndex | None: + shard_index_size = self._shard_index_size(chunks_per_shard) + if self.index_location == ShardingCodecIndexLocation.start: + index_bytes = await byte_getter.get( + prototype=numpy_buffer_prototype(), + byte_range=RangeByteRequest(0, shard_index_size), + ) + else: + index_bytes = await byte_getter.get( + prototype=numpy_buffer_prototype(), byte_range=SuffixByteRequest(shard_index_size) + ) + if index_bytes is not None: + return await self._decode_shard_index(index_bytes, chunks_per_shard) + return None + + async def _load_shard_index( + self, byte_getter: ByteGetter, chunks_per_shard: ChunkCoords + ) -> _ShardIndex: + return ( + await self._load_shard_index_maybe(byte_getter, chunks_per_shard) + ) or _ShardIndex.create_empty(chunks_per_shard) + + async def _load_full_shard_maybe( + self, byte_getter: ByteGetter, prototype: BufferPrototype, chunks_per_shard: ChunkCoords + ) -> _ShardReader | None: + shard_bytes = await byte_getter.get(prototype=prototype) + + return ( + await _ShardReader.from_bytes(shard_bytes, self, chunks_per_shard) + if shard_bytes + else None + ) + + def compute_encoded_size(self, input_byte_length: int, shard_spec: ArraySpec) -> int: + chunks_per_shard = self._get_chunks_per_shard(shard_spec) + return input_byte_length + self._shard_index_size(chunks_per_shard) + + +register_codec("sharding_indexed", ShardingCodec) diff --git a/src/zarr/codecs/transpose.py b/src/zarr/codecs/transpose.py new file mode 100644 index 0000000000..be89690441 --- /dev/null +++ b/src/zarr/codecs/transpose.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, cast + +import numpy as np + +from zarr.abc.codec import ArrayArrayCodec +from zarr.core.array_spec import ArraySpec +from zarr.core.common import JSON, ChunkCoordsLike, parse_named_configuration +from zarr.registry import register_codec + +if TYPE_CHECKING: + from typing import Self + + from zarr.core.buffer import NDBuffer + from zarr.core.chunk_grids import ChunkGrid + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + + +def parse_transpose_order(data: JSON | Iterable[int]) -> tuple[int, ...]: + if not isinstance(data, Iterable): + raise TypeError(f"Expected an iterable. Got {data} instead.") + if not all(isinstance(a, int) for a in data): + raise TypeError(f"Expected an iterable of integers. Got {data} instead.") + return tuple(cast("Iterable[int]", data)) + + +@dataclass(frozen=True) +class TransposeCodec(ArrayArrayCodec): + is_fixed_size = True + + order: tuple[int, ...] + + def __init__(self, *, order: ChunkCoordsLike) -> None: + order_parsed = parse_transpose_order(order) + + object.__setattr__(self, "order", order_parsed) + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> Self: + _, configuration_parsed = parse_named_configuration(data, "transpose") + return cls(**configuration_parsed) # type: ignore[arg-type] + + def to_dict(self) -> dict[str, JSON]: + return {"name": "transpose", "configuration": {"order": tuple(self.order)}} + + def validate( + self, + shape: tuple[int, ...], + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_grid: ChunkGrid, + ) -> None: + if len(self.order) != len(shape): + raise ValueError( + f"The `order` tuple needs have as many entries as there are dimensions in the array. Got {self.order}." + ) + if len(self.order) != len(set(self.order)): + raise ValueError( + f"There must not be duplicates in the `order` tuple. Got {self.order}." + ) + if not all(0 <= x < len(shape) for x in self.order): + raise ValueError( + f"All entries in the `order` tuple must be between 0 and the number of dimensions in the array. Got {self.order}." + ) + + def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: + ndim = array_spec.ndim + if len(self.order) != ndim: + raise ValueError( + f"The `order` tuple needs have as many entries as there are dimensions in the array. Got {self.order}." + ) + if len(self.order) != len(set(self.order)): + raise ValueError( + f"There must not be duplicates in the `order` tuple. Got {self.order}." + ) + if not all(0 <= x < ndim for x in self.order): + raise ValueError( + f"All entries in the `order` tuple must be between 0 and the number of dimensions in the array. Got {self.order}." + ) + order = tuple(self.order) + + if order != self.order: + return replace(self, order=order) + return self + + def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: + return ArraySpec( + shape=tuple(chunk_spec.shape[self.order[i]] for i in range(chunk_spec.ndim)), + dtype=chunk_spec.dtype, + fill_value=chunk_spec.fill_value, + config=chunk_spec.config, + prototype=chunk_spec.prototype, + ) + + async def _decode_single( + self, + chunk_array: NDBuffer, + chunk_spec: ArraySpec, + ) -> NDBuffer: + inverse_order = np.argsort(self.order) + return chunk_array.transpose(inverse_order) + + async def _encode_single( + self, + chunk_array: NDBuffer, + _chunk_spec: ArraySpec, + ) -> NDBuffer | None: + return chunk_array.transpose(self.order) + + def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: + return input_byte_length + + +register_codec("transpose", TransposeCodec) diff --git a/src/zarr/codecs/vlen_utf8.py b/src/zarr/codecs/vlen_utf8.py new file mode 100644 index 0000000000..b7c0418b2e --- /dev/null +++ b/src/zarr/codecs/vlen_utf8.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING +from warnings import warn + +import numpy as np +from numcodecs.vlen import VLenBytes, VLenUTF8 + +from zarr.abc.codec import ArrayBytesCodec +from zarr.core.buffer import Buffer, NDBuffer +from zarr.core.common import JSON, parse_named_configuration +from zarr.registry import register_codec + +if TYPE_CHECKING: + from typing import Self + + from zarr.core.array_spec import ArraySpec + + +# can use a global because there are no parameters +_vlen_utf8_codec = VLenUTF8() +_vlen_bytes_codec = VLenBytes() + + +@dataclass(frozen=True) +class VLenUTF8Codec(ArrayBytesCodec): + def __init__(self) -> None: + warn( + "The codec `vlen-utf8` is currently not part in the Zarr format 3 specification. It " + "may not be supported by other zarr implementations and may change in the future.", + category=UserWarning, + stacklevel=2, + ) + super().__init__() + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> Self: + _, configuration_parsed = parse_named_configuration( + data, "vlen-utf8", require_configuration=False + ) + configuration_parsed = configuration_parsed or {} + return cls(**configuration_parsed) + + def to_dict(self) -> dict[str, JSON]: + return {"name": "vlen-utf8", "configuration": {}} + + def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: + return self + + # TODO: expand the tests for this function + async def _decode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> NDBuffer: + assert isinstance(chunk_bytes, Buffer) + + raw_bytes = chunk_bytes.as_array_like() + decoded = _vlen_utf8_codec.decode(raw_bytes) + assert decoded.dtype == np.object_ + decoded.shape = chunk_spec.shape + as_string_dtype = decoded.astype(chunk_spec.dtype.to_native_dtype(), copy=False) + return chunk_spec.prototype.nd_buffer.from_numpy_array(as_string_dtype) + + async def _encode_single( + self, + chunk_array: NDBuffer, + chunk_spec: ArraySpec, + ) -> Buffer | None: + assert isinstance(chunk_array, NDBuffer) + return chunk_spec.prototype.buffer.from_bytes( + _vlen_utf8_codec.encode(chunk_array.as_numpy_array()) + ) + + def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: + # what is input_byte_length for an object dtype? + raise NotImplementedError("compute_encoded_size is not implemented for VLen codecs") + + +@dataclass(frozen=True) +class VLenBytesCodec(ArrayBytesCodec): + def __init__(self) -> None: + warn( + "The codec `vlen-bytes` is currently not part in the Zarr format 3 specification. It " + "may not be supported by other zarr implementations and may change in the future.", + category=UserWarning, + stacklevel=2, + ) + super().__init__() + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> Self: + _, configuration_parsed = parse_named_configuration( + data, "vlen-bytes", require_configuration=False + ) + configuration_parsed = configuration_parsed or {} + return cls(**configuration_parsed) + + def to_dict(self) -> dict[str, JSON]: + return {"name": "vlen-bytes", "configuration": {}} + + def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: + return self + + async def _decode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> NDBuffer: + assert isinstance(chunk_bytes, Buffer) + + raw_bytes = chunk_bytes.as_array_like() + decoded = _vlen_bytes_codec.decode(raw_bytes) + assert decoded.dtype == np.object_ + decoded.shape = chunk_spec.shape + return chunk_spec.prototype.nd_buffer.from_numpy_array(decoded) + + async def _encode_single( + self, + chunk_array: NDBuffer, + chunk_spec: ArraySpec, + ) -> Buffer | None: + assert isinstance(chunk_array, NDBuffer) + return chunk_spec.prototype.buffer.from_bytes( + _vlen_bytes_codec.encode(chunk_array.as_numpy_array()) + ) + + def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: + # what is input_byte_length for an object dtype? + raise NotImplementedError("compute_encoded_size is not implemented for VLen codecs") + + +register_codec("vlen-utf8", VLenUTF8Codec) +register_codec("vlen-bytes", VLenBytesCodec) diff --git a/src/zarr/codecs/zstd.py b/src/zarr/codecs/zstd.py new file mode 100644 index 0000000000..b4a4a13c29 --- /dev/null +++ b/src/zarr/codecs/zstd.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from functools import cached_property +from typing import TYPE_CHECKING + +import numcodecs +from numcodecs.zstd import Zstd +from packaging.version import Version + +from zarr.abc.codec import BytesBytesCodec +from zarr.core.buffer.cpu import as_numpy_array_wrapper +from zarr.core.common import JSON, parse_named_configuration +from zarr.registry import register_codec + +if TYPE_CHECKING: + from typing import Self + + from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer + + +def parse_zstd_level(data: JSON) -> int: + if isinstance(data, int): + if data >= 23: + raise ValueError(f"Value must be less than or equal to 22. Got {data} instead.") + return data + raise TypeError(f"Got value with type {type(data)}, but expected an int.") + + +def parse_checksum(data: JSON) -> bool: + if isinstance(data, bool): + return data + raise TypeError(f"Expected bool. Got {type(data)}.") + + +@dataclass(frozen=True) +class ZstdCodec(BytesBytesCodec): + is_fixed_size = True + + level: int = 0 + checksum: bool = False + + def __init__(self, *, level: int = 0, checksum: bool = False) -> None: + # numcodecs 0.13.0 introduces the checksum attribute for the zstd codec + _numcodecs_version = Version(numcodecs.__version__) + if _numcodecs_version < Version("0.13.0"): + raise RuntimeError( + "numcodecs version >= 0.13.0 is required to use the zstd codec. " + f"Version {_numcodecs_version} is currently installed." + ) + + level_parsed = parse_zstd_level(level) + checksum_parsed = parse_checksum(checksum) + + object.__setattr__(self, "level", level_parsed) + object.__setattr__(self, "checksum", checksum_parsed) + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> Self: + _, configuration_parsed = parse_named_configuration(data, "zstd") + return cls(**configuration_parsed) # type: ignore[arg-type] + + def to_dict(self) -> dict[str, JSON]: + return {"name": "zstd", "configuration": {"level": self.level, "checksum": self.checksum}} + + @cached_property + def _zstd_codec(self) -> Zstd: + config_dict = {"level": self.level, "checksum": self.checksum} + return Zstd.from_config(config_dict) + + async def _decode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> Buffer: + return await asyncio.to_thread( + as_numpy_array_wrapper, self._zstd_codec.decode, chunk_bytes, chunk_spec.prototype + ) + + async def _encode_single( + self, + chunk_bytes: Buffer, + chunk_spec: ArraySpec, + ) -> Buffer | None: + return await asyncio.to_thread( + as_numpy_array_wrapper, self._zstd_codec.encode, chunk_bytes, chunk_spec.prototype + ) + + def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) -> int: + raise NotImplementedError + + +register_codec("zstd", ZstdCodec) diff --git a/src/zarr/convenience.py b/src/zarr/convenience.py new file mode 100644 index 0000000000..88f10663b7 --- /dev/null +++ b/src/zarr/convenience.py @@ -0,0 +1,45 @@ +""" +Convenience helpers. + +.. warning:: + + This sub-module is deprecated. All functions here are defined + in the top level zarr namespace instead. +""" + +import warnings + +from zarr.api.synchronous import ( + consolidate_metadata, + copy, + copy_all, + copy_store, + load, + open, + open_consolidated, + save, + save_array, + save_group, + tree, +) + +__all__ = [ + "consolidate_metadata", + "copy", + "copy_all", + "copy_store", + "load", + "open", + "open_consolidated", + "save", + "save_array", + "save_group", + "tree", +] + +warnings.warn( + "zarr.convenience is deprecated. " + "Import these functions from the top level zarr. namespace instead.", + DeprecationWarning, + stacklevel=2, +) diff --git a/src/zarr/core/__init__.py b/src/zarr/core/__init__.py new file mode 100644 index 0000000000..03a108dbbf --- /dev/null +++ b/src/zarr/core/__init__.py @@ -0,0 +1,9 @@ +""" +The ``zarr.core`` module is considered private API and should not be imported +directly by 3rd-party code. +""" + +from __future__ import annotations + +from zarr.core.buffer import Buffer, NDBuffer # noqa: F401 +from zarr.core.codec_pipeline import BatchedCodecPipeline # noqa: F401 diff --git a/src/zarr/core/_info.py b/src/zarr/core/_info.py new file mode 100644 index 0000000000..d57d17f934 --- /dev/null +++ b/src/zarr/core/_info.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import dataclasses +import textwrap +from typing import TYPE_CHECKING, Literal + +if TYPE_CHECKING: + import numcodecs.abc + + from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec + from zarr.core.common import ZarrFormat + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + + +@dataclasses.dataclass(kw_only=True) +class GroupInfo: + """ + Visual summary for a Group. + + Note that this method and its properties is not part of + Zarr's public API. + """ + + _name: str + _type: Literal["Group"] = "Group" + _zarr_format: ZarrFormat + _read_only: bool + _store_type: str + _count_members: int | None = None + _count_arrays: int | None = None + _count_groups: int | None = None + + def __repr__(self) -> str: + template = textwrap.dedent("""\ + Name : {_name} + Type : {_type} + Zarr format : {_zarr_format} + Read-only : {_read_only} + Store type : {_store_type}""") + + if self._count_members is not None: + template += "\nNo. members : {_count_members}" + if self._count_arrays is not None: + template += "\nNo. arrays : {_count_arrays}" + if self._count_groups is not None: + template += "\nNo. groups : {_count_groups}" + return template.format(**dataclasses.asdict(self)) + + +def human_readable_size(size: int) -> str: + if size < 2**10: + return f"{size}" + elif size < 2**20: + return f"{size / float(2**10):.1f}K" + elif size < 2**30: + return f"{size / float(2**20):.1f}M" + elif size < 2**40: + return f"{size / float(2**30):.1f}G" + elif size < 2**50: + return f"{size / float(2**40):.1f}T" + else: + return f"{size / float(2**50):.1f}P" + + +def byte_info(size: int) -> str: + if size < 2**10: + return str(size) + else: + return f"{size} ({human_readable_size(size)})" + + +@dataclasses.dataclass(kw_only=True, frozen=True, slots=True) +class ArrayInfo: + """ + Visual summary for an Array. + + Note that this method and its properties is not part of + Zarr's public API. + """ + + _type: Literal["Array"] = "Array" + _zarr_format: ZarrFormat + _data_type: ZDType[TBaseDType, TBaseScalar] + _fill_value: object + _shape: tuple[int, ...] + _shard_shape: tuple[int, ...] | None = None + _chunk_shape: tuple[int, ...] | None = None + _order: Literal["C", "F"] + _read_only: bool + _store_type: str + _filters: tuple[numcodecs.abc.Codec, ...] | tuple[ArrayArrayCodec, ...] = () + _serializer: ArrayBytesCodec | None = None + _compressors: tuple[numcodecs.abc.Codec, ...] | tuple[BytesBytesCodec, ...] = () + _count_bytes: int | None = None + _count_bytes_stored: int | None = None + _count_chunks_initialized: int | None = None + + def __repr__(self) -> str: + template = textwrap.dedent("""\ + Type : {_type} + Zarr format : {_zarr_format} + Data type : {_data_type} + Fill value : {_fill_value} + Shape : {_shape}""") + + if self._shard_shape is not None: + template += textwrap.dedent(""" + Shard shape : {_shard_shape}""") + + template += textwrap.dedent(""" + Chunk shape : {_chunk_shape} + Order : {_order} + Read-only : {_read_only} + Store type : {_store_type}""") + + # We can't use dataclasses.asdict, because we only want a shallow dict + kwargs = {field.name: getattr(self, field.name) for field in dataclasses.fields(self)} + + if self._chunk_shape is None: + # for non-regular chunk grids + kwargs["chunk_shape"] = "" + + template += "\nFilters : {_filters}" + + if self._serializer is not None: + template += "\nSerializer : {_serializer}" + + template += "\nCompressors : {_compressors}" + + if self._count_bytes is not None: + template += "\nNo. bytes : {_count_bytes}" + kwargs["_count_bytes"] = byte_info(self._count_bytes) + + if self._count_bytes_stored is not None: + template += "\nNo. bytes stored : {_count_bytes_stored}" + kwargs["_count_stored"] = byte_info(self._count_bytes_stored) + + if ( + self._count_bytes is not None + and self._count_bytes_stored is not None + and self._count_bytes_stored > 0 + ): + template += "\nStorage ratio : {_storage_ratio}" + kwargs["_storage_ratio"] = f"{self._count_bytes / self._count_bytes_stored:.1f}" + + if self._count_chunks_initialized is not None: + if self._shard_shape is not None: + template += "\nShards Initialized : {_count_chunks_initialized}" + else: + template += "\nChunks Initialized : {_count_chunks_initialized}" + return template.format(**kwargs) diff --git a/src/zarr/core/_tree.py b/src/zarr/core/_tree.py new file mode 100644 index 0000000000..eed807ec95 --- /dev/null +++ b/src/zarr/core/_tree.py @@ -0,0 +1,65 @@ +import io +import os +from collections.abc import Sequence +from typing import Any + +from zarr.core.group import AsyncGroup + +try: + import rich + import rich.console + import rich.tree +except ImportError as e: + raise ImportError("'rich' is required for Group.tree") from e + + +class TreeRepr: + """ + A simple object with a tree-like repr for the Zarr Group. + + Note that this object and it's implementation isn't considered part + of Zarr's public API. + """ + + def __init__(self, tree: rich.tree.Tree) -> None: + self._tree = tree + + def __repr__(self) -> str: + color_system = os.environ.get("OVERRIDE_COLOR_SYSTEM", rich.get_console().color_system) + console = rich.console.Console(file=io.StringIO(), color_system=color_system) + console.print(self._tree) + return str(console.file.getvalue()) + + def _repr_mimebundle_( + self, + include: Sequence[str], + exclude: Sequence[str], + **kwargs: Any, + ) -> dict[str, str]: + # For jupyter support. + # Unsure why mypy infers the return type to by Any + return self._tree._repr_mimebundle_(include=include, exclude=exclude, **kwargs) # type: ignore[no-any-return] + + +async def group_tree_async(group: AsyncGroup, max_depth: int | None = None) -> TreeRepr: + tree = rich.tree.Tree(label=f"[bold]{group.name}[/bold]") + nodes = {"": tree} + members = sorted([x async for x in group.members(max_depth=max_depth)]) + + for key, node in members: + if key.count("/") == 0: + parent_key = "" + else: + parent_key = key.rsplit("/", 1)[0] + parent = nodes[parent_key] + + # We want what the spec calls the node "name", the part excluding all leading + # /'s and path segments. But node.name includes all that, so we build it here. + name = key.rsplit("/")[-1] + if isinstance(node, AsyncGroup): + label = f"[bold]{name}[/bold]" + else: + label = f"[bold]{name}[/bold] {node.shape} {node.dtype}" + nodes[key] = parent.add(label) + + return TreeRepr(tree) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py new file mode 100644 index 0000000000..cd6b33a28c --- /dev/null +++ b/src/zarr/core/array.py @@ -0,0 +1,4806 @@ +from __future__ import annotations + +import json +import warnings +from asyncio import gather +from collections.abc import Iterable +from dataclasses import dataclass, field, replace +from itertools import starmap +from logging import getLogger +from typing import ( + TYPE_CHECKING, + Any, + Generic, + Literal, + TypeAlias, + TypedDict, + cast, + overload, +) +from warnings import warn + +import numcodecs +import numcodecs.abc +import numpy as np +from typing_extensions import deprecated + +import zarr +from zarr._compat import _deprecate_positional_args +from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec +from zarr.abc.store import Store, set_or_delete +from zarr.codecs._v2 import V2Codec +from zarr.codecs.bytes import BytesCodec +from zarr.core._info import ArrayInfo +from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, parse_array_config +from zarr.core.attributes import Attributes +from zarr.core.buffer import ( + BufferPrototype, + NDArrayLike, + NDArrayLikeOrScalar, + NDBuffer, + default_buffer_prototype, +) +from zarr.core.buffer.cpu import buffer_prototype as cpu_buffer_prototype +from zarr.core.chunk_grids import RegularChunkGrid, _auto_partition, normalize_chunks +from zarr.core.chunk_key_encodings import ( + ChunkKeyEncoding, + ChunkKeyEncodingLike, + DefaultChunkKeyEncoding, + V2ChunkKeyEncoding, +) +from zarr.core.common import ( + JSON, + ZARR_JSON, + ZARRAY_JSON, + ZATTRS_JSON, + ChunkCoords, + DimensionNames, + MemoryOrder, + ShapeLike, + ZarrFormat, + _default_zarr_format, + _warn_order_kwarg, + concurrent_map, + parse_order, + parse_shapelike, + product, +) +from zarr.core.config import categorize_data_type +from zarr.core.config import config as zarr_config +from zarr.core.dtype import ( + ZDType, + ZDTypeLike, + parse_data_type, +) +from zarr.core.dtype.common import HasEndianness, HasItemSize +from zarr.core.indexing import ( + BasicIndexer, + BasicSelection, + BlockIndex, + BlockIndexer, + CoordinateIndexer, + CoordinateSelection, + Fields, + Indexer, + MaskIndexer, + MaskSelection, + OIndex, + OrthogonalIndexer, + OrthogonalSelection, + Selection, + VIndex, + _iter_grid, + ceildiv, + check_fields, + check_no_multi_fields, + is_pure_fancy_indexing, + is_pure_orthogonal_indexing, + is_scalar, + pop_fields, +) +from zarr.core.metadata import ( + ArrayMetadata, + ArrayMetadataDict, + ArrayV2Metadata, + ArrayV2MetadataDict, + ArrayV3Metadata, + ArrayV3MetadataDict, + T_ArrayMetadata, +) +from zarr.core.metadata.v2 import ( + CompressorLikev2, + parse_compressor, + parse_filters, +) +from zarr.core.metadata.v3 import parse_node_type_array +from zarr.core.sync import sync +from zarr.errors import MetadataValidationError +from zarr.registry import ( + _parse_array_array_codec, + _parse_array_bytes_codec, + _parse_bytes_bytes_codec, + get_pipeline_class, +) +from zarr.storage._common import StorePath, ensure_no_existing_node, make_store_path +from zarr.storage._utils import _relativize_path + +if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + from typing import Self + + import numpy.typing as npt + + from zarr.abc.codec import CodecPipeline + from zarr.codecs.sharding import ShardingCodecIndexLocation + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar + from zarr.core.group import AsyncGroup + from zarr.storage import StoreLike + + +# Array and AsyncArray are defined in the base ``zarr`` namespace +__all__ = ["create_codec_pipeline", "parse_array_metadata"] + +logger = getLogger(__name__) + + +def parse_array_metadata(data: Any) -> ArrayMetadata: + if isinstance(data, ArrayMetadata): + return data + elif isinstance(data, dict): + zarr_format = data.get("zarr_format") + if zarr_format == 3: + meta_out = ArrayV3Metadata.from_dict(data) + if len(meta_out.storage_transformers) > 0: + msg = ( + f"Array metadata contains storage transformers: {meta_out.storage_transformers}." + "Arrays with storage transformers are not supported in zarr-python at this time." + ) + raise ValueError(msg) + return meta_out + elif zarr_format == 2: + return ArrayV2Metadata.from_dict(data) + else: + raise ValueError(f"Invalid zarr_format: {zarr_format}. Expected 2 or 3") + raise TypeError # pragma: no cover + + +def create_codec_pipeline(metadata: ArrayMetadata) -> CodecPipeline: + if isinstance(metadata, ArrayV3Metadata): + return get_pipeline_class().from_codecs(metadata.codecs) + elif isinstance(metadata, ArrayV2Metadata): + v2_codec = V2Codec(filters=metadata.filters, compressor=metadata.compressor) + return get_pipeline_class().from_codecs([v2_codec]) + raise TypeError # pragma: no cover + + +async def get_array_metadata( + store_path: StorePath, zarr_format: ZarrFormat | None = 3 +) -> dict[str, JSON]: + if zarr_format == 2: + zarray_bytes, zattrs_bytes = await gather( + (store_path / ZARRAY_JSON).get(prototype=cpu_buffer_prototype), + (store_path / ZATTRS_JSON).get(prototype=cpu_buffer_prototype), + ) + if zarray_bytes is None: + raise FileNotFoundError(store_path) + elif zarr_format == 3: + zarr_json_bytes = await (store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype) + if zarr_json_bytes is None: + raise FileNotFoundError(store_path) + elif zarr_format is None: + zarr_json_bytes, zarray_bytes, zattrs_bytes = await gather( + (store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype), + (store_path / ZARRAY_JSON).get(prototype=cpu_buffer_prototype), + (store_path / ZATTRS_JSON).get(prototype=cpu_buffer_prototype), + ) + if zarr_json_bytes is not None and zarray_bytes is not None: + # warn and favor v3 + msg = f"Both zarr.json (Zarr format 3) and .zarray (Zarr format 2) metadata objects exist at {store_path}. Zarr v3 will be used." + warnings.warn(msg, stacklevel=1) + if zarr_json_bytes is None and zarray_bytes is None: + raise FileNotFoundError(store_path) + # set zarr_format based on which keys were found + if zarr_json_bytes is not None: + zarr_format = 3 + else: + zarr_format = 2 + else: + raise MetadataValidationError("zarr_format", "2, 3, or None", zarr_format) + + metadata_dict: dict[str, JSON] + if zarr_format == 2: + # V2 arrays are comprised of a .zarray and .zattrs objects + assert zarray_bytes is not None + metadata_dict = json.loads(zarray_bytes.to_bytes()) + zattrs_dict = json.loads(zattrs_bytes.to_bytes()) if zattrs_bytes is not None else {} + metadata_dict["attributes"] = zattrs_dict + else: + # V3 arrays are comprised of a zarr.json object + assert zarr_json_bytes is not None + metadata_dict = json.loads(zarr_json_bytes.to_bytes()) + + parse_node_type_array(metadata_dict.get("node_type")) + + return metadata_dict + + +@dataclass(frozen=True) +class AsyncArray(Generic[T_ArrayMetadata]): + """ + An asynchronous array class representing a chunked array stored in a Zarr store. + + Parameters + ---------- + metadata : ArrayMetadata + The metadata of the array. + store_path : StorePath + The path to the Zarr store. + config : ArrayConfigLike, optional + The runtime configuration of the array, by default None. + + Attributes + ---------- + metadata : ArrayMetadata + The metadata of the array. + store_path : StorePath + The path to the Zarr store. + codec_pipeline : CodecPipeline + The codec pipeline used for encoding and decoding chunks. + _config : ArrayConfig + The runtime configuration of the array. + """ + + metadata: T_ArrayMetadata + store_path: StorePath + codec_pipeline: CodecPipeline = field(init=False) + _config: ArrayConfig + + @overload + def __init__( + self: AsyncArray[ArrayV2Metadata], + metadata: ArrayV2Metadata | ArrayV2MetadataDict, + store_path: StorePath, + config: ArrayConfigLike | None = None, + ) -> None: ... + + @overload + def __init__( + self: AsyncArray[ArrayV3Metadata], + metadata: ArrayV3Metadata | ArrayV3MetadataDict, + store_path: StorePath, + config: ArrayConfigLike | None = None, + ) -> None: ... + + def __init__( + self, + metadata: ArrayMetadata | ArrayMetadataDict, + store_path: StorePath, + config: ArrayConfigLike | None = None, + ) -> None: + metadata_parsed = parse_array_metadata(metadata) + config_parsed = parse_array_config(config) + + object.__setattr__(self, "metadata", metadata_parsed) + object.__setattr__(self, "store_path", store_path) + object.__setattr__(self, "_config", config_parsed) + object.__setattr__(self, "codec_pipeline", create_codec_pipeline(metadata=metadata_parsed)) + + # this overload defines the function signature when zarr_format is 2 + @overload + @classmethod + async def create( + cls, + store: StoreLike, + *, + # v2 and v3 + shape: ShapeLike, + dtype: ZDTypeLike, + zarr_format: Literal[2], + fill_value: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunks: ShapeLike | None = None, + dimension_separator: Literal[".", "/"] | None = None, + order: MemoryOrder | None = None, + filters: list[dict[str, JSON]] | None = None, + compressor: CompressorLikev2 | Literal["auto"] = "auto", + # runtime + overwrite: bool = False, + data: npt.ArrayLike | None = None, + config: ArrayConfigLike | None = None, + ) -> AsyncArray[ArrayV2Metadata]: ... + + # this overload defines the function signature when zarr_format is 3 + @overload + @classmethod + async def create( + cls, + store: StoreLike, + *, + # v2 and v3 + shape: ShapeLike, + dtype: ZDTypeLike, + zarr_format: Literal[3], + fill_value: Any | None = None, + attributes: dict[str, JSON] | None = None, + # v3 only + chunk_shape: ShapeLike | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + # runtime + overwrite: bool = False, + data: npt.ArrayLike | None = None, + config: ArrayConfigLike | None = None, + ) -> AsyncArray[ArrayV3Metadata]: ... + + @overload + @classmethod + async def create( + cls, + store: StoreLike, + *, + # v2 and v3 + shape: ShapeLike, + dtype: ZDTypeLike, + zarr_format: Literal[3] = 3, + fill_value: Any | None = None, + attributes: dict[str, JSON] | None = None, + # v3 only + chunk_shape: ShapeLike | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + # runtime + overwrite: bool = False, + data: npt.ArrayLike | None = None, + config: ArrayConfigLike | None = None, + ) -> AsyncArray[ArrayV3Metadata]: ... + + @overload + @classmethod + async def create( + cls, + store: StoreLike, + *, + # v2 and v3 + shape: ShapeLike, + dtype: ZDTypeLike, + zarr_format: ZarrFormat, + fill_value: Any | None = None, + attributes: dict[str, JSON] | None = None, + # v3 only + chunk_shape: ShapeLike | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + # v2 only + chunks: ShapeLike | None = None, + dimension_separator: Literal[".", "/"] | None = None, + order: MemoryOrder | None = None, + filters: list[dict[str, JSON]] | None = None, + compressor: CompressorLike = "auto", + # runtime + overwrite: bool = False, + data: npt.ArrayLike | None = None, + config: ArrayConfigLike | None = None, + ) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]: ... + + @classmethod + @deprecated("Use zarr.api.asynchronous.create_array instead.") + @_deprecate_positional_args + async def create( + cls, + store: StoreLike, + *, + # v2 and v3 + shape: ShapeLike, + dtype: ZDTypeLike, + zarr_format: ZarrFormat = 3, + fill_value: Any | None = None, + attributes: dict[str, JSON] | None = None, + # v3 only + chunk_shape: ShapeLike | None = None, + chunk_key_encoding: ( + ChunkKeyEncodingLike + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + # v2 only + chunks: ShapeLike | None = None, + dimension_separator: Literal[".", "/"] | None = None, + order: MemoryOrder | None = None, + filters: list[dict[str, JSON]] | None = None, + compressor: CompressorLike = "auto", + # runtime + overwrite: bool = False, + data: npt.ArrayLike | None = None, + config: ArrayConfigLike | None = None, + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Method to create a new asynchronous array instance. + + .. deprecated:: 3.0.0 + Deprecated in favor of :func:`zarr.api.asynchronous.create_array`. + + Parameters + ---------- + store : StoreLike + The store where the array will be created. + shape : ShapeLike + The shape of the array. + dtype : ZDTypeLike + The data type of the array. + zarr_format : ZarrFormat, optional + The Zarr format version (default is 3). + fill_value : Any, optional + The fill value of the array (default is None). + attributes : dict[str, JSON], optional + The attributes of the array (default is None). + chunk_shape : ChunkCoords, optional + The shape of the array's chunks + Zarr format 3 only. Zarr format 2 arrays should use `chunks` instead. + If not specified, default are guessed based on the shape and dtype. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + Zarr format 3 only. Zarr format 2 arrays should use `dimension_separator` instead. + Default is ``("default", "/")``. + codecs : Sequence of Codecs or dicts, optional + An iterable of Codec or dict serializations of Codecs. The elements of + this collection specify the transformation from array values to stored bytes. + Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead. + + If no codecs are provided, default codecs will be used: + + - For numeric arrays, the default is ``BytesCodec`` and ``ZstdCodec``. + - For Unicode strings, the default is ``VLenUTF8Codec`` and ``ZstdCodec``. + - For bytes or objects, the default is ``VLenBytesCodec`` and ``ZstdCodec``. + + These defaults can be changed by modifying the value of ``array.v3_default_filters``, + ``array.v3_default_serializer`` and ``array.v3_default_compressors`` in :mod:`zarr.core.config`. + dimension_names : Iterable[str | None], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + chunks : ShapeLike, optional + The shape of the array's chunks. + Zarr format 2 only. Zarr format 3 arrays should use ``chunk_shape`` instead. + If not specified, default are guessed based on the shape and dtype. + dimension_separator : Literal[".", "/"], optional + The dimension separator (default is "."). + Zarr format 2 only. Zarr format 3 arrays should use ``chunk_key_encoding`` instead. + order : Literal["C", "F"], optional + The memory of the array (default is "C"). + If ``zarr_format`` is 2, this parameter sets the memory order of the array. + If `zarr_format`` is 3, then this parameter is deprecated, because memory order + is a runtime parameter for Zarr 3 arrays. The recommended way to specify the memory + order for Zarr 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + filters : list[dict[str, JSON]], optional + Sequence of filters to use to encode chunk data prior to compression. + Zarr format 2 only. Zarr format 3 arrays should use ``codecs`` instead. If no ``filters`` + are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v2_default_filters`` in :mod:`zarr.core.config`. + compressor : dict[str, JSON], optional + The compressor used to compress the data (default is None). + Zarr format 2 only. Zarr format 3 arrays should use ``codecs`` instead. + + If no ``compressor`` is provided, a default compressor will be used: + + - For numeric arrays, the default is ``ZstdCodec``. + - For Unicode strings, the default is ``VLenUTF8Codec``. + - For bytes or objects, the default is ``VLenBytesCodec``. + + These defaults can be changed by modifying the value of ``array.v2_default_compressor`` in :mod:`zarr.core.config`. + overwrite : bool, optional + Whether to raise an error if the store already exists (default is False). + data : npt.ArrayLike, optional + The data to be inserted into the array (default is None). + config : ArrayConfigLike, optional + Runtime configuration for the array. + + Returns + ------- + AsyncArray + The created asynchronous array instance. + """ + return await cls._create( + store, + # v2 and v3 + shape=shape, + dtype=dtype, + zarr_format=zarr_format, + fill_value=fill_value, + attributes=attributes, + # v3 only + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + # v2 only + chunks=chunks, + dimension_separator=dimension_separator, + order=order, + filters=filters, + compressor=compressor, + # runtime + overwrite=overwrite, + data=data, + config=config, + ) + + @classmethod + async def _create( + cls, + store: StoreLike, + *, + # v2 and v3 + shape: ShapeLike, + dtype: ZDTypeLike | ZDType[TBaseDType, TBaseScalar], + zarr_format: ZarrFormat = 3, + fill_value: Any | None = None, + attributes: dict[str, JSON] | None = None, + # v3 only + chunk_shape: ShapeLike | None = None, + chunk_key_encoding: ( + ChunkKeyEncodingLike + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + # v2 only + chunks: ShapeLike | None = None, + dimension_separator: Literal[".", "/"] | None = None, + order: MemoryOrder | None = None, + filters: list[dict[str, JSON]] | None = None, + compressor: CompressorLike = "auto", + # runtime + overwrite: bool = False, + data: npt.ArrayLike | None = None, + config: ArrayConfigLike | None = None, + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Method to create a new asynchronous array instance. + See :func:`AsyncArray.create` for more details. + Deprecated in favor of :func:`zarr.api.asynchronous.create_array`. + """ + + dtype_parsed = parse_data_type(dtype, zarr_format=zarr_format) + store_path = await make_store_path(store) + + shape = parse_shapelike(shape) + + if chunks is not None and chunk_shape is not None: + raise ValueError("Only one of chunk_shape or chunks can be provided.") + item_size = 1 + if isinstance(dtype_parsed, HasItemSize): + item_size = dtype_parsed.item_size + if chunks: + _chunks = normalize_chunks(chunks, shape, item_size) + else: + _chunks = normalize_chunks(chunk_shape, shape, item_size) + config_parsed = parse_array_config(config) + + result: AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] + if zarr_format == 3: + if dimension_separator is not None: + raise ValueError( + "dimension_separator cannot be used for arrays with zarr_format 3. Use chunk_key_encoding instead." + ) + if filters is not None: + raise ValueError( + "filters cannot be used for arrays with zarr_format 3. Use array-to-array codecs instead." + ) + if compressor != "auto": + raise ValueError( + "compressor cannot be used for arrays with zarr_format 3. Use bytes-to-bytes codecs instead." + ) + + if order is not None: + _warn_order_kwarg() + config_parsed = replace(config_parsed, order=order) + + result = await cls._create_v3( + store_path, + shape=shape, + dtype=dtype_parsed, + chunk_shape=_chunks, + fill_value=fill_value, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + attributes=attributes, + overwrite=overwrite, + config=config_parsed, + ) + elif zarr_format == 2: + if codecs is not None: + raise ValueError( + "codecs cannot be used for arrays with zarr_format 2. Use filters and compressor instead." + ) + if chunk_key_encoding is not None: + raise ValueError( + "chunk_key_encoding cannot be used for arrays with zarr_format 2. Use dimension_separator instead." + ) + if dimension_names is not None: + raise ValueError("dimension_names cannot be used for arrays with zarr_format 2.") + + if order is None: + order_parsed = parse_order(zarr_config.get("array.order")) + else: + order_parsed = order + + result = await cls._create_v2( + store_path, + shape=shape, + dtype=dtype_parsed, + chunks=_chunks, + dimension_separator=dimension_separator, + fill_value=fill_value, + order=order_parsed, + config=config_parsed, + filters=filters, + compressor=compressor, + attributes=attributes, + overwrite=overwrite, + ) + else: + raise ValueError(f"Insupported zarr_format. Got: {zarr_format}") + + if data is not None: + # insert user-provided data + await result.setitem(..., data) + + return result + + @staticmethod + def _create_metadata_v3( + shape: ShapeLike, + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_shape: ChunkCoords, + fill_value: Any | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + attributes: dict[str, JSON] | None = None, + ) -> ArrayV3Metadata: + """ + Create an instance of ArrayV3Metadata. + """ + filters: tuple[ArrayArrayCodec, ...] + compressors: tuple[BytesBytesCodec, ...] + + shape = parse_shapelike(shape) + if codecs is None: + filters, serializer, compressors = _get_default_chunk_encoding_v3(dtype) + codecs_parsed = (*filters, serializer, *compressors) + else: + codecs_parsed = tuple(codecs) + + chunk_key_encoding_parsed: ChunkKeyEncodingLike + if chunk_key_encoding is None: + chunk_key_encoding_parsed = {"name": "default", "separator": "/"} + else: + chunk_key_encoding_parsed = chunk_key_encoding + + if fill_value is None: + # v3 spec will not allow a null fill value + fill_value_parsed = dtype.default_scalar() + else: + fill_value_parsed = fill_value + + chunk_grid_parsed = RegularChunkGrid(chunk_shape=chunk_shape) + return ArrayV3Metadata( + shape=shape, + data_type=dtype, + chunk_grid=chunk_grid_parsed, + chunk_key_encoding=chunk_key_encoding_parsed, + fill_value=fill_value_parsed, + codecs=codecs_parsed, # type: ignore[arg-type] + dimension_names=tuple(dimension_names) if dimension_names else None, + attributes=attributes or {}, + ) + + @classmethod + async def _create_v3( + cls, + store_path: StorePath, + *, + shape: ShapeLike, + dtype: ZDType[TBaseDType, TBaseScalar], + chunk_shape: ChunkCoords, + config: ArrayConfig, + fill_value: Any | None = None, + chunk_key_encoding: ( + ChunkKeyEncodingLike + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + attributes: dict[str, JSON] | None = None, + overwrite: bool = False, + ) -> AsyncArray[ArrayV3Metadata]: + if overwrite: + if store_path.store.supports_deletes: + await store_path.delete_dir() + else: + await ensure_no_existing_node(store_path, zarr_format=3) + else: + await ensure_no_existing_node(store_path, zarr_format=3) + + if isinstance(chunk_key_encoding, tuple): + chunk_key_encoding = ( + V2ChunkKeyEncoding(separator=chunk_key_encoding[1]) + if chunk_key_encoding[0] == "v2" + else DefaultChunkKeyEncoding(separator=chunk_key_encoding[1]) + ) + + metadata = cls._create_metadata_v3( + shape=shape, + dtype=dtype, + chunk_shape=chunk_shape, + fill_value=fill_value, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + attributes=attributes, + ) + + array = cls(metadata=metadata, store_path=store_path, config=config) + await array._save_metadata(metadata, ensure_parents=True) + return array + + @staticmethod + def _create_metadata_v2( + shape: ChunkCoords, + dtype: ZDType[TBaseDType, TBaseScalar], + chunks: ChunkCoords, + order: MemoryOrder, + dimension_separator: Literal[".", "/"] | None = None, + fill_value: float | None = None, + filters: Iterable[dict[str, JSON] | numcodecs.abc.Codec] | None = None, + compressor: CompressorLikev2 = None, + attributes: dict[str, JSON] | None = None, + ) -> ArrayV2Metadata: + if dimension_separator is None: + dimension_separator = "." + if fill_value is None: + fill_value = dtype.default_scalar() # type: ignore[assignment] + return ArrayV2Metadata( + shape=shape, + dtype=dtype, + chunks=chunks, + order=order, + dimension_separator=dimension_separator, + fill_value=fill_value, + compressor=compressor, + filters=filters, + attributes=attributes, + ) + + @classmethod + async def _create_v2( + cls, + store_path: StorePath, + *, + shape: ChunkCoords, + dtype: ZDType[TBaseDType, TBaseScalar], + chunks: ChunkCoords, + order: MemoryOrder, + config: ArrayConfig, + dimension_separator: Literal[".", "/"] | None = None, + fill_value: float | None = None, + filters: Iterable[dict[str, JSON] | numcodecs.abc.Codec] | None = None, + compressor: CompressorLike = "auto", + attributes: dict[str, JSON] | None = None, + overwrite: bool = False, + ) -> AsyncArray[ArrayV2Metadata]: + if overwrite: + if store_path.store.supports_deletes: + await store_path.delete_dir() + else: + await ensure_no_existing_node(store_path, zarr_format=2) + else: + await ensure_no_existing_node(store_path, zarr_format=2) + + compressor_parsed: CompressorLikev2 + if compressor == "auto": + _, compressor_parsed = _get_default_chunk_encoding_v2(dtype) + elif isinstance(compressor, BytesBytesCodec): + raise ValueError( + "Cannot use a BytesBytesCodec as a compressor for zarr v2 arrays. " + "Use a numcodecs codec directly instead." + ) + else: + compressor_parsed = compressor + + metadata = cls._create_metadata_v2( + shape=shape, + dtype=dtype, + chunks=chunks, + order=order, + dimension_separator=dimension_separator, + fill_value=fill_value, + filters=filters, + compressor=compressor_parsed, + attributes=attributes, + ) + + array = cls(metadata=metadata, store_path=store_path, config=config) + await array._save_metadata(metadata, ensure_parents=True) + return array + + @classmethod + def from_dict( + cls, + store_path: StorePath, + data: dict[str, JSON], + ) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]: + """ + Create a Zarr array from a dictionary, with support for both Zarr format 2 and 3 metadata. + + Parameters + ---------- + store_path : StorePath + The path within the store where the array should be created. + + data : dict + A dictionary representing the array data. This dictionary should include necessary metadata + for the array, such as shape, dtype, and other attributes. The format of the metadata + will determine whether a Zarr format 2 or 3 array is created. + + Returns + ------- + AsyncArray[ArrayV3Metadata] or AsyncArray[ArrayV2Metadata] + The created Zarr array, either using Zarr format 2 or 3 metadata based on the provided data. + + Raises + ------ + ValueError + If the dictionary data is invalid or incompatible with either Zarr format 2 or 3 array creation. + """ + metadata = parse_array_metadata(data) + return cls(metadata=metadata, store_path=store_path) + + @classmethod + async def open( + cls, + store: StoreLike, + zarr_format: ZarrFormat | None = 3, + ) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]: + """ + Async method to open an existing Zarr array from a given store. + + Parameters + ---------- + store : StoreLike + The store containing the Zarr array. + zarr_format : ZarrFormat | None, optional + The Zarr format version (default is 3). + + Returns + ------- + AsyncArray + The opened Zarr array. + + Examples + -------- + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> async_arr = await AsyncArray.open(store) # doctest: +ELLIPSIS + + """ + store_path = await make_store_path(store) + metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) + # TODO: remove this cast when we have better type hints + _metadata_dict = cast("ArrayV3MetadataDict", metadata_dict) + return cls(store_path=store_path, metadata=_metadata_dict) + + @property + def store(self) -> Store: + return self.store_path.store + + @property + def ndim(self) -> int: + """Returns the number of dimensions in the Array. + + Returns + ------- + int + The number of dimensions in the Array. + """ + return len(self.metadata.shape) + + @property + def shape(self) -> ChunkCoords: + """Returns the shape of the Array. + + Returns + ------- + tuple + The shape of the Array. + """ + return self.metadata.shape + + @property + def chunks(self) -> ChunkCoords: + """Returns the chunk shape of the Array. + If sharding is used the inner chunk shape is returned. + + Only defined for arrays using using `RegularChunkGrid`. + If array doesn't use `RegularChunkGrid`, `NotImplementedError` is raised. + + Returns + ------- + ChunkCoords: + The chunk shape of the Array. + """ + return self.metadata.chunks + + @property + def shards(self) -> ChunkCoords | None: + """Returns the shard shape of the Array. + Returns None if sharding is not used. + + Only defined for arrays using using `RegularChunkGrid`. + If array doesn't use `RegularChunkGrid`, `NotImplementedError` is raised. + + Returns + ------- + ChunkCoords: + The shard shape of the Array. + """ + return self.metadata.shards + + @property + def size(self) -> int: + """Returns the total number of elements in the array + + Returns + ------- + int + Total number of elements in the array + """ + return np.prod(self.metadata.shape).item() + + @property + def filters(self) -> tuple[numcodecs.abc.Codec, ...] | tuple[ArrayArrayCodec, ...]: + """ + Filters that are applied to each chunk of the array, in order, before serializing that + chunk to bytes. + """ + if self.metadata.zarr_format == 2: + filters = self.metadata.filters + if filters is None: + return () + return filters + + return tuple( + codec for codec in self.metadata.inner_codecs if isinstance(codec, ArrayArrayCodec) + ) + + @property + def serializer(self) -> ArrayBytesCodec | None: + """ + Array-to-bytes codec to use for serializing the chunks into bytes. + """ + if self.metadata.zarr_format == 2: + return None + + return next( + codec for codec in self.metadata.inner_codecs if isinstance(codec, ArrayBytesCodec) + ) + + @property + @deprecated("Use AsyncArray.compressors instead.") + def compressor(self) -> numcodecs.abc.Codec | None: + """ + Compressor that is applied to each chunk of the array. + + .. deprecated:: 3.0.0 + `array.compressor` is deprecated and will be removed in a future release. + Use `array.compressors` instead. + """ + if self.metadata.zarr_format == 2: + return self.metadata.compressor + raise TypeError("`compressor` is not available for Zarr format 3 arrays.") + + @property + def compressors(self) -> tuple[numcodecs.abc.Codec, ...] | tuple[BytesBytesCodec, ...]: + """ + Compressors that are applied to each chunk of the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + """ + if self.metadata.zarr_format == 2: + if self.metadata.compressor is not None: + return (self.metadata.compressor,) + return () + + return tuple( + codec for codec in self.metadata.inner_codecs if isinstance(codec, BytesBytesCodec) + ) + + @property + def _zdtype(self) -> ZDType[TBaseDType, TBaseScalar]: + """ + The zarr-specific representation of the array data type + """ + if self.metadata.zarr_format == 2: + return self.metadata.dtype + else: + return self.metadata.data_type + + @property + def dtype(self) -> TBaseDType: + """Returns the data type of the array. + + Returns + ------- + np.dtype + Data type of the array + """ + return self._zdtype.to_native_dtype() + + @property + def order(self) -> MemoryOrder: + """Returns the memory order of the array. + + Returns + ------- + bool + Memory order of the array + """ + if self.metadata.zarr_format == 2: + return self.metadata.order + else: + return self._config.order + + @property + def attrs(self) -> dict[str, JSON]: + """Returns the attributes of the array. + + Returns + ------- + dict + Attributes of the array + """ + return self.metadata.attributes + + @property + def read_only(self) -> bool: + """Returns True if the array is read-only. + + Returns + ------- + bool + True if the array is read-only + """ + # Backwards compatibility for 2.x + return self.store_path.read_only + + @property + def path(self) -> str: + """Storage path. + + Returns + ------- + str + The path to the array in the Zarr store. + """ + return self.store_path.path + + @property + def name(self) -> str: + """Array name following h5py convention. + + Returns + ------- + str + The name of the array. + """ + # follow h5py convention: add leading slash + name = self.path + if not name.startswith("/"): + name = "/" + name + return name + + @property + def basename(self) -> str: + """Final component of name. + + Returns + ------- + str + The basename or final component of the array name. + """ + return self.name.split("/")[-1] + + @property + def cdata_shape(self) -> ChunkCoords: + """ + The shape of the chunk grid for this array. + + Returns + ------- + Tuple[int] + The shape of the chunk grid for this array. + """ + return tuple(starmap(ceildiv, zip(self.shape, self.chunks, strict=False))) + + @property + def nchunks(self) -> int: + """ + The number of chunks in the stored representation of this array. + + Returns + ------- + int + The total number of chunks in the array. + """ + return product(self.cdata_shape) + + async def nchunks_initialized(self) -> int: + """ + Calculate the number of chunks that have been initialized, i.e. the number of chunks that have + been persisted to the storage backend. + + Returns + ------- + nchunks_initialized : int + The number of chunks that have been initialized. + + Notes + ----- + On :class:`AsyncArray` this is an asynchronous method, unlike the (synchronous) + property :attr:`Array.nchunks_initialized`. + + Examples + -------- + >>> arr = await zarr.api.asynchronous.create(shape=(10,), chunks=(2,)) + >>> await arr.nchunks_initialized() + 0 + >>> await arr.setitem(slice(5), 1) + >>> await arr.nchunks_initialized() + 3 + """ + return len(await chunks_initialized(self)) + + async def nbytes_stored(self) -> int: + return await self.store_path.store.getsize_prefix(self.store_path.path) + + def _iter_chunk_coords( + self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None + ) -> Iterator[ChunkCoords]: + """ + Create an iterator over the coordinates of chunks in chunk grid space. If the `origin` + keyword is used, iteration will start at the chunk index specified by `origin`. + The default behavior is to start at the origin of the grid coordinate space. + If the `selection_shape` keyword is used, iteration will be bounded over a contiguous region + ranging from `[origin, origin selection_shape]`, where the upper bound is exclusive as + per python indexing conventions. + + Parameters + ---------- + origin : Sequence[int] | None, default=None + The origin of the selection relative to the array's chunk grid. + selection_shape : Sequence[int] | None, default=None + The shape of the selection in chunk grid coordinates. + + Yields + ------ + chunk_coords: ChunkCoords + The coordinates of each chunk in the selection. + """ + return _iter_grid(self.cdata_shape, origin=origin, selection_shape=selection_shape) + + def _iter_chunk_keys( + self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None + ) -> Iterator[str]: + """ + Iterate over the storage keys of each chunk, relative to an optional origin, and optionally + limited to a contiguous region in chunk grid coordinates. + + Parameters + ---------- + origin : Sequence[int] | None, default=None + The origin of the selection relative to the array's chunk grid. + selection_shape : Sequence[int] | None, default=None + The shape of the selection in chunk grid coordinates. + + Yields + ------ + key: str + The storage key of each chunk in the selection. + """ + # Iterate over the coordinates of chunks in chunk grid space. + for k in self._iter_chunk_coords(origin=origin, selection_shape=selection_shape): + # Encode the chunk key from the chunk coordinates. + yield self.metadata.encode_chunk_key(k) + + def _iter_chunk_regions( + self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None + ) -> Iterator[tuple[slice, ...]]: + """ + Iterate over the regions spanned by each chunk. + + Parameters + ---------- + origin : Sequence[int] | None, default=None + The origin of the selection relative to the array's chunk grid. + selection_shape : Sequence[int] | None, default=None + The shape of the selection in chunk grid coordinates. + + Yields + ------ + region: tuple[slice, ...] + A tuple of slice objects representing the region spanned by each chunk in the selection. + """ + for cgrid_position in self._iter_chunk_coords( + origin=origin, selection_shape=selection_shape + ): + out: tuple[slice, ...] = () + for c_pos, c_shape in zip(cgrid_position, self.chunks, strict=False): + start = c_pos * c_shape + stop = start + c_shape + out += (slice(start, stop, 1),) + yield out + + @property + def nbytes(self) -> int: + """ + The total number of bytes that can be stored in the chunks of this array. + + Notes + ----- + This value is calculated by multiplying the number of elements in the array and the size + of each element, the latter of which is determined by the dtype of the array. + For this reason, ``nbytes`` will likely be inaccurate for arrays with variable-length + dtypes. It is not possible to determine the size of an array with variable-length elements + from the shape and dtype alone. + """ + return self.size * self.dtype.itemsize + + async def _get_selection( + self, + indexer: Indexer, + *, + prototype: BufferPrototype, + out: NDBuffer | None = None, + fields: Fields | None = None, + ) -> NDArrayLikeOrScalar: + # check fields are sensible + out_dtype = check_fields(fields, self.dtype) + + # setup output buffer + if out is not None: + if isinstance(out, NDBuffer): + out_buffer = out + else: + raise TypeError(f"out argument needs to be an NDBuffer. Got {type(out)!r}") + if out_buffer.shape != indexer.shape: + raise ValueError( + f"shape of out argument doesn't match. Expected {indexer.shape}, got {out.shape}" + ) + else: + out_buffer = prototype.nd_buffer.create( + shape=indexer.shape, + dtype=out_dtype, + order=self.order, + fill_value=self.metadata.fill_value, + ) + if product(indexer.shape) > 0: + # need to use the order from the metadata for v2 + _config = self._config + if self.metadata.zarr_format == 2: + _config = replace(_config, order=self.order) + + # reading chunks and decoding them + await self.codec_pipeline.read( + [ + ( + self.store_path / self.metadata.encode_chunk_key(chunk_coords), + self.metadata.get_chunk_spec(chunk_coords, _config, prototype=prototype), + chunk_selection, + out_selection, + is_complete_chunk, + ) + for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer + ], + out_buffer, + drop_axes=indexer.drop_axes, + ) + if isinstance(indexer, BasicIndexer) and indexer.shape == (): + return out_buffer.as_scalar() + return out_buffer.as_ndarray_like() + + async def getitem( + self, + selection: BasicSelection, + *, + prototype: BufferPrototype | None = None, + ) -> NDArrayLikeOrScalar: + """ + Asynchronous function that retrieves a subset of the array's data based on the provided selection. + + Parameters + ---------- + selection : BasicSelection + A selection object specifying the subset of data to retrieve. + prototype : BufferPrototype, optional + A buffer prototype to use for the retrieved data (default is None). + + Returns + ------- + NDArrayLikeOrScalar + The retrieved subset of the array's data. + + Examples + -------- + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> async_arr = await zarr.api.asynchronous.create_array( + ... store=store, + ... shape=(100,100), + ... chunks=(10,10), + ... dtype='i4', + ... fill_value=0) + + >>> await async_arr.getitem((0,1)) # doctest: +ELLIPSIS + array(0, dtype=int32) + + """ + if prototype is None: + prototype = default_buffer_prototype() + indexer = BasicIndexer( + selection, + shape=self.metadata.shape, + chunk_grid=self.metadata.chunk_grid, + ) + return await self._get_selection(indexer, prototype=prototype) + + async def _save_metadata(self, metadata: ArrayMetadata, ensure_parents: bool = False) -> None: + """ + Asynchronously save the array metadata. + """ + to_save = metadata.to_buffer_dict(cpu_buffer_prototype) + awaitables = [set_or_delete(self.store_path / key, value) for key, value in to_save.items()] + + if ensure_parents: + # To enable zarr.create(store, path="a/b/c"), we need to create all the intermediate groups. + parents = _build_parents(self) + + for parent in parents: + awaitables.extend( + [ + (parent.store_path / key).set_if_not_exists(value) + for key, value in parent.metadata.to_buffer_dict( + cpu_buffer_prototype + ).items() + ] + ) + + await gather(*awaitables) + + async def _set_selection( + self, + indexer: Indexer, + value: npt.ArrayLike, + *, + prototype: BufferPrototype, + fields: Fields | None = None, + ) -> None: + # check fields are sensible + check_fields(fields, self.dtype) + fields = check_no_multi_fields(fields) + + # check value shape + if np.isscalar(value): + array_like = prototype.buffer.create_zero_length().as_array_like() + if isinstance(array_like, np._typing._SupportsArrayFunc): + # TODO: need to handle array types that don't support __array_function__ + # like PyTorch and JAX + array_like_ = cast("np._typing._SupportsArrayFunc", array_like) + value = np.asanyarray(value, dtype=self.dtype, like=array_like_) + else: + if not hasattr(value, "shape"): + value = np.asarray(value, self.dtype) + # assert ( + # value.shape == indexer.shape + # ), f"shape of value doesn't match indexer shape. Expected {indexer.shape}, got {value.shape}" + if not hasattr(value, "dtype") or value.dtype.name != self.dtype.name: + if hasattr(value, "astype"): + # Handle things that are already NDArrayLike more efficiently + value = value.astype(dtype=self.dtype, order="A") + else: + value = np.array(value, dtype=self.dtype, order="A") + value = cast("NDArrayLike", value) + + # We accept any ndarray like object from the user and convert it + # to a NDBuffer (or subclass). From this point onwards, we only pass + # Buffer and NDBuffer between components. + value_buffer = prototype.nd_buffer.from_ndarray_like(value) + + # need to use the order from the metadata for v2 + _config = self._config + if self.metadata.zarr_format == 2: + _config = replace(_config, order=self.metadata.order) + + # merging with existing data and encoding chunks + await self.codec_pipeline.write( + [ + ( + self.store_path / self.metadata.encode_chunk_key(chunk_coords), + self.metadata.get_chunk_spec(chunk_coords, _config, prototype), + chunk_selection, + out_selection, + is_complete_chunk, + ) + for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer + ], + value_buffer, + drop_axes=indexer.drop_axes, + ) + + async def setitem( + self, + selection: BasicSelection, + value: npt.ArrayLike, + prototype: BufferPrototype | None = None, + ) -> None: + """ + Asynchronously set values in the array using basic indexing. + + Parameters + ---------- + selection : BasicSelection + The selection defining the region of the array to set. + + value : numpy.typing.ArrayLike + The values to be written into the selected region of the array. + + prototype : BufferPrototype or None, optional + A prototype buffer that defines the structure and properties of the array chunks being modified. + If None, the default buffer prototype is used. Default is None. + + Returns + ------- + None + This method does not return any value. + + Raises + ------ + IndexError + If the selection is out of bounds for the array. + + ValueError + If the values are not compatible with the array's dtype or shape. + + Notes + ----- + - This method is asynchronous and should be awaited. + - Supports basic indexing, where the selection is contiguous and does not involve advanced indexing. + """ + if prototype is None: + prototype = default_buffer_prototype() + indexer = BasicIndexer( + selection, + shape=self.metadata.shape, + chunk_grid=self.metadata.chunk_grid, + ) + return await self._set_selection(indexer, value, prototype=prototype) + + async def resize(self, new_shape: ShapeLike, delete_outside_chunks: bool = True) -> None: + """ + Asynchronously resize the array to a new shape. + + Parameters + ---------- + new_shape : ChunkCoords + The desired new shape of the array. + + delete_outside_chunks : bool, optional + If True (default), chunks that fall outside the new shape will be deleted. If False, + the data in those chunks will be preserved. + + Returns + ------- + AsyncArray + The resized array. + + Raises + ------ + ValueError + If the new shape is incompatible with the current array's chunking configuration. + + Notes + ----- + - This method is asynchronous and should be awaited. + """ + new_shape = parse_shapelike(new_shape) + assert len(new_shape) == len(self.metadata.shape) + new_metadata = self.metadata.update_shape(new_shape) + + if delete_outside_chunks: + # Remove all chunks outside of the new shape + old_chunk_coords = set(self.metadata.chunk_grid.all_chunk_coords(self.metadata.shape)) + new_chunk_coords = set(self.metadata.chunk_grid.all_chunk_coords(new_shape)) + + async def _delete_key(key: str) -> None: + await (self.store_path / key).delete() + + await concurrent_map( + [ + (self.metadata.encode_chunk_key(chunk_coords),) + for chunk_coords in old_chunk_coords.difference(new_chunk_coords) + ], + _delete_key, + zarr_config.get("async.concurrency"), + ) + + # Write new metadata + await self._save_metadata(new_metadata) + + # Update metadata (in place) + object.__setattr__(self, "metadata", new_metadata) + + async def append(self, data: npt.ArrayLike, axis: int = 0) -> ChunkCoords: + """Append `data` to `axis`. + + Parameters + ---------- + data : array-like + Data to be appended. + axis : int + Axis along which to append. + + Returns + ------- + new_shape : tuple + + Notes + ----- + The size of all dimensions other than `axis` must match between this + array and `data`. + """ + # ensure data is array-like + if not hasattr(data, "shape"): + data = np.asanyarray(data) + + self_shape_preserved = tuple(s for i, s in enumerate(self.shape) if i != axis) + data_shape_preserved = tuple(s for i, s in enumerate(data.shape) if i != axis) + if self_shape_preserved != data_shape_preserved: + raise ValueError( + f"shape of data to append is not compatible with the array. " + f"The shape of the data is ({data_shape_preserved})" + f"and the shape of the array is ({self_shape_preserved})." + "All dimensions must match except for the dimension being " + "appended." + ) + # remember old shape + old_shape = self.shape + + # determine new shape + new_shape = tuple( + self.shape[i] if i != axis else self.shape[i] + data.shape[i] + for i in range(len(self.shape)) + ) + + # resize + await self.resize(new_shape) + + # store data + append_selection = tuple( + slice(None) if i != axis else slice(old_shape[i], new_shape[i]) + for i in range(len(self.shape)) + ) + await self.setitem(append_selection, data) + + return new_shape + + async def update_attributes(self, new_attributes: dict[str, JSON]) -> Self: + """ + Asynchronously update the array's attributes. + + Parameters + ---------- + new_attributes : dict of str to JSON + A dictionary of new attributes to update or add to the array. The keys represent attribute + names, and the values must be JSON-compatible. + + Returns + ------- + AsyncArray + The array with the updated attributes. + + Raises + ------ + ValueError + If the attributes are invalid or incompatible with the array's metadata. + + Notes + ----- + - This method is asynchronous and should be awaited. + - The updated attributes will be merged with existing attributes, and any conflicts will be + overwritten by the new values. + """ + self.metadata.attributes.update(new_attributes) + + # Write new metadata + await self._save_metadata(self.metadata) + + return self + + def __repr__(self) -> str: + return f"" + + @property + def info(self) -> Any: + """ + Return the statically known information for an array. + + Returns + ------- + ArrayInfo + + See Also + -------- + AsyncArray.info_complete + All information about a group, including dynamic information + like the number of bytes and chunks written. + + Examples + -------- + + >>> arr = await zarr.api.asynchronous.create( + ... path="array", shape=(3, 4, 5), chunks=(2, 2, 2)) + ... ) + >>> arr.info + Type : Array + Zarr format : 3 + Data type : DataType.float64 + Shape : (3, 4, 5) + Chunk shape : (2, 2, 2) + Order : C + Read-only : False + Store type : MemoryStore + Codecs : [{'endian': }] + No. bytes : 480 + """ + return self._info() + + async def info_complete(self) -> Any: + """ + Return all the information for an array, including dynamic information like a storage size. + + In addition to the static information, this provides + + - The count of chunks initialized + - The sum of the bytes written + + Returns + ------- + ArrayInfo + + See Also + -------- + AsyncArray.info + A property giving just the statically known information about an array. + """ + return self._info( + await self.nchunks_initialized(), + await self.store_path.store.getsize_prefix(self.store_path.path), + ) + + def _info( + self, count_chunks_initialized: int | None = None, count_bytes_stored: int | None = None + ) -> Any: + return ArrayInfo( + _zarr_format=self.metadata.zarr_format, + _data_type=self._zdtype, + _fill_value=self.metadata.fill_value, + _shape=self.shape, + _order=self.order, + _shard_shape=self.shards, + _chunk_shape=self.chunks, + _read_only=self.read_only, + _compressors=self.compressors, + _filters=self.filters, + _serializer=self.serializer, + _store_type=type(self.store_path.store).__name__, + _count_bytes=self.nbytes, + _count_bytes_stored=count_bytes_stored, + _count_chunks_initialized=count_chunks_initialized, + ) + + +# TODO: Array can be a frozen data class again once property setters (e.g. shape) are removed +@dataclass(frozen=False) +class Array: + """ + A Zarr array. + """ + + _async_array: AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] + + @classmethod + @deprecated("Use zarr.create_array instead.") + @_deprecate_positional_args + def create( + cls, + store: StoreLike, + *, + # v2 and v3 + shape: ChunkCoords, + dtype: ZDTypeLike, + zarr_format: ZarrFormat = 3, + fill_value: Any | None = None, + attributes: dict[str, JSON] | None = None, + # v3 only + chunk_shape: ChunkCoords | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + # v2 only + chunks: ChunkCoords | None = None, + dimension_separator: Literal[".", "/"] | None = None, + order: MemoryOrder | None = None, + filters: list[dict[str, JSON]] | None = None, + compressor: CompressorLike = "auto", + # runtime + overwrite: bool = False, + config: ArrayConfigLike | None = None, + ) -> Array: + """Creates a new Array instance from an initialized store. + + .. deprecated:: 3.0.0 + Deprecated in favor of :func:`zarr.create_array`. + + Parameters + ---------- + store : StoreLike + The array store that has already been initialized. + shape : ChunkCoords + The shape of the array. + dtype : ZDTypeLike + The data type of the array. + chunk_shape : ChunkCoords, optional + The shape of the Array's chunks. + Zarr format 3 only. Zarr format 2 arrays should use `chunks` instead. + If not specified, default are guessed based on the shape and dtype. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + Zarr format 3 only. Zarr format 2 arrays should use `dimension_separator` instead. + Default is ``("default", "/")``. + codecs : Sequence of Codecs or dicts, optional + An iterable of Codec or dict serializations of Codecs. The elements of + this collection specify the transformation from array values to stored bytes. + Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead. + + If no codecs are provided, default codecs will be used: + + - For numeric arrays, the default is ``BytesCodec`` and ``ZstdCodec``. + - For Unicode strings, the default is ``VLenUTF8Codec`` and ``ZstdCodec``. + - For bytes or objects, the default is ``VLenBytesCodec`` and ``ZstdCodec``. + + These defaults can be changed by modifying the value of ``array.v3_default_filters``, + ``array.v3_default_serializer`` and ``array.v3_default_compressors`` in :mod:`zarr.core.config`. + dimension_names : Iterable[str | None], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + chunks : ChunkCoords, optional + The shape of the array's chunks. + Zarr format 2 only. Zarr format 3 arrays should use ``chunk_shape`` instead. + If not specified, default are guessed based on the shape and dtype. + dimension_separator : Literal[".", "/"], optional + The dimension separator (default is "."). + Zarr format 2 only. Zarr format 3 arrays should use ``chunk_key_encoding`` instead. + order : Literal["C", "F"], optional + The memory of the array (default is "C"). + If ``zarr_format`` is 2, this parameter sets the memory order of the array. + If `zarr_format`` is 3, then this parameter is deprecated, because memory order + is a runtime parameter for Zarr 3 arrays. The recommended way to specify the memory + order for Zarr 3 arrays is via the ``config`` parameter, e.g. ``{'order': 'C'}``. + filters : list[dict[str, JSON]], optional + Sequence of filters to use to encode chunk data prior to compression. + Zarr format 2 only. Zarr format 3 arrays should use ``codecs`` instead. If no ``filters`` + are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v2_default_filters`` in :mod:`zarr.core.config`. + compressor : dict[str, JSON], optional + Primary compressor to compress chunk data. + Zarr format 2 only. Zarr format 3 arrays should use ``codecs`` instead. + + If no ``compressor`` is provided, a default compressor will be used: + + - For numeric arrays, the default is ``ZstdCodec``. + - For Unicode strings, the default is ``VLenUTF8Codec``. + - For bytes or objects, the default is ``VLenBytesCodec``. + + These defaults can be changed by modifying the value of ``array.v2_default_compressor`` in :mod:`zarr.core.config`. + overwrite : bool, optional + Whether to raise an error if the store already exists (default is False). + + Returns + ------- + Array + Array created from the store. + """ + return cls._create( + store, + # v2 and v3 + shape=shape, + dtype=dtype, + zarr_format=zarr_format, + attributes=attributes, + fill_value=fill_value, + # v3 only + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + # v2 only + chunks=chunks, + dimension_separator=dimension_separator, + order=order, + filters=filters, + compressor=compressor, + # runtime + overwrite=overwrite, + config=config, + ) + + @classmethod + def _create( + cls, + store: StoreLike, + *, + # v2 and v3 + shape: ChunkCoords, + dtype: ZDTypeLike, + zarr_format: ZarrFormat = 3, + fill_value: Any | None = None, + attributes: dict[str, JSON] | None = None, + # v3 only + chunk_shape: ChunkCoords | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNames = None, + # v2 only + chunks: ChunkCoords | None = None, + dimension_separator: Literal[".", "/"] | None = None, + order: MemoryOrder | None = None, + filters: list[dict[str, JSON]] | None = None, + compressor: CompressorLike = "auto", + # runtime + overwrite: bool = False, + config: ArrayConfigLike | None = None, + ) -> Array: + """Creates a new Array instance from an initialized store. + See :func:`Array.create` for more details. + Deprecated in favor of :func:`zarr.create_array`. + """ + async_array = sync( + AsyncArray._create( + store=store, + shape=shape, + dtype=dtype, + zarr_format=zarr_format, + attributes=attributes, + fill_value=fill_value, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + chunks=chunks, + dimension_separator=dimension_separator, + order=order, + filters=filters, + compressor=compressor, + overwrite=overwrite, + config=config, + ), + ) + return cls(async_array) + + @classmethod + def from_dict( + cls, + store_path: StorePath, + data: dict[str, JSON], + ) -> Array: + """ + Create a Zarr array from a dictionary. + + Parameters + ---------- + store_path : StorePath + The path within the store where the array should be created. + + data : dict + A dictionary representing the array data. This dictionary should include necessary metadata + for the array, such as shape, dtype, fill value, and attributes. + + Returns + ------- + Array + The created Zarr array. + + Raises + ------ + ValueError + If the dictionary data is invalid or missing required fields for array creation. + """ + async_array = AsyncArray.from_dict(store_path=store_path, data=data) + return cls(async_array) + + @classmethod + def open( + cls, + store: StoreLike, + ) -> Array: + """Opens an existing Array from a store. + + Parameters + ---------- + store : Store + Store containing the Array. + + Returns + ------- + Array + Array opened from the store. + """ + async_array = sync(AsyncArray.open(store)) + return cls(async_array) + + @property + def store(self) -> Store: + return self._async_array.store + + @property + def ndim(self) -> int: + """Returns the number of dimensions in the array. + + Returns + ------- + int + The number of dimensions in the array. + """ + return self._async_array.ndim + + @property + def shape(self) -> ChunkCoords: + """Returns the shape of the array. + + Returns + ------- + ChunkCoords + The shape of the array. + """ + return self._async_array.shape + + @shape.setter + def shape(self, value: ChunkCoords) -> None: + """Sets the shape of the array by calling resize.""" + self.resize(value) + + @property + def chunks(self) -> ChunkCoords: + """Returns a tuple of integers describing the length of each dimension of a chunk of the array. + If sharding is used the inner chunk shape is returned. + + Only defined for arrays using using `RegularChunkGrid`. + If array doesn't use `RegularChunkGrid`, `NotImplementedError` is raised. + + Returns + ------- + tuple + A tuple of integers representing the length of each dimension of a chunk. + """ + return self._async_array.chunks + + @property + def shards(self) -> ChunkCoords | None: + """Returns a tuple of integers describing the length of each dimension of a shard of the array. + Returns None if sharding is not used. + + Only defined for arrays using using `RegularChunkGrid`. + If array doesn't use `RegularChunkGrid`, `NotImplementedError` is raised. + + Returns + ------- + tuple | None + A tuple of integers representing the length of each dimension of a shard or None if sharding is not used. + """ + return self._async_array.shards + + @property + def size(self) -> int: + """Returns the total number of elements in the array. + + Returns + ------- + int + Total number of elements in the array. + """ + return self._async_array.size + + @property + def dtype(self) -> np.dtype[Any]: + """Returns the NumPy data type. + + Returns + ------- + np.dtype + The NumPy data type. + """ + return self._async_array.dtype + + @property + def attrs(self) -> Attributes: + """Returns a MutableMapping containing user-defined attributes. + + Returns + ------- + attrs : MutableMapping + A MutableMapping object containing user-defined attributes. + + Notes + ----- + Note that attribute values must be JSON serializable. + """ + return Attributes(self) + + @property + def path(self) -> str: + """Storage path.""" + return self._async_array.path + + @property + def name(self) -> str: + """Array name following h5py convention.""" + return self._async_array.name + + @property + def basename(self) -> str: + """Final component of name.""" + return self._async_array.basename + + @property + def metadata(self) -> ArrayMetadata: + return self._async_array.metadata + + @property + def store_path(self) -> StorePath: + return self._async_array.store_path + + @property + def order(self) -> MemoryOrder: + return self._async_array.order + + @property + def read_only(self) -> bool: + return self._async_array.read_only + + @property + def fill_value(self) -> Any: + return self.metadata.fill_value + + @property + def filters(self) -> tuple[numcodecs.abc.Codec, ...] | tuple[ArrayArrayCodec, ...]: + """ + Filters that are applied to each chunk of the array, in order, before serializing that + chunk to bytes. + """ + return self._async_array.filters + + @property + def serializer(self) -> None | ArrayBytesCodec: + """ + Array-to-bytes codec to use for serializing the chunks into bytes. + """ + return self._async_array.serializer + + @property + @deprecated("Use Array.compressors instead.") + def compressor(self) -> numcodecs.abc.Codec | None: + """ + Compressor that is applied to each chunk of the array. + + .. deprecated:: 3.0.0 + `array.compressor` is deprecated and will be removed in a future release. + Use `array.compressors` instead. + """ + return self._async_array.compressor + + @property + def compressors(self) -> tuple[numcodecs.abc.Codec, ...] | tuple[BytesBytesCodec, ...]: + """ + Compressors that are applied to each chunk of the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + """ + return self._async_array.compressors + + @property + def cdata_shape(self) -> ChunkCoords: + """ + The shape of the chunk grid for this array. + """ + return tuple(starmap(ceildiv, zip(self.shape, self.chunks, strict=False))) + + @property + def nchunks(self) -> int: + """ + The number of chunks in the stored representation of this array. + """ + return self._async_array.nchunks + + def _iter_chunk_coords( + self, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None + ) -> Iterator[ChunkCoords]: + """ + Create an iterator over the coordinates of chunks in chunk grid space. If the `origin` + keyword is used, iteration will start at the chunk index specified by `origin`. + The default behavior is to start at the origin of the grid coordinate space. + If the `selection_shape` keyword is used, iteration will be bounded over a contiguous region + ranging from `[origin, origin + selection_shape]`, where the upper bound is exclusive as + per python indexing conventions. + + Parameters + ---------- + origin : Sequence[int] | None, default=None + The origin of the selection relative to the array's chunk grid. + selection_shape : Sequence[int] | None, default=None + The shape of the selection in chunk grid coordinates. + + Yields + ------ + chunk_coords: ChunkCoords + The coordinates of each chunk in the selection. + """ + yield from self._async_array._iter_chunk_coords( + origin=origin, selection_shape=selection_shape + ) + + @property + def nbytes(self) -> int: + """ + The total number of bytes that can be stored in the chunks of this array. + + Notes + ----- + This value is calculated by multiplying the number of elements in the array and the size + of each element, the latter of which is determined by the dtype of the array. + For this reason, ``nbytes`` will likely be inaccurate for arrays with variable-length + dtypes. It is not possible to determine the size of an array with variable-length elements + from the shape and dtype alone. + """ + return self._async_array.nbytes + + @property + def nchunks_initialized(self) -> int: + """ + Calculate the number of chunks that have been initialized, i.e. the number of chunks that have + been persisted to the storage backend. + + Returns + ------- + nchunks_initialized : int + The number of chunks that have been initialized. + + Notes + ----- + On :class:`Array` this is a (synchronous) property, unlike asynchronous function + :meth:`AsyncArray.nchunks_initialized`. + + Examples + -------- + >>> arr = await zarr.create(shape=(10,), chunks=(2,)) + >>> arr.nchunks_initialized + 0 + >>> arr[:5] = 1 + >>> arr.nchunks_initialized + 3 + """ + return sync(self._async_array.nchunks_initialized()) + + def nbytes_stored(self) -> int: + """ + Determine the size, in bytes, of the array actually written to the store. + + Returns + ------- + size : int + """ + return sync(self._async_array.nbytes_stored()) + + def _iter_chunk_keys( + self, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None + ) -> Iterator[str]: + """ + Iterate over the storage keys of each chunk, relative to an optional origin, and optionally + limited to a contiguous region in chunk grid coordinates. + + Parameters + ---------- + origin : Sequence[int] | None, default=None + The origin of the selection relative to the array's chunk grid. + selection_shape : Sequence[int] | None, default=None + The shape of the selection in chunk grid coordinates. + + Yields + ------ + key: str + The storage key of each chunk in the selection. + """ + yield from self._async_array._iter_chunk_keys( + origin=origin, selection_shape=selection_shape + ) + + def _iter_chunk_regions( + self, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None + ) -> Iterator[tuple[slice, ...]]: + """ + Iterate over the regions spanned by each chunk. + + Parameters + ---------- + origin : Sequence[int] | None, default=None + The origin of the selection relative to the array's chunk grid. + selection_shape : Sequence[int] | None, default=None + The shape of the selection in chunk grid coordinates. + + Yields + ------ + region: tuple[slice, ...] + A tuple of slice objects representing the region spanned by each chunk in the selection. + """ + yield from self._async_array._iter_chunk_regions( + origin=origin, selection_shape=selection_shape + ) + + def __array__( + self, dtype: npt.DTypeLike | None = None, copy: bool | None = None + ) -> NDArrayLike: + """ + This method is used by numpy when converting zarr.Array into a numpy array. + For more information, see https://numpy.org/devdocs/user/basics.interoperability.html#the-array-method + """ + if copy is False: + msg = "`copy=False` is not supported. This method always creates a copy." + raise ValueError(msg) + + arr = self[...] + arr_np: NDArrayLike = np.array(arr, dtype=dtype) + + if dtype is not None: + arr_np = arr_np.astype(dtype) + + return arr_np + + def __getitem__(self, selection: Selection) -> NDArrayLikeOrScalar: + """Retrieve data for an item or region of the array. + + Parameters + ---------- + selection : tuple + An integer index or slice or tuple of int/slice objects specifying the + requested item or region for each dimension of the array. + + Returns + ------- + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested region. + + Examples + -------- + Setup a 1-dimensional array:: + + >>> import zarr + >>> import numpy as np + >>> data = np.arange(100, dtype="uint16") + >>> z = zarr.create_array( + >>> StorePath(MemoryStore(mode="w")), + >>> shape=data.shape, + >>> chunks=(10,), + >>> dtype=data.dtype, + >>> ) + >>> z[:] = data + + Retrieve a single item:: + + >>> z[5] + 5 + + Retrieve a region via slicing:: + + >>> z[:5] + array([0, 1, 2, 3, 4]) + >>> z[-5:] + array([95, 96, 97, 98, 99]) + >>> z[5:10] + array([5, 6, 7, 8, 9]) + >>> z[5:10:2] + array([5, 7, 9]) + >>> z[::2] + array([ 0, 2, 4, ..., 94, 96, 98]) + + Load the entire array into memory:: + + >>> z[...] + array([ 0, 1, 2, ..., 97, 98, 99]) + + Setup a 2-dimensional array:: + + >>> data = np.arange(100, dtype="uint16").reshape(10, 10) + >>> z = zarr.create_array( + >>> StorePath(MemoryStore(mode="w")), + >>> shape=data.shape, + >>> chunks=(10, 10), + >>> dtype=data.dtype, + >>> ) + >>> z[:] = data + + Retrieve an item:: + + >>> z[2, 2] + 22 + + Retrieve a region via slicing:: + + >>> z[1:3, 1:3] + array([[11, 12], + [21, 22]]) + >>> z[1:3, :] + array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]]) + >>> z[:, 1:3] + array([[ 1, 2], + [11, 12], + [21, 22], + [31, 32], + [41, 42], + [51, 52], + [61, 62], + [71, 72], + [81, 82], + [91, 92]]) + >>> z[0:5:2, 0:5:2] + array([[ 0, 2, 4], + [20, 22, 24], + [40, 42, 44]]) + >>> z[::2, ::2] + array([[ 0, 2, 4, 6, 8], + [20, 22, 24, 26, 28], + [40, 42, 44, 46, 48], + [60, 62, 64, 66, 68], + [80, 82, 84, 86, 88]]) + + Load the entire array into memory:: + + >>> z[...] + array([[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], + [30, 31, 32, 33, 34, 35, 36, 37, 38, 39], + [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], + [50, 51, 52, 53, 54, 55, 56, 57, 58, 59], + [60, 61, 62, 63, 64, 65, 66, 67, 68, 69], + [70, 71, 72, 73, 74, 75, 76, 77, 78, 79], + [80, 81, 82, 83, 84, 85, 86, 87, 88, 89], + [90, 91, 92, 93, 94, 95, 96, 97, 98, 99]]) + + Notes + ----- + Slices with step > 1 are supported, but slices with negative step are not. + + For arrays with a structured dtype, see Zarr format 2 for examples of how to use + fields + + Currently the implementation for __getitem__ is provided by + :func:`vindex` if the indexing is pure fancy indexing (ie a + broadcast-compatible tuple of integer array indices), or by + :func:`set_basic_selection` otherwise. + + Effectively, this means that the following indexing modes are supported: + + - integer indexing + - slice indexing + - mixed slice and integer indexing + - boolean indexing + - fancy indexing (vectorized list of integers) + + For specific indexing options including outer indexing, see the + methods listed under See Also. + + See Also + -------- + get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, + get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, + set_orthogonal_selection, get_block_selection, set_block_selection, + vindex, oindex, blocks, __setitem__ + + """ + fields, pure_selection = pop_fields(selection) + if is_pure_fancy_indexing(pure_selection, self.ndim): + return self.vindex[cast("CoordinateSelection | MaskSelection", selection)] + elif is_pure_orthogonal_indexing(pure_selection, self.ndim): + return self.get_orthogonal_selection(pure_selection, fields=fields) + else: + return self.get_basic_selection(cast("BasicSelection", pure_selection), fields=fields) + + def __setitem__(self, selection: Selection, value: npt.ArrayLike) -> None: + """Modify data for an item or region of the array. + + Parameters + ---------- + selection : tuple + An integer index or slice or tuple of int/slice specifying the requested + region for each dimension of the array. + value : npt.ArrayLike + An array-like containing the data to be stored in the selection. + + Examples + -------- + Setup a 1-dimensional array:: + + >>> import zarr + >>> z = zarr.zeros( + >>> shape=(100,), + >>> store=StorePath(MemoryStore(mode="w")), + >>> chunk_shape=(5,), + >>> dtype="i4", + >>> ) + + Set all array elements to the same scalar value:: + + >>> z[...] = 42 + >>> z[...] + array([42, 42, 42, ..., 42, 42, 42]) + + Set a portion of the array:: + + >>> z[:10] = np.arange(10) + >>> z[-10:] = np.arange(10)[::-1] + >>> z[...] + array([ 0, 1, 2, ..., 2, 1, 0]) + + Setup a 2-dimensional array:: + + >>> z = zarr.zeros( + >>> shape=(5, 5), + >>> store=StorePath(MemoryStore(mode="w")), + >>> chunk_shape=(5, 5), + >>> dtype="i4", + >>> ) + + Set all array elements to the same scalar value:: + + >>> z[...] = 42 + + Set a portion of the array:: + + >>> z[0, :] = np.arange(z.shape[1]) + >>> z[:, 0] = np.arange(z.shape[0]) + >>> z[...] + array([[ 0, 1, 2, 3, 4], + [ 1, 42, 42, 42, 42], + [ 2, 42, 42, 42, 42], + [ 3, 42, 42, 42, 42], + [ 4, 42, 42, 42, 42]]) + + Notes + ----- + Slices with step > 1 are supported, but slices with negative step are not. + + For arrays with a structured dtype, see Zarr format 2 for examples of how to use + fields + + Currently the implementation for __setitem__ is provided by + :func:`vindex` if the indexing is pure fancy indexing (ie a + broadcast-compatible tuple of integer array indices), or by + :func:`set_basic_selection` otherwise. + + Effectively, this means that the following indexing modes are supported: + + - integer indexing + - slice indexing + - mixed slice and integer indexing + - boolean indexing + - fancy indexing (vectorized list of integers) + + For specific indexing options including outer indexing, see the + methods listed under See Also. + + See Also + -------- + get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, + get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, + set_orthogonal_selection, get_block_selection, set_block_selection, + vindex, oindex, blocks, __getitem__ + + """ + fields, pure_selection = pop_fields(selection) + if is_pure_fancy_indexing(pure_selection, self.ndim): + self.vindex[cast("CoordinateSelection | MaskSelection", selection)] = value + elif is_pure_orthogonal_indexing(pure_selection, self.ndim): + self.set_orthogonal_selection(pure_selection, value, fields=fields) + else: + self.set_basic_selection(cast("BasicSelection", pure_selection), value, fields=fields) + + @_deprecate_positional_args + def get_basic_selection( + self, + selection: BasicSelection = Ellipsis, + *, + out: NDBuffer | None = None, + prototype: BufferPrototype | None = None, + fields: Fields | None = None, + ) -> NDArrayLikeOrScalar: + """Retrieve data for an item or region of the array. + + Parameters + ---------- + selection : tuple + A tuple specifying the requested item or region for each dimension of the + array. May be any combination of int and/or slice or ellipsis for multidimensional arrays. + out : NDBuffer, optional + If given, load the selected data directly into this buffer. + prototype : BufferPrototype, optional + The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used. + fields : str or sequence of str, optional + For arrays with a structured dtype, one or more fields can be specified to + extract data for. + + Returns + ------- + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested region. + + Examples + -------- + Setup a 1-dimensional array:: + + >>> import zarr + >>> import numpy as np + >>> data = np.arange(100, dtype="uint16") + >>> z = zarr.create_array( + >>> StorePath(MemoryStore(mode="w")), + >>> shape=data.shape, + >>> chunks=(3,), + >>> dtype=data.dtype, + >>> ) + >>> z[:] = data + + Retrieve a single item:: + + >>> z.get_basic_selection(5) + 5 + + Retrieve a region via slicing:: + + >>> z.get_basic_selection(slice(5)) + array([0, 1, 2, 3, 4]) + >>> z.get_basic_selection(slice(-5, None)) + array([95, 96, 97, 98, 99]) + >>> z.get_basic_selection(slice(5, 10)) + array([5, 6, 7, 8, 9]) + >>> z.get_basic_selection(slice(5, 10, 2)) + array([5, 7, 9]) + >>> z.get_basic_selection(slice(None, None, 2)) + array([ 0, 2, 4, ..., 94, 96, 98]) + + Setup a 3-dimensional array:: + + >>> data = np.arange(1000).reshape(10, 10, 10) + >>> z = zarr.create_array( + >>> StorePath(MemoryStore(mode="w")), + >>> shape=data.shape, + >>> chunks=(5, 5, 5), + >>> dtype=data.dtype, + >>> ) + >>> z[:] = data + + Retrieve an item:: + + >>> z.get_basic_selection((1, 2, 3)) + 123 + + Retrieve a region via slicing and Ellipsis:: + + >>> z.get_basic_selection((slice(1, 3), slice(1, 3), 0)) + array([[110, 120], + [210, 220]]) + >>> z.get_basic_selection(0, (slice(1, 3), slice(None))) + array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]]) + >>> z.get_basic_selection((..., 5)) + array([[ 2 12 22 32 42 52 62 72 82 92] + [102 112 122 132 142 152 162 172 182 192] + ... + [802 812 822 832 842 852 862 872 882 892] + [902 912 922 932 942 952 962 972 982 992]] + + Notes + ----- + Slices with step > 1 are supported, but slices with negative step are not. + + For arrays with a structured dtype, see Zarr format 2 for examples of how to use + the `fields` parameter. + + This method provides the implementation for accessing data via the + square bracket notation (__getitem__). See :func:`__getitem__` for examples + using the alternative notation. + + See Also + -------- + set_basic_selection, get_mask_selection, set_mask_selection, + get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, + set_orthogonal_selection, get_block_selection, set_block_selection, + vindex, oindex, blocks, __getitem__, __setitem__ + + """ + + if prototype is None: + prototype = default_buffer_prototype() + return sync( + self._async_array._get_selection( + BasicIndexer(selection, self.shape, self.metadata.chunk_grid), + out=out, + fields=fields, + prototype=prototype, + ) + ) + + @_deprecate_positional_args + def set_basic_selection( + self, + selection: BasicSelection, + value: npt.ArrayLike, + *, + fields: Fields | None = None, + prototype: BufferPrototype | None = None, + ) -> None: + """Modify data for an item or region of the array. + + Parameters + ---------- + selection : tuple + A tuple specifying the requested item or region for each dimension of the + array. May be any combination of int and/or slice or ellipsis for multidimensional arrays. + value : npt.ArrayLike + An array-like containing values to be stored into the array. + fields : str or sequence of str, optional + For arrays with a structured dtype, one or more fields can be specified to set + data for. + prototype : BufferPrototype, optional + The prototype of the buffer used for setting the data. If not provided, the + default buffer prototype is used. + + Examples + -------- + Setup a 1-dimensional array:: + + >>> import zarr + >>> z = zarr.zeros( + >>> shape=(100,), + >>> store=StorePath(MemoryStore(mode="w")), + >>> chunk_shape=(100,), + >>> dtype="i4", + >>> ) + + Set all array elements to the same scalar value:: + + >>> z.set_basic_selection(..., 42) + >>> z[...] + array([42, 42, 42, ..., 42, 42, 42]) + + Set a portion of the array:: + + >>> z.set_basic_selection(slice(10), np.arange(10)) + >>> z.set_basic_selection(slice(-10, None), np.arange(10)[::-1]) + >>> z[...] + array([ 0, 1, 2, ..., 2, 1, 0]) + + Setup a 2-dimensional array:: + + >>> z = zarr.zeros( + >>> shape=(5, 5), + >>> store=StorePath(MemoryStore(mode="w")), + >>> chunk_shape=(5, 5), + >>> dtype="i4", + >>> ) + + Set all array elements to the same scalar value:: + + >>> z.set_basic_selection(..., 42) + + Set a portion of the array:: + + >>> z.set_basic_selection((0, slice(None)), np.arange(z.shape[1])) + >>> z.set_basic_selection((slice(None), 0), np.arange(z.shape[0])) + >>> z[...] + array([[ 0, 1, 2, 3, 4], + [ 1, 42, 42, 42, 42], + [ 2, 42, 42, 42, 42], + [ 3, 42, 42, 42, 42], + [ 4, 42, 42, 42, 42]]) + + Notes + ----- + For arrays with a structured dtype, see Zarr format 2 for examples of how to use + the `fields` parameter. + + This method provides the underlying implementation for modifying data via square + bracket notation, see :func:`__setitem__` for equivalent examples using the + alternative notation. + + See Also + -------- + get_basic_selection, get_mask_selection, set_mask_selection, + get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, + set_orthogonal_selection, get_block_selection, set_block_selection, + vindex, oindex, blocks, __getitem__, __setitem__ + + """ + if prototype is None: + prototype = default_buffer_prototype() + indexer = BasicIndexer(selection, self.shape, self.metadata.chunk_grid) + sync(self._async_array._set_selection(indexer, value, fields=fields, prototype=prototype)) + + @_deprecate_positional_args + def get_orthogonal_selection( + self, + selection: OrthogonalSelection, + *, + out: NDBuffer | None = None, + fields: Fields | None = None, + prototype: BufferPrototype | None = None, + ) -> NDArrayLikeOrScalar: + """Retrieve data by making a selection for each dimension of the array. For + example, if an array has 2 dimensions, allows selecting specific rows and/or + columns. The selection for each dimension can be either an integer (indexing a + single item), a slice, an array of integers, or a Boolean array where True + values indicate a selection. + + Parameters + ---------- + selection : tuple + A selection for each dimension of the array. May be any combination of int, + slice, integer array or Boolean array. + out : NDBuffer, optional + If given, load the selected data directly into this buffer. + fields : str or sequence of str, optional + For arrays with a structured dtype, one or more fields can be specified to + extract data for. + prototype : BufferPrototype, optional + The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used. + + Returns + ------- + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested selection. + + Examples + -------- + Setup a 2-dimensional array:: + + >>> import zarr + >>> import numpy as np + >>> data = np.arange(100).reshape(10, 10) + >>> z = zarr.create_array( + >>> StorePath(MemoryStore(mode="w")), + >>> shape=data.shape, + >>> chunks=data.shape, + >>> dtype=data.dtype, + >>> ) + >>> z[:] = data + + Retrieve rows and columns via any combination of int, slice, integer array and/or + Boolean array:: + + >>> z.get_orthogonal_selection(([1, 4], slice(None))) + array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + [40, 41, 42, 43, 44, 45, 46, 47, 48, 49]]) + >>> z.get_orthogonal_selection((slice(None), [1, 4])) + array([[ 1, 4], + [11, 14], + [21, 24], + [31, 34], + [41, 44], + [51, 54], + [61, 64], + [71, 74], + [81, 84], + [91, 94]]) + >>> z.get_orthogonal_selection(([1, 4], [1, 4])) + array([[11, 14], + [41, 44]]) + >>> sel = np.zeros(z.shape[0], dtype=bool) + >>> sel[1] = True + >>> sel[4] = True + >>> z.get_orthogonal_selection((sel, sel)) + array([[11, 14], + [41, 44]]) + + For convenience, the orthogonal selection functionality is also available via the + `oindex` property, e.g.:: + + >>> z.oindex[[1, 4], :] + array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + [40, 41, 42, 43, 44, 45, 46, 47, 48, 49]]) + >>> z.oindex[:, [1, 4]] + array([[ 1, 4], + [11, 14], + [21, 24], + [31, 34], + [41, 44], + [51, 54], + [61, 64], + [71, 74], + [81, 84], + [91, 94]]) + >>> z.oindex[[1, 4], [1, 4]] + array([[11, 14], + [41, 44]]) + >>> sel = np.zeros(z.shape[0], dtype=bool) + >>> sel[1] = True + >>> sel[4] = True + >>> z.oindex[sel, sel] + array([[11, 14], + [41, 44]]) + + Notes + ----- + Orthogonal indexing is also known as outer indexing. + + Slices with step > 1 are supported, but slices with negative step are not. + + See Also + -------- + get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, + get_coordinate_selection, set_coordinate_selection, set_orthogonal_selection, + get_block_selection, set_block_selection, + vindex, oindex, blocks, __getitem__, __setitem__ + + """ + if prototype is None: + prototype = default_buffer_prototype() + indexer = OrthogonalIndexer(selection, self.shape, self.metadata.chunk_grid) + return sync( + self._async_array._get_selection( + indexer=indexer, out=out, fields=fields, prototype=prototype + ) + ) + + @_deprecate_positional_args + def set_orthogonal_selection( + self, + selection: OrthogonalSelection, + value: npt.ArrayLike, + *, + fields: Fields | None = None, + prototype: BufferPrototype | None = None, + ) -> None: + """Modify data via a selection for each dimension of the array. + + Parameters + ---------- + selection : tuple + A selection for each dimension of the array. May be any combination of int, + slice, integer array or Boolean array. + value : npt.ArrayLike + An array-like array containing the data to be stored in the array. + fields : str or sequence of str, optional + For arrays with a structured dtype, one or more fields can be specified to set + data for. + prototype : BufferPrototype, optional + The prototype of the buffer used for setting the data. If not provided, the + default buffer prototype is used. + + Examples + -------- + Setup a 2-dimensional array:: + + >>> import zarr + >>> z = zarr.zeros( + >>> shape=(5, 5), + >>> store=StorePath(MemoryStore(mode="w")), + >>> chunk_shape=(5, 5), + >>> dtype="i4", + >>> ) + + + Set data for a selection of rows:: + + >>> z.set_orthogonal_selection(([1, 4], slice(None)), 1) + >>> z[...] + array([[0, 0, 0, 0, 0], + [1, 1, 1, 1, 1], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [1, 1, 1, 1, 1]]) + + Set data for a selection of columns:: + + >>> z.set_orthogonal_selection((slice(None), [1, 4]), 2) + >>> z[...] + array([[0, 2, 0, 0, 2], + [1, 2, 1, 1, 2], + [0, 2, 0, 0, 2], + [0, 2, 0, 0, 2], + [1, 2, 1, 1, 2]]) + + Set data for a selection of rows and columns:: + + >>> z.set_orthogonal_selection(([1, 4], [1, 4]), 3) + >>> z[...] + array([[0, 2, 0, 0, 2], + [1, 3, 1, 1, 3], + [0, 2, 0, 0, 2], + [0, 2, 0, 0, 2], + [1, 3, 1, 1, 3]]) + + Set data from a 2D array:: + + >>> values = np.arange(10).reshape(2, 5) + >>> z.set_orthogonal_selection(([0, 3], ...), values) + >>> z[...] + array([[0, 1, 2, 3, 4], + [1, 3, 1, 1, 3], + [0, 2, 0, 0, 2], + [5, 6, 7, 8, 9], + [1, 3, 1, 1, 3]]) + + For convenience, this functionality is also available via the `oindex` property. + E.g.:: + + >>> z.oindex[[1, 4], [1, 4]] = 4 + >>> z[...] + array([[0, 1, 2, 3, 4], + [1, 4, 1, 1, 4], + [0, 2, 0, 0, 2], + [5, 6, 7, 8, 9], + [1, 4, 1, 1, 4]]) + + Notes + ----- + Orthogonal indexing is also known as outer indexing. + + Slices with step > 1 are supported, but slices with negative step are not. + + See Also + -------- + get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, + get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, + get_block_selection, set_block_selection, + vindex, oindex, blocks, __getitem__, __setitem__ + + """ + if prototype is None: + prototype = default_buffer_prototype() + indexer = OrthogonalIndexer(selection, self.shape, self.metadata.chunk_grid) + return sync( + self._async_array._set_selection(indexer, value, fields=fields, prototype=prototype) + ) + + @_deprecate_positional_args + def get_mask_selection( + self, + mask: MaskSelection, + *, + out: NDBuffer | None = None, + fields: Fields | None = None, + prototype: BufferPrototype | None = None, + ) -> NDArrayLikeOrScalar: + """Retrieve a selection of individual items, by providing a Boolean array of the + same shape as the array against which the selection is being made, where True + values indicate a selected item. + + Parameters + ---------- + mask : ndarray, bool + A Boolean array of the same shape as the array against which the selection is + being made. + out : NDBuffer, optional + If given, load the selected data directly into this buffer. + fields : str or sequence of str, optional + For arrays with a structured dtype, one or more fields can be specified to + extract data for. + prototype : BufferPrototype, optional + The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used. + + Returns + ------- + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested selection. + + Examples + -------- + Setup a 2-dimensional array:: + + >>> import zarr + >>> import numpy as np + >>> data = np.arange(100).reshape(10, 10) + >>> z = zarr.create_array( + >>> StorePath(MemoryStore(mode="w")), + >>> shape=data.shape, + >>> chunks=data.shape, + >>> dtype=data.dtype, + >>> ) + >>> z[:] = data + + Retrieve items by specifying a mask:: + + >>> sel = np.zeros_like(z, dtype=bool) + >>> sel[1, 1] = True + >>> sel[4, 4] = True + >>> z.get_mask_selection(sel) + array([11, 44]) + + For convenience, the mask selection functionality is also available via the + `vindex` property, e.g.:: + + >>> z.vindex[sel] + array([11, 44]) + + Notes + ----- + Mask indexing is a form of vectorized or inner indexing, and is equivalent to + coordinate indexing. Internally the mask array is converted to coordinate + arrays by calling `np.nonzero`. + + See Also + -------- + get_basic_selection, set_basic_selection, set_mask_selection, + get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, + set_coordinate_selection, get_block_selection, set_block_selection, + vindex, oindex, blocks, __getitem__, __setitem__ + """ + + if prototype is None: + prototype = default_buffer_prototype() + indexer = MaskIndexer(mask, self.shape, self.metadata.chunk_grid) + return sync( + self._async_array._get_selection( + indexer=indexer, out=out, fields=fields, prototype=prototype + ) + ) + + @_deprecate_positional_args + def set_mask_selection( + self, + mask: MaskSelection, + value: npt.ArrayLike, + *, + fields: Fields | None = None, + prototype: BufferPrototype | None = None, + ) -> None: + """Modify a selection of individual items, by providing a Boolean array of the + same shape as the array against which the selection is being made, where True + values indicate a selected item. + + Parameters + ---------- + mask : ndarray, bool + A Boolean array of the same shape as the array against which the selection is + being made. + value : npt.ArrayLike + An array-like containing values to be stored into the array. + fields : str or sequence of str, optional + For arrays with a structured dtype, one or more fields can be specified to set + data for. + + Examples + -------- + Setup a 2-dimensional array:: + + >>> import zarr + >>> z = zarr.zeros( + >>> shape=(5, 5), + >>> store=StorePath(MemoryStore(mode="w")), + >>> chunk_shape=(5, 5), + >>> dtype="i4", + >>> ) + + Set data for a selection of items:: + + >>> sel = np.zeros_like(z, dtype=bool) + >>> sel[1, 1] = True + >>> sel[4, 4] = True + >>> z.set_mask_selection(sel, 1) + >>> z[...] + array([[0, 0, 0, 0, 0], + [0, 1, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 1]]) + + For convenience, this functionality is also available via the `vindex` property. + E.g.:: + + >>> z.vindex[sel] = 2 + >>> z[...] + array([[0, 0, 0, 0, 0], + [0, 2, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 2]]) + + Notes + ----- + Mask indexing is a form of vectorized or inner indexing, and is equivalent to + coordinate indexing. Internally the mask array is converted to coordinate + arrays by calling `np.nonzero`. + + See Also + -------- + get_basic_selection, set_basic_selection, get_mask_selection, + get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, + set_coordinate_selection, get_block_selection, set_block_selection, + vindex, oindex, blocks, __getitem__, __setitem__ + + """ + if prototype is None: + prototype = default_buffer_prototype() + indexer = MaskIndexer(mask, self.shape, self.metadata.chunk_grid) + sync(self._async_array._set_selection(indexer, value, fields=fields, prototype=prototype)) + + @_deprecate_positional_args + def get_coordinate_selection( + self, + selection: CoordinateSelection, + *, + out: NDBuffer | None = None, + fields: Fields | None = None, + prototype: BufferPrototype | None = None, + ) -> NDArrayLikeOrScalar: + """Retrieve a selection of individual items, by providing the indices + (coordinates) for each selected item. + + Parameters + ---------- + selection : tuple + An integer (coordinate) array for each dimension of the array. + out : NDBuffer, optional + If given, load the selected data directly into this buffer. + fields : str or sequence of str, optional + For arrays with a structured dtype, one or more fields can be specified to + extract data for. + prototype : BufferPrototype, optional + The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used. + + Returns + ------- + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested coordinate selection. + + Examples + -------- + Setup a 2-dimensional array:: + + >>> import zarr + >>> import numpy as np + >>> data = np.arange(0, 100, dtype="uint16").reshape((10, 10)) + >>> z = zarr.create_array( + >>> StorePath(MemoryStore(mode="w")), + >>> shape=data.shape, + >>> chunks=(3, 3), + >>> dtype=data.dtype, + >>> ) + >>> z[:] = data + + Retrieve items by specifying their coordinates:: + + >>> z.get_coordinate_selection(([1, 4], [1, 4])) + array([11, 44]) + + For convenience, the coordinate selection functionality is also available via the + `vindex` property, e.g.:: + + >>> z.vindex[[1, 4], [1, 4]] + array([11, 44]) + + Notes + ----- + Coordinate indexing is also known as point selection, and is a form of vectorized + or inner indexing. + + Slices are not supported. Coordinate arrays must be provided for all dimensions + of the array. + + Coordinate arrays may be multidimensional, in which case the output array will + also be multidimensional. Coordinate arrays are broadcast against each other + before being applied. The shape of the output will be the same as the shape of + each coordinate array after broadcasting. + + See Also + -------- + get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, + get_orthogonal_selection, set_orthogonal_selection, set_coordinate_selection, + get_block_selection, set_block_selection, + vindex, oindex, blocks, __getitem__, __setitem__ + + """ + if prototype is None: + prototype = default_buffer_prototype() + indexer = CoordinateIndexer(selection, self.shape, self.metadata.chunk_grid) + out_array = sync( + self._async_array._get_selection( + indexer=indexer, out=out, fields=fields, prototype=prototype + ) + ) + + if hasattr(out_array, "shape"): + # restore shape + out_array = np.array(out_array).reshape(indexer.sel_shape) + return out_array + + @_deprecate_positional_args + def set_coordinate_selection( + self, + selection: CoordinateSelection, + value: npt.ArrayLike, + *, + fields: Fields | None = None, + prototype: BufferPrototype | None = None, + ) -> None: + """Modify a selection of individual items, by providing the indices (coordinates) + for each item to be modified. + + Parameters + ---------- + selection : tuple + An integer (coordinate) array for each dimension of the array. + value : npt.ArrayLike + An array-like containing values to be stored into the array. + fields : str or sequence of str, optional + For arrays with a structured dtype, one or more fields can be specified to set + data for. + + Examples + -------- + Setup a 2-dimensional array:: + + >>> import zarr + >>> z = zarr.zeros( + >>> shape=(5, 5), + >>> store=StorePath(MemoryStore(mode="w")), + >>> chunk_shape=(5, 5), + >>> dtype="i4", + >>> ) + + Set data for a selection of items:: + + >>> z.set_coordinate_selection(([1, 4], [1, 4]), 1) + >>> z[...] + array([[0, 0, 0, 0, 0], + [0, 1, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 1]]) + + For convenience, this functionality is also available via the `vindex` property. + E.g.:: + + >>> z.vindex[[1, 4], [1, 4]] = 2 + >>> z[...] + array([[0, 0, 0, 0, 0], + [0, 2, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 2]]) + + Notes + ----- + Coordinate indexing is also known as point selection, and is a form of vectorized + or inner indexing. + + Slices are not supported. Coordinate arrays must be provided for all dimensions + of the array. + + See Also + -------- + get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, + get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, + get_block_selection, set_block_selection, + vindex, oindex, blocks, __getitem__, __setitem__ + + """ + if prototype is None: + prototype = default_buffer_prototype() + # setup indexer + indexer = CoordinateIndexer(selection, self.shape, self.metadata.chunk_grid) + + # handle value - need ndarray-like flatten value + if not is_scalar(value, self.dtype): + try: + from numcodecs.compat import ensure_ndarray_like + + value = ensure_ndarray_like(value) # TODO replace with agnostic + except TypeError: + # Handle types like `list` or `tuple` + value = np.array(value) # TODO replace with agnostic + if hasattr(value, "shape") and len(value.shape) > 1: + value = np.array(value).reshape(-1) + + if not is_scalar(value, self.dtype) and ( + isinstance(value, NDArrayLike) and indexer.shape != value.shape + ): + raise ValueError( + f"Attempting to set a selection of {indexer.sel_shape[0]} " + f"elements with an array of {value.shape[0]} elements." + ) + + sync(self._async_array._set_selection(indexer, value, fields=fields, prototype=prototype)) + + @_deprecate_positional_args + def get_block_selection( + self, + selection: BasicSelection, + *, + out: NDBuffer | None = None, + fields: Fields | None = None, + prototype: BufferPrototype | None = None, + ) -> NDArrayLikeOrScalar: + """Retrieve a selection of individual items, by providing the indices + (coordinates) for each selected item. + + Parameters + ---------- + selection : int or slice or tuple of int or slice + An integer (coordinate) or slice for each dimension of the array. + out : NDBuffer, optional + If given, load the selected data directly into this buffer. + fields : str or sequence of str, optional + For arrays with a structured dtype, one or more fields can be specified to + extract data for. + prototype : BufferPrototype, optional + The prototype of the buffer to use for the output data. If not provided, the default buffer prototype is used. + + Returns + ------- + NDArrayLikeOrScalar + An array-like or scalar containing the data for the requested block selection. + + Examples + -------- + Setup a 2-dimensional array:: + + >>> import zarr + >>> import numpy as np + >>> data = np.arange(0, 100, dtype="uint16").reshape((10, 10)) + >>> z = zarr.create_array( + >>> StorePath(MemoryStore(mode="w")), + >>> shape=data.shape, + >>> chunks=(3, 3), + >>> dtype=data.dtype, + >>> ) + >>> z[:] = data + + Retrieve items by specifying their block coordinates:: + + >>> z.get_block_selection((1, slice(None))) + array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], + [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], + [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]]) + + Which is equivalent to:: + + >>> z[3:6, :] + array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], + [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], + [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]]) + + For convenience, the block selection functionality is also available via the + `blocks` property, e.g.:: + + >>> z.blocks[1] + array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], + [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], + [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]]) + + Notes + ----- + Block indexing is a convenience indexing method to work on individual chunks + with chunk index slicing. It has the same concept as Dask's `Array.blocks` + indexing. + + Slices are supported. However, only with a step size of one. + + Block index arrays may be multidimensional to index multidimensional arrays. + For example:: + + >>> z.blocks[0, 1:3] + array([[ 3, 4, 5, 6, 7, 8], + [13, 14, 15, 16, 17, 18], + [23, 24, 25, 26, 27, 28]]) + + See Also + -------- + get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, + get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, + set_coordinate_selection, set_block_selection, + vindex, oindex, blocks, __getitem__, __setitem__ + + """ + if prototype is None: + prototype = default_buffer_prototype() + indexer = BlockIndexer(selection, self.shape, self.metadata.chunk_grid) + return sync( + self._async_array._get_selection( + indexer=indexer, out=out, fields=fields, prototype=prototype + ) + ) + + @_deprecate_positional_args + def set_block_selection( + self, + selection: BasicSelection, + value: npt.ArrayLike, + *, + fields: Fields | None = None, + prototype: BufferPrototype | None = None, + ) -> None: + """Modify a selection of individual blocks, by providing the chunk indices + (coordinates) for each block to be modified. + + Parameters + ---------- + selection : tuple + An integer (coordinate) or slice for each dimension of the array. + value : npt.ArrayLike + An array-like containing the data to be stored in the block selection. + fields : str or sequence of str, optional + For arrays with a structured dtype, one or more fields can be specified to set + data for. + prototype : BufferPrototype, optional + The prototype of the buffer used for setting the data. If not provided, the + default buffer prototype is used. + + Examples + -------- + Set up a 2-dimensional array:: + + >>> import zarr + >>> z = zarr.zeros( + >>> shape=(6, 6), + >>> store=StorePath(MemoryStore(mode="w")), + >>> chunk_shape=(2, 2), + >>> dtype="i4", + >>> ) + + Set data for a selection of items:: + + >>> z.set_block_selection((1, 0), 1) + >>> z[...] + array([[0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0]]) + + For convenience, this functionality is also available via the `blocks` property. + E.g.:: + + >>> z.blocks[2, 1] = 4 + >>> z[...] + array([[0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [0, 0, 4, 4, 0, 0], + [0, 0, 4, 4, 0, 0]]) + + >>> z.blocks[:, 2] = 7 + >>> z[...] + array([[0, 0, 0, 0, 7, 7], + [0, 0, 0, 0, 7, 7], + [1, 1, 0, 0, 7, 7], + [1, 1, 0, 0, 7, 7], + [0, 0, 4, 4, 7, 7], + [0, 0, 4, 4, 7, 7]]) + + Notes + ----- + Block indexing is a convenience indexing method to work on individual chunks + with chunk index slicing. It has the same concept as Dask's `Array.blocks` + indexing. + + Slices are supported. However, only with a step size of one. + + See Also + -------- + get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, + get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, + get_block_selection, set_block_selection, + vindex, oindex, blocks, __getitem__, __setitem__ + + """ + if prototype is None: + prototype = default_buffer_prototype() + indexer = BlockIndexer(selection, self.shape, self.metadata.chunk_grid) + sync(self._async_array._set_selection(indexer, value, fields=fields, prototype=prototype)) + + @property + def vindex(self) -> VIndex: + """Shortcut for vectorized (inner) indexing, see :func:`get_coordinate_selection`, + :func:`set_coordinate_selection`, :func:`get_mask_selection` and + :func:`set_mask_selection` for documentation and examples.""" + return VIndex(self) + + @property + def oindex(self) -> OIndex: + """Shortcut for orthogonal (outer) indexing, see :func:`get_orthogonal_selection` and + :func:`set_orthogonal_selection` for documentation and examples.""" + return OIndex(self) + + @property + def blocks(self) -> BlockIndex: + """Shortcut for blocked chunked indexing, see :func:`get_block_selection` and + :func:`set_block_selection` for documentation and examples.""" + return BlockIndex(self) + + def resize(self, new_shape: ShapeLike) -> None: + """ + Change the shape of the array by growing or shrinking one or more + dimensions. + + Parameters + ---------- + new_shape : tuple + New shape of the array. + + Notes + ----- + If one or more dimensions are shrunk, any chunks falling outside the + new array shape will be deleted from the underlying store. + However, it is noteworthy that the chunks partially falling inside the new array + (i.e. boundary chunks) will remain intact, and therefore, + the data falling outside the new array but inside the boundary chunks + would be restored by a subsequent resize operation that grows the array size. + + Examples + -------- + >>> import zarr + >>> z = zarr.zeros(shape=(10000, 10000), + >>> chunk_shape=(1000, 1000), + >>> dtype="i4",) + >>> z.shape + (10000, 10000) + >>> z = z.resize(20000, 1000) + >>> z.shape + (20000, 1000) + >>> z2 = z.resize(50, 50) + >>> z.shape + (20000, 1000) + >>> z2.shape + (50, 50) + """ + sync(self._async_array.resize(new_shape)) + + def append(self, data: npt.ArrayLike, axis: int = 0) -> ChunkCoords: + """Append `data` to `axis`. + + Parameters + ---------- + data : array-like + Data to be appended. + axis : int + Axis along which to append. + + Returns + ------- + new_shape : tuple + + Notes + ----- + The size of all dimensions other than `axis` must match between this + array and `data`. + + Examples + -------- + >>> import numpy as np + >>> import zarr + >>> a = np.arange(10000000, dtype='i4').reshape(10000, 1000) + >>> z = zarr.array(a, chunks=(1000, 100)) + >>> z.shape + (10000, 1000) + >>> z.append(a) + (20000, 1000) + >>> z.append(np.vstack([a, a]), axis=1) + (20000, 2000) + >>> z.shape + (20000, 2000) + """ + return sync(self._async_array.append(data, axis=axis)) + + def update_attributes(self, new_attributes: dict[str, JSON]) -> Array: + """ + Update the array's attributes. + + Parameters + ---------- + new_attributes : dict + A dictionary of new attributes to update or add to the array. The keys represent attribute + names, and the values must be JSON-compatible. + + Returns + ------- + Array + The array with the updated attributes. + + Raises + ------ + ValueError + If the attributes are invalid or incompatible with the array's metadata. + + Notes + ----- + - The updated attributes will be merged with existing attributes, and any conflicts will be + overwritten by the new values. + """ + # TODO: remove this cast when type inference improves + new_array = sync(self._async_array.update_attributes(new_attributes)) + # TODO: remove this cast when type inference improves + _new_array = cast("AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]", new_array) + return type(self)(_new_array) + + def __repr__(self) -> str: + return f"" + + @property + def info(self) -> Any: + """ + Return the statically known information for an array. + + Returns + ------- + ArrayInfo + + See Also + -------- + Array.info_complete + All information about a group, including dynamic information + like the number of bytes and chunks written. + + Examples + -------- + >>> arr = zarr.create(shape=(10,), chunks=(2,), dtype="float32") + >>> arr.info + Type : Array + Zarr format : 3 + Data type : DataType.float32 + Shape : (10,) + Chunk shape : (2,) + Order : C + Read-only : False + Store type : MemoryStore + Codecs : [BytesCodec(endian=)] + No. bytes : 40 + """ + return self._async_array.info + + def info_complete(self) -> Any: + """ + Returns all the information about an array, including information from the Store. + + In addition to the statically known information like ``name`` and ``zarr_format``, + this includes additional information like the size of the array in bytes and + the number of chunks written. + + Note that this method will need to read metadata from the store. + + Returns + ------- + ArrayInfo + + See Also + -------- + Array.info + The statically known subset of metadata about an array. + """ + return sync(self._async_array.info_complete()) + + +async def chunks_initialized( + array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], +) -> tuple[str, ...]: + """ + Return the keys of the chunks that have been persisted to the storage backend. + + Parameters + ---------- + array : AsyncArray + The array to inspect. + + Returns + ------- + chunks_initialized : tuple[str, ...] + The keys of the chunks that have been initialized. + + See Also + -------- + nchunks_initialized + + """ + store_contents = [ + x async for x in array.store_path.store.list_prefix(prefix=array.store_path.path) + ] + store_contents_relative = [ + _relativize_path(path=key, prefix=array.store_path.path) for key in store_contents + ] + return tuple( + chunk_key for chunk_key in array._iter_chunk_keys() if chunk_key in store_contents_relative + ) + + +def _build_parents( + node: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup, +) -> list[AsyncGroup]: + from zarr.core.group import AsyncGroup, GroupMetadata + + store = node.store_path.store + path = node.store_path.path + if not path: + return [] + + required_parts = path.split("/")[:-1] + parents = [ + # the root group + AsyncGroup( + metadata=GroupMetadata(zarr_format=node.metadata.zarr_format), + store_path=StorePath(store=store, path=""), + ) + ] + + for i, part in enumerate(required_parts): + p = "/".join(required_parts[:i] + [part]) + parents.append( + AsyncGroup( + metadata=GroupMetadata(zarr_format=node.metadata.zarr_format), + store_path=StorePath(store=store, path=p), + ) + ) + + return parents + + +FiltersLike: TypeAlias = ( + Iterable[dict[str, JSON] | ArrayArrayCodec | numcodecs.abc.Codec] + | ArrayArrayCodec + | Iterable[numcodecs.abc.Codec] + | numcodecs.abc.Codec + | Literal["auto"] + | None +) +# Union of acceptable types for users to pass in for both v2 and v3 compressors +CompressorLike: TypeAlias = ( + dict[str, JSON] | BytesBytesCodec | numcodecs.abc.Codec | Literal["auto"] | None +) + +CompressorsLike: TypeAlias = ( + Iterable[dict[str, JSON] | BytesBytesCodec | numcodecs.abc.Codec] + | dict[str, JSON] + | BytesBytesCodec + | numcodecs.abc.Codec + | Literal["auto"] + | None +) +SerializerLike: TypeAlias = dict[str, JSON] | ArrayBytesCodec | Literal["auto"] + + +class ShardsConfigParam(TypedDict): + shape: ChunkCoords + index_location: ShardingCodecIndexLocation | None + + +ShardsLike: TypeAlias = ChunkCoords | ShardsConfigParam | Literal["auto"] + + +async def from_array( + store: str | StoreLike, + *, + data: Array | npt.ArrayLike, + write_data: bool = True, + name: str | None = None, + chunks: Literal["auto", "keep"] | ChunkCoords = "keep", + shards: ShardsLike | None | Literal["keep"] = "keep", + filters: FiltersLike | Literal["keep"] = "keep", + compressors: CompressorsLike | Literal["keep"] = "keep", + serializer: SerializerLike | Literal["keep"] = "keep", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfig | ArrayConfigLike | None = None, +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array from an existing array or array-like. + + Parameters + ---------- + store : str or Store + Store or path to directory in file system or name of zip file for the new array. + data : Array | array-like + The array to copy. + write_data : bool, default True + Whether to copy the data from the input array to the new array. + If ``write_data`` is ``False``, the new array will be created with the same metadata as the + input array, but without any data. + name : str or None, optional + The name of the array within the store. If ``name`` is ``None``, the array will be located + at the root of the store. + chunks : ChunkCoords or "auto" or "keep", optional + Chunk shape of the array. + Following values are supported: + + - "auto": Automatically determine the chunk shape based on the array's shape and dtype. + - "keep": Retain the chunk shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the chunk shape. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise "auto". + shards : ChunkCoords, optional + Shard shape of the array. + Following values are supported: + + - "auto": Automatically determine the shard shape based on the array's shape and chunk shape. + - "keep": Retain the shard shape of the data array if it is a zarr Array. + - ChunkCoords: A tuple of integers representing the shard shape. + - None: No sharding. + + If not specified, defaults to "keep" if data is a zarr Array, otherwise None. + filters : Iterable[Codec] or "auto" or "keep", optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + + Following values are supported: + + - Iterable[Codec]: List of filters to apply to the array. + - "auto": Automatically determine the filters based on the array's dtype. + - "keep": Retain the filters of the data array if it is a zarr Array. + + If no ``filters`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + compressors : Iterable[Codec] or "auto" or "keep", optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + + Following values are supported: + + - Iterable[Codec]: List of compressors to apply to the array. + - "auto": Automatically determine the compressors based on the array's dtype. + - "keep": Retain the compressors of the input array if it is a zarr Array. + + If no ``compressors`` are provided, defaults to "keep" if data is a zarr Array, otherwise "auto". + serializer : dict[str, JSON] | ArrayBytesCodec or "auto" or "keep", optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + + Following values are supported: + + - dict[str, JSON]: A dict representation of an ``ArrayBytesCodec``. + - ArrayBytesCodec: An instance of ``ArrayBytesCodec``. + - "auto": a default serializer will be used. These defaults can be changed by modifying the value of + ``array.v3_default_serializer`` in :mod:`zarr.core.config`. + - "keep": Retain the serializer of the input array if it is a zarr Array. + + fill_value : Any, optional + Fill value for the array. + If not specified, defaults to the fill value of the data array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If not specified, defaults to the memory order of the data array. + zarr_format : {2, 3}, optional + The zarr format to use when saving. + If not specified, defaults to the zarr format of the data array. + attributes : dict, optional + Attributes for the array. + If not specified, defaults to the attributes of the data array. + chunk_key_encoding : ChunkKeyEncoding, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + If not specified and the data array has the same zarr format as the target array, + the chunk key encoding of the data array is used. + dimension_names : Iterable[str | None], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + If not specified, defaults to the dimension names of the data array. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfig or ArrayConfigLike, optional + Runtime configuration for the array. + + Returns + ------- + AsyncArray + The array. + + Examples + -------- + Create an array from an existing Array:: + + >>> import zarr + >>> store = zarr.storage.MemoryStore() + >>> store2 = zarr.storage.LocalStore('example.zarr') + >>> arr = zarr.create_array( + >>> store=store, + >>> shape=(100,100), + >>> chunks=(10,10), + >>> dtype='int32', + >>> fill_value=0) + >>> arr2 = await zarr.api.asynchronous.from_array(store2, data=arr) + + + Create an array from an existing NumPy array:: + + >>> arr3 = await zarr.api.asynchronous.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=np.arange(10000, dtype='i4').reshape(100, 100), + >>> ) + + + Create an array from any array-like object:: + + >>> arr4 = await zarr.api.asynchronous.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=[[1, 2], [3, 4]], + >>> ) + + >>> await arr4.getitem(...) + array([[1, 2],[3, 4]]) + + Create an array from an existing Array without copying the data:: + + >>> arr5 = await zarr.api.asynchronous.from_array( + >>> zarr.storage.MemoryStore(), + >>> data=Array(arr4), + >>> write_data=False, + >>> ) + + >>> await arr5.getitem(...) + array([[0, 0],[0, 0]]) + """ + mode: Literal["a"] = "a" + config_parsed = parse_array_config(config) + store_path = await make_store_path(store, path=name, mode=mode, storage_options=storage_options) + + ( + chunks, + shards, + filters, + compressors, + serializer, + fill_value, + order, + zarr_format, + chunk_key_encoding, + dimension_names, + ) = _parse_keep_array_attr( + data=data, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + ) + if not hasattr(data, "dtype") or not hasattr(data, "shape"): + data = np.array(data) + + result = await init_array( + store_path=store_path, + shape=data.shape, + dtype=data.dtype, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + overwrite=overwrite, + config=config_parsed, + ) + + if write_data: + if isinstance(data, Array): + + async def _copy_array_region(chunk_coords: ChunkCoords | slice, _data: Array) -> None: + arr = await _data._async_array.getitem(chunk_coords) + await result.setitem(chunk_coords, arr) + + # Stream data from the source array to the new array + await concurrent_map( + [(region, data) for region in result._iter_chunk_regions()], + _copy_array_region, + zarr.core.config.config.get("async.concurrency"), + ) + else: + + async def _copy_arraylike_region(chunk_coords: slice, _data: NDArrayLike) -> None: + await result.setitem(chunk_coords, _data[chunk_coords]) + + # Stream data from the source array to the new array + await concurrent_map( + [(region, data) for region in result._iter_chunk_regions()], + _copy_arraylike_region, + zarr.core.config.config.get("async.concurrency"), + ) + return result + + +async def init_array( + *, + store_path: StorePath, + shape: ShapeLike, + dtype: ZDTypeLike, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat | None = 3, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, + overwrite: bool = False, + config: ArrayConfigLike | None, +) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata]: + """Create and persist an array metadata document. + + Parameters + ---------- + store_path : StorePath + StorePath instance. The path attribute is the name of the array to initialize. + shape : ChunkCoords + Shape of the array. + dtype : ZDTypeLike + Data type of the array. + chunks : ChunkCoords, optional + Chunk shape of the array. + If not specified, default are guessed based on the shape and dtype. + shards : ChunkCoords, optional + Shard shape of the array. The default value of ``None`` results in no sharding at all. + filters : Iterable[Codec], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v3_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v2_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no ``compressor`` is provided, a default compressor will be used. + in :mod:`zarr.core.config`. + Use ``None`` to omit the default compressor. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` + in :mod:`zarr.core.config`. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If no ``order`` is provided, a default order will be used. + This default can be changed by modifying the value of ``array.order`` in :mod:`zarr.core.config`. + zarr_format : {2, 3}, optional + The zarr format to use when saving. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfigLike or None, optional + Configuration for this array. + + Returns + ------- + AsyncArray + The AsyncArray. + """ + + if zarr_format is None: + zarr_format = _default_zarr_format() + + from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation + + zdtype = parse_data_type(dtype, zarr_format=zarr_format) + shape_parsed = parse_shapelike(shape) + chunk_key_encoding_parsed = _parse_chunk_key_encoding( + chunk_key_encoding, zarr_format=zarr_format + ) + + if overwrite: + if store_path.store.supports_deletes: + await store_path.delete_dir() + else: + await ensure_no_existing_node(store_path, zarr_format=zarr_format) + else: + await ensure_no_existing_node(store_path, zarr_format=zarr_format) + + item_size = 1 + if isinstance(zdtype, HasItemSize): + item_size = zdtype.item_size + + shard_shape_parsed, chunk_shape_parsed = _auto_partition( + array_shape=shape_parsed, + shard_shape=shards, + chunk_shape=chunks, + item_size=item_size, + ) + chunks_out: tuple[int, ...] + meta: ArrayV2Metadata | ArrayV3Metadata + if zarr_format == 2: + if shard_shape_parsed is not None: + msg = ( + "Zarr format 2 arrays can only be created with `shard_shape` set to `None`. " + f"Got `shard_shape={shards}` instead." + ) + + raise ValueError(msg) + if serializer != "auto": + raise ValueError("Zarr format 2 arrays do not support `serializer`.") + + filters_parsed, compressor_parsed = _parse_chunk_encoding_v2( + compressor=compressors, filters=filters, dtype=zdtype + ) + if dimension_names is not None: + raise ValueError("Zarr format 2 arrays do not support dimension names.") + if order is None: + order_parsed = zarr_config.get("array.order") + else: + order_parsed = order + + meta = AsyncArray._create_metadata_v2( + shape=shape_parsed, + dtype=zdtype, + chunks=chunk_shape_parsed, + dimension_separator=chunk_key_encoding_parsed.separator, + fill_value=fill_value, + order=order_parsed, + filters=filters_parsed, + compressor=compressor_parsed, + attributes=attributes, + ) + else: + array_array, array_bytes, bytes_bytes = _parse_chunk_encoding_v3( + compressors=compressors, + filters=filters, + serializer=serializer, + dtype=zdtype, + ) + sub_codecs = cast("tuple[Codec, ...]", (*array_array, array_bytes, *bytes_bytes)) + codecs_out: tuple[Codec, ...] + if shard_shape_parsed is not None: + index_location = None + if isinstance(shards, dict): + index_location = ShardingCodecIndexLocation(shards.get("index_location", None)) + if index_location is None: + index_location = ShardingCodecIndexLocation.end + sharding_codec = ShardingCodec( + chunk_shape=chunk_shape_parsed, codecs=sub_codecs, index_location=index_location + ) + sharding_codec.validate( + shape=chunk_shape_parsed, + dtype=zdtype, + chunk_grid=RegularChunkGrid(chunk_shape=shard_shape_parsed), + ) + codecs_out = (sharding_codec,) + chunks_out = shard_shape_parsed + else: + chunks_out = chunk_shape_parsed + codecs_out = sub_codecs + + if config is None: + config = {} + if order is not None and isinstance(config, dict): + config["order"] = config.get("order", order) + + meta = AsyncArray._create_metadata_v3( + shape=shape_parsed, + dtype=zdtype, + fill_value=fill_value, + chunk_shape=chunks_out, + chunk_key_encoding=chunk_key_encoding_parsed, + codecs=codecs_out, + dimension_names=dimension_names, + attributes=attributes, + ) + + arr = AsyncArray(metadata=meta, store_path=store_path, config=config) + await arr._save_metadata(meta, ensure_parents=True) + return arr + + +async def create_array( + store: str | StoreLike, + *, + name: str | None = None, + shape: ShapeLike | None = None, + dtype: ZDTypeLike | None = None, + data: np.ndarray[Any, np.dtype[Any]] | None = None, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = None, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat | None = 3, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfigLike | None = None, + write_data: bool = True, +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array. + + Parameters + ---------- + store : str or Store + Store or path to directory in file system or name of zip file. + name : str or None, optional + The name of the array within the store. If ``name`` is ``None``, the array will be located + at the root of the store. + shape : ChunkCoords, optional + Shape of the array. Can be ``None`` if ``data`` is provided. + dtype : ZDTypeLike | None + Data type of the array. Can be ``None`` if ``data`` is provided. + data : Array-like data to use for initializing the array. If this parameter is provided, the + ``shape`` and ``dtype`` parameters must be identical to ``data.shape`` and ``data.dtype``, + or ``None``. + chunks : ChunkCoords, optional + Chunk shape of the array. + If not specified, default are guessed based on the shape and dtype. + shards : ChunkCoords, optional + Shard shape of the array. The default value of ``None`` results in no sharding at all. + filters : Iterable[Codec], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v3_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v2_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no ``compressor`` is provided, a default compressor will be used. + in :mod:`zarr.core.config`. + Use ``None`` to omit the default compressor. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` + in :mod:`zarr.core.config`. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If no ``order`` is provided, a default order will be used. + This default can be changed by modifying the value of ``array.order`` in :mod:`zarr.core.config`. + zarr_format : {2, 3}, optional + The zarr format to use when saving. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncodingLike, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfigLike, optional + Runtime configuration for the array. + write_data : bool + If a pre-existing array-like object was provided to this function via the ``data`` parameter + then ``write_data`` determines whether the values in that array-like object should be + written to the Zarr array created by this function. If ``write_data`` is ``False``, then the + array will be left empty. + + Returns + ------- + AsyncArray + The array. + + Examples + -------- + >>> import zarr + >>> store = zarr.storage.MemoryStore(mode='w') + >>> async_arr = await zarr.api.asynchronous.create_array( + >>> store=store, + >>> shape=(100,100), + >>> chunks=(10,10), + >>> dtype='i4', + >>> fill_value=0) + + """ + data_parsed, shape_parsed, dtype_parsed = _parse_data_params( + data=data, shape=shape, dtype=dtype + ) + if data_parsed is not None: + return await from_array( + store, + data=data_parsed, + write_data=write_data, + name=name, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, + overwrite=overwrite, + config=config, + ) + else: + mode: Literal["a"] = "a" + + store_path = await make_store_path( + store, path=name, mode=mode, storage_options=storage_options + ) + return await init_array( + store_path=store_path, + shape=shape_parsed, + dtype=dtype_parsed, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + overwrite=overwrite, + config=config, + ) + + +def _parse_keep_array_attr( + data: Array | npt.ArrayLike, + chunks: Literal["auto", "keep"] | ChunkCoords, + shards: ShardsLike | None | Literal["keep"], + filters: FiltersLike | Literal["keep"], + compressors: CompressorsLike | Literal["keep"], + serializer: SerializerLike | Literal["keep"], + fill_value: Any | None, + order: MemoryOrder | None, + zarr_format: ZarrFormat | None, + chunk_key_encoding: ChunkKeyEncodingLike | None, + dimension_names: DimensionNames, +) -> tuple[ + ChunkCoords | Literal["auto"], + ShardsLike | None, + FiltersLike, + CompressorsLike, + SerializerLike, + Any | None, + MemoryOrder | None, + ZarrFormat, + ChunkKeyEncodingLike | None, + DimensionNames, +]: + if isinstance(data, Array): + if chunks == "keep": + chunks = data.chunks + if shards == "keep": + shards = data.shards + if zarr_format is None: + zarr_format = data.metadata.zarr_format + if filters == "keep": + if zarr_format == data.metadata.zarr_format: + filters = data.filters or None + else: + filters = "auto" + if compressors == "keep": + if zarr_format == data.metadata.zarr_format: + compressors = data.compressors or None + else: + compressors = "auto" + if serializer == "keep": + if zarr_format == 3 and data.metadata.zarr_format == 3: + serializer = cast("SerializerLike", data.serializer) + else: + serializer = "auto" + if fill_value is None: + fill_value = data.fill_value + if order is None: + order = data.order + if chunk_key_encoding is None and zarr_format == data.metadata.zarr_format: + if isinstance(data.metadata, ArrayV2Metadata): + chunk_key_encoding = {"name": "v2", "separator": data.metadata.dimension_separator} + elif isinstance(data.metadata, ArrayV3Metadata): + chunk_key_encoding = data.metadata.chunk_key_encoding + if dimension_names is None and data.metadata.zarr_format == 3: + dimension_names = data.metadata.dimension_names + else: + if chunks == "keep": + chunks = "auto" + if shards == "keep": + shards = None + if zarr_format is None: + zarr_format = 3 + if filters == "keep": + filters = "auto" + if compressors == "keep": + compressors = "auto" + if serializer == "keep": + serializer = "auto" + return ( + chunks, + shards, + filters, + compressors, + serializer, + fill_value, + order, + zarr_format, + chunk_key_encoding, + dimension_names, + ) + + +def _parse_chunk_key_encoding( + data: ChunkKeyEncodingLike | None, zarr_format: ZarrFormat +) -> ChunkKeyEncoding: + """ + Take an implicit specification of a chunk key encoding and parse it into a ChunkKeyEncoding object. + """ + if data is None: + if zarr_format == 2: + result = ChunkKeyEncoding.from_dict({"name": "v2", "separator": "."}) + else: + result = ChunkKeyEncoding.from_dict({"name": "default", "separator": "/"}) + elif isinstance(data, ChunkKeyEncoding): + result = data + else: + result = ChunkKeyEncoding.from_dict(data) + if zarr_format == 2 and result.name != "v2": + msg = ( + "Invalid chunk key encoding. For Zarr format 2 arrays, the `name` field of the " + f"chunk key encoding must be 'v2'. Got `name` = {result.name} instead." + ) + raise ValueError(msg) + return result + + +def _get_default_chunk_encoding_v3( + dtype: ZDType[TBaseDType, TBaseScalar], +) -> tuple[tuple[ArrayArrayCodec, ...], ArrayBytesCodec, tuple[BytesBytesCodec, ...]]: + """ + Get the default ArrayArrayCodecs, ArrayBytesCodec, and BytesBytesCodec for a given dtype. + """ + + dtype_category = categorize_data_type(dtype) + + filters = zarr_config.get("array.v3_default_filters").get(dtype_category) + compressors = zarr_config.get("array.v3_default_compressors").get(dtype_category) + serializer = zarr_config.get("array.v3_default_serializer").get(dtype_category) + + return ( + tuple(_parse_array_array_codec(f) for f in filters), + _parse_array_bytes_codec(serializer), + tuple(_parse_bytes_bytes_codec(c) for c in compressors), + ) + + +def _get_default_chunk_encoding_v2( + dtype: ZDType[TBaseDType, TBaseScalar], +) -> tuple[tuple[numcodecs.abc.Codec, ...] | None, numcodecs.abc.Codec | None]: + """ + Get the default chunk encoding for Zarr format 2 arrays, given a dtype + """ + dtype_category = categorize_data_type(dtype) + filters = zarr_config.get("array.v2_default_filters").get(dtype_category) + compressor = zarr_config.get("array.v2_default_compressor").get(dtype_category) + if filters is not None: + filters = tuple(numcodecs.get_codec(f) for f in filters) + + return filters, numcodecs.get_codec(compressor) + + +def _parse_chunk_encoding_v2( + *, + compressor: CompressorsLike, + filters: FiltersLike, + dtype: ZDType[TBaseDType, TBaseScalar], +) -> tuple[tuple[numcodecs.abc.Codec, ...] | None, numcodecs.abc.Codec | None]: + """ + Generate chunk encoding classes for Zarr format 2 arrays with optional defaults. + """ + default_filters, default_compressor = _get_default_chunk_encoding_v2(dtype) + _filters: tuple[numcodecs.abc.Codec, ...] | None + _compressor: numcodecs.abc.Codec | None + + if compressor is None or compressor == (): + _compressor = None + elif compressor == "auto": + _compressor = default_compressor + elif isinstance(compressor, tuple | list) and len(compressor) == 1: + _compressor = parse_compressor(compressor[0]) + else: + if isinstance(compressor, Iterable) and not isinstance(compressor, dict): + msg = f"For Zarr format 2 arrays, the `compressor` must be a single codec. Got an iterable with type {type(compressor)} instead." + raise TypeError(msg) + _compressor = parse_compressor(compressor) + + if filters is None: + _filters = None + elif filters == "auto": + _filters = default_filters + else: + if isinstance(filters, Iterable): + for idx, f in enumerate(filters): + if not isinstance(f, numcodecs.abc.Codec): + msg = ( + "For Zarr format 2 arrays, all elements of `filters` must be numcodecs codecs. " + f"Element at index {idx} has type {type(f)}, which is not a numcodecs codec." + ) + raise TypeError(msg) + _filters = parse_filters(filters) + + return _filters, _compressor + + +def _parse_chunk_encoding_v3( + *, + compressors: CompressorsLike, + filters: FiltersLike, + serializer: SerializerLike, + dtype: ZDType[TBaseDType, TBaseScalar], +) -> tuple[tuple[ArrayArrayCodec, ...], ArrayBytesCodec, tuple[BytesBytesCodec, ...]]: + """ + Generate chunk encoding classes for v3 arrays with optional defaults. + """ + default_array_array, default_array_bytes, default_bytes_bytes = _get_default_chunk_encoding_v3( + dtype + ) + + if filters is None: + out_array_array: tuple[ArrayArrayCodec, ...] = () + elif filters == "auto": + out_array_array = default_array_array + else: + maybe_array_array: Iterable[Codec | dict[str, JSON]] + if isinstance(filters, dict | Codec): + maybe_array_array = (filters,) + else: + maybe_array_array = cast("Iterable[Codec | dict[str, JSON]]", filters) + out_array_array = tuple(_parse_array_array_codec(c) for c in maybe_array_array) + + if serializer == "auto": + out_array_bytes = default_array_bytes + else: + # TODO: ensure that the serializer is compatible with the ndarray produced by the + # array-array codecs. For example, if a sequence of array-array codecs produces an + # array with a single-byte data type, then the serializer should not specify endiannesss. + out_array_bytes = _parse_array_bytes_codec(serializer) + + if compressors is None: + out_bytes_bytes: tuple[BytesBytesCodec, ...] = () + elif compressors == "auto": + out_bytes_bytes = default_bytes_bytes + else: + maybe_bytes_bytes: Iterable[Codec | dict[str, JSON]] + if isinstance(compressors, dict | Codec): + maybe_bytes_bytes = (compressors,) + else: + maybe_bytes_bytes = cast("Iterable[Codec | dict[str, JSON]]", compressors) + + out_bytes_bytes = tuple(_parse_bytes_bytes_codec(c) for c in maybe_bytes_bytes) + + # specialize codecs as needed given the dtype + + # TODO: refactor so that the config only contains the name of the codec, and we use the dtype + # to create the codec instance, instead of storing a dict representation of a full codec. + + # TODO: ensure that the serializer is compatible with the ndarray produced by the + # array-array codecs. For example, if a sequence of array-array codecs produces an + # array with a single-byte data type, then the serializer should not specify endiannesss. + if isinstance(out_array_bytes, BytesCodec) and not isinstance(dtype, HasEndianness): + # The default endianness in the bytescodec might not be None, so we need to replace it + out_array_bytes = replace(out_array_bytes, endian=None) + return out_array_array, out_array_bytes, out_bytes_bytes + + +def _parse_deprecated_compressor( + compressor: CompressorLike | None, compressors: CompressorsLike, zarr_format: int = 3 +) -> CompressorsLike | None: + if compressor != "auto": + if compressors != "auto": + raise ValueError("Cannot specify both `compressor` and `compressors`.") + if zarr_format == 3: + warn( + "The `compressor` argument is deprecated. Use `compressors` instead.", + category=UserWarning, + stacklevel=2, + ) + if compressor is None: + # "no compression" + compressors = () + else: + compressors = (compressor,) + elif zarr_format == 2 and compressor == compressors == "auto": + compressors = ({"id": "blosc"},) + return compressors + + +def _parse_data_params( + *, + data: np.ndarray[Any, np.dtype[Any]] | None, + shape: ShapeLike | None, + dtype: ZDTypeLike | None, +) -> tuple[np.ndarray[Any, np.dtype[Any]] | None, ShapeLike, ZDTypeLike]: + """ + Ensure an array-like ``data`` parameter is consistent with the ``dtype`` and ``shape`` + parameters. + """ + if data is None: + if shape is None: + msg = ( + "The data parameter was set to None, but shape was not specified. " + "Either provide a value for data, or specify shape." + ) + raise ValueError(msg) + shape_out = shape + if dtype is None: + msg = ( + "The data parameter was set to None, but dtype was not specified." + "Either provide an array-like value for data, or specify dtype." + ) + raise ValueError(msg) + dtype_out = dtype + else: + if shape is not None: + msg = ( + "The data parameter was used, but the shape parameter was also " + "used. This is an error. Either use the data parameter, or the shape parameter, " + "but not both." + ) + raise ValueError(msg) + shape_out = data.shape + if dtype is not None: + msg = ( + "The data parameter was used, but the dtype parameter was also " + "used. This is an error. Either use the data parameter, or the dtype parameter, " + "but not both." + ) + raise ValueError(msg) + dtype_out = data.dtype + return data, shape_out, dtype_out diff --git a/src/zarr/core/array_spec.py b/src/zarr/core/array_spec.py new file mode 100644 index 0000000000..279bf6edf0 --- /dev/null +++ b/src/zarr/core/array_spec.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +from dataclasses import dataclass, fields +from typing import TYPE_CHECKING, Any, Literal, Self, TypedDict, cast + +from zarr.core.common import ( + MemoryOrder, + parse_bool, + parse_fill_value, + parse_order, + parse_shapelike, +) +from zarr.core.config import config as zarr_config + +if TYPE_CHECKING: + from typing import NotRequired + + from zarr.core.buffer import BufferPrototype + from zarr.core.common import ChunkCoords + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + + +class ArrayConfigParams(TypedDict): + """ + A TypedDict model of the attributes of an ArrayConfig class, but with no required fields. + This allows for partial construction of an ArrayConfig, with the assumption that the unset + keys will be taken from a global configuration. + """ + + order: NotRequired[MemoryOrder] + write_empty_chunks: NotRequired[bool] + + +@dataclass(frozen=True) +class ArrayConfig: + """ + A model of the runtime configuration of an array. + + Parameters + ---------- + order : MemoryOrder + The memory layout of the arrays returned when reading data from the store. + write_empty_chunks : bool + If True, empty chunks will be written to the store. + """ + + order: MemoryOrder + write_empty_chunks: bool + + def __init__(self, order: MemoryOrder, write_empty_chunks: bool) -> None: + order_parsed = parse_order(order) + write_empty_chunks_parsed = parse_bool(write_empty_chunks) + + object.__setattr__(self, "order", order_parsed) + object.__setattr__(self, "write_empty_chunks", write_empty_chunks_parsed) + + @classmethod + def from_dict(cls, data: ArrayConfigParams) -> Self: + """ + Create an ArrayConfig from a dict. The keys of that dict are a subset of the + attributes of the ArrayConfig class. Any keys missing from that dict will be set to the + the values in the ``array`` namespace of ``zarr.config``. + """ + kwargs_out: ArrayConfigParams = {} + for f in fields(ArrayConfig): + field_name = cast("Literal['order', 'write_empty_chunks']", f.name) + if field_name not in data: + kwargs_out[field_name] = zarr_config.get(f"array.{field_name}") + else: + kwargs_out[field_name] = data[field_name] + return cls(**kwargs_out) + + +ArrayConfigLike = ArrayConfig | ArrayConfigParams + + +def parse_array_config(data: ArrayConfigLike | None) -> ArrayConfig: + """ + Convert various types of data to an ArrayConfig. + """ + if data is None: + return ArrayConfig.from_dict({}) + elif isinstance(data, ArrayConfig): + return data + else: + return ArrayConfig.from_dict(data) + + +@dataclass(frozen=True) +class ArraySpec: + shape: ChunkCoords + dtype: ZDType[TBaseDType, TBaseScalar] + fill_value: Any + config: ArrayConfig + prototype: BufferPrototype + + def __init__( + self, + shape: ChunkCoords, + dtype: ZDType[TBaseDType, TBaseScalar], + fill_value: Any, + config: ArrayConfig, + prototype: BufferPrototype, + ) -> None: + shape_parsed = parse_shapelike(shape) + fill_value_parsed = parse_fill_value(fill_value) + + object.__setattr__(self, "shape", shape_parsed) + object.__setattr__(self, "dtype", dtype) + object.__setattr__(self, "fill_value", fill_value_parsed) + object.__setattr__(self, "config", config) + object.__setattr__(self, "prototype", prototype) + + @property + def ndim(self) -> int: + return len(self.shape) + + @property + def order(self) -> MemoryOrder: + return self.config.order diff --git a/src/zarr/core/attributes.py b/src/zarr/core/attributes.py new file mode 100644 index 0000000000..e699c4f66d --- /dev/null +++ b/src/zarr/core/attributes.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from collections.abc import MutableMapping +from typing import TYPE_CHECKING + +from zarr.core.common import JSON + +if TYPE_CHECKING: + from collections.abc import Iterator + + from zarr.core.array import Array + from zarr.core.group import Group + + +class Attributes(MutableMapping[str, JSON]): + def __init__(self, obj: Array | Group) -> None: + # key=".zattrs", read_only=False, cache=True, synchronizer=None + self._obj = obj + + def __getitem__(self, key: str) -> JSON: + return self._obj.metadata.attributes[key] + + def __setitem__(self, key: str, value: JSON) -> None: + new_attrs = dict(self._obj.metadata.attributes) + new_attrs[key] = value + self._obj = self._obj.update_attributes(new_attrs) + + def __delitem__(self, key: str) -> None: + new_attrs = dict(self._obj.metadata.attributes) + del new_attrs[key] + self.put(new_attrs) + + def __iter__(self) -> Iterator[str]: + return iter(self._obj.metadata.attributes) + + def __len__(self) -> int: + return len(self._obj.metadata.attributes) + + def put(self, d: dict[str, JSON]) -> None: + """ + Overwrite all attributes with the values from `d`. + + Equivalent to the following pseudo-code, but performed atomically. + + .. code-block:: python + + >>> attrs = {"a": 1, "b": 2} + >>> attrs.clear() + >>> attrs.update({"a": 3", "c": 4}) + >>> attrs + {'a': 3, 'c': 4} + """ + self._obj.metadata.attributes.clear() + self._obj = self._obj.update_attributes(d) + + def asdict(self) -> dict[str, JSON]: + return dict(self._obj.metadata.attributes) diff --git a/src/zarr/core/buffer/__init__.py b/src/zarr/core/buffer/__init__.py new file mode 100644 index 0000000000..ebec61a372 --- /dev/null +++ b/src/zarr/core/buffer/__init__.py @@ -0,0 +1,21 @@ +from zarr.core.buffer.core import ( + ArrayLike, + Buffer, + BufferPrototype, + NDArrayLike, + NDArrayLikeOrScalar, + NDBuffer, + default_buffer_prototype, +) +from zarr.core.buffer.cpu import numpy_buffer_prototype + +__all__ = [ + "ArrayLike", + "Buffer", + "BufferPrototype", + "NDArrayLike", + "NDArrayLikeOrScalar", + "NDBuffer", + "default_buffer_prototype", + "numpy_buffer_prototype", +] diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py new file mode 100644 index 0000000000..0e24c5b326 --- /dev/null +++ b/src/zarr/core/buffer/core.py @@ -0,0 +1,537 @@ +from __future__ import annotations + +import sys +from abc import ABC, abstractmethod +from typing import ( + TYPE_CHECKING, + Any, + Literal, + NamedTuple, + Protocol, + SupportsIndex, + cast, + runtime_checkable, +) + +import numpy as np +import numpy.typing as npt + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + from typing import Self + + from zarr.codecs.bytes import Endian + from zarr.core.common import BytesLike, ChunkCoords + +# Everything here is imported into ``zarr.core.buffer`` namespace. +__all__: list[str] = [] + + +@runtime_checkable +class ArrayLike(Protocol): + """Protocol for the array-like type that underlie Buffer""" + + @property + def dtype(self) -> np.dtype[Any]: ... + + @property + def ndim(self) -> int: ... + + @property + def size(self) -> int: ... + + def __getitem__(self, key: slice) -> Self: ... + + def __setitem__(self, key: slice, value: Any) -> None: ... + + +@runtime_checkable +class NDArrayLike(Protocol): + """Protocol for the nd-array-like type that underlie NDBuffer""" + + @property + def dtype(self) -> np.dtype[Any]: ... + + @property + def ndim(self) -> int: ... + + @property + def size(self) -> int: ... + + @property + def shape(self) -> ChunkCoords: ... + + def __len__(self) -> int: ... + + def __getitem__(self, key: slice) -> Self: ... + + def __setitem__(self, key: slice, value: Any) -> None: ... + + def __array__(self) -> npt.NDArray[Any]: ... + + def reshape( + self, shape: ChunkCoords | Literal[-1], *, order: Literal["A", "C", "F"] = ... + ) -> Self: ... + + def view(self, dtype: npt.DTypeLike) -> Self: ... + + def astype( + self, + dtype: npt.DTypeLike, + order: Literal["K", "A", "C", "F"] = ..., + *, + copy: bool = ..., + ) -> Self: ... + + def fill(self, value: Any) -> None: ... + + def copy(self) -> Self: ... + + def transpose(self, axes: SupportsIndex | Sequence[SupportsIndex] | None) -> Self: ... + + def ravel(self, order: Literal["K", "A", "C", "F"] = ...) -> Self: ... + + def all(self) -> bool: ... + + def __eq__(self, other: object) -> Self: # type: ignore[explicit-override, override] + """Element-wise equal + + Notes + ----- + Type checkers such as mypy complains because the return type isn't a bool like + its supertype "object", which violates the Liskov substitution principle. + This is true, but since NumPy's ndarray is defined as an element-wise equal, + our hands are tied. + """ + + +ScalarType = int | float | complex | bytes | str | bool | np.generic +NDArrayLikeOrScalar = ScalarType | NDArrayLike + + +def check_item_key_is_1d_contiguous(key: Any) -> None: + """Raises error if `key` isn't a 1d contiguous slice""" + if not isinstance(key, slice): + raise TypeError( + f"Item key has incorrect type (expected slice, got {key.__class__.__name__})" + ) + if not (key.step is None or key.step == 1): + raise ValueError("slice must be contiguous") + + +class Buffer(ABC): + """A flat contiguous memory block + + We use Buffer throughout Zarr to represent a contiguous block of memory. + + A Buffer is backed by a underlying array-like instance that represents + the memory. The memory type is unspecified; can be regular host memory, + CUDA device memory, or something else. The only requirement is that the + array-like instance can be copied/converted to a regular Numpy array + (host memory). + + Notes + ----- + This buffer is untyped, so all indexing and sizes are in bytes. + + Parameters + ---------- + array_like + array-like object that must be 1-dim, contiguous, and byte dtype. + """ + + def __init__(self, array_like: ArrayLike) -> None: + if array_like.ndim != 1: + raise ValueError("array_like: only 1-dim allowed") + if array_like.dtype != np.dtype("B"): + raise ValueError("array_like: only byte dtype allowed") + self._data = array_like + + @classmethod + @abstractmethod + def create_zero_length(cls) -> Self: + """Create an empty buffer with length zero + + Returns + ------- + New empty 0-length buffer + """ + if cls is Buffer: + raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") + return cls( + cast("ArrayLike", None) + ) # This line will never be reached, but it satisfies the type checker + + @classmethod + def from_array_like(cls, array_like: ArrayLike) -> Self: + """Create a new buffer of an array-like object + + Parameters + ---------- + array_like + array-like object that must be 1-dim, contiguous, and byte dtype. + + Returns + ------- + New buffer representing `array_like` + """ + return cls(array_like) + + @classmethod + @abstractmethod + def from_buffer(cls, buffer: Buffer) -> Self: + """Create a new buffer of an existing Buffer + + This is useful if you want to ensure that an existing buffer is + of the correct subclass of Buffer. E.g., MemoryStore uses this + to return a buffer instance of the subclass specified by its + BufferPrototype argument. + + Typically, this only copies data if the data has to be moved between + memory types, such as from host to device memory. + + Parameters + ---------- + buffer + buffer object. + + Returns + ------- + A new buffer representing the content of the input buffer + + Notes + ----- + Subclasses of `Buffer` must override this method to implement + more optimal conversions that avoid copies where possible + """ + if cls is Buffer: + raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") + return cls( + cast("ArrayLike", None) + ) # This line will never be reached, but it satisfies the type checker + + @classmethod + @abstractmethod + def from_bytes(cls, bytes_like: BytesLike) -> Self: + """Create a new buffer of a bytes-like object (host memory) + + Parameters + ---------- + bytes_like + bytes-like object + + Returns + ------- + New buffer representing `bytes_like` + """ + if cls is Buffer: + raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") + return cls( + cast("ArrayLike", None) + ) # This line will never be reached, but it satisfies the type checker + + def as_array_like(self) -> ArrayLike: + """Returns the underlying array (host or device memory) of this buffer + + This will never copy data. + + Returns + ------- + The underlying 1d array such as a NumPy or CuPy array. + """ + return self._data + + @abstractmethod + def as_numpy_array(self) -> npt.NDArray[Any]: + """Returns the buffer as a NumPy array (host memory). + + Notes + ----- + Might have to copy data, consider using `.as_array_like()` instead. + + Returns + ------- + NumPy array of this buffer (might be a data copy) + """ + ... + + def as_buffer_like(self) -> BytesLike: + """Returns the buffer as an object that implements the Python buffer protocol. + + Notes + ----- + Might have to copy data, since the implementation uses `.as_numpy_array()`. + + Returns + ------- + An object that implements the Python buffer protocol + """ + return memoryview(self.as_numpy_array()) # type: ignore[arg-type] + + def to_bytes(self) -> bytes: + """Returns the buffer as `bytes` (host memory). + + Warnings + -------- + Will always copy data, only use this method for small buffers such as metadata + buffers. If possible, use `.as_numpy_array()` or `.as_array_like()` instead. + + Returns + ------- + `bytes` of this buffer (data copy) + """ + return bytes(self.as_numpy_array()) + + def __getitem__(self, key: slice) -> Self: + check_item_key_is_1d_contiguous(key) + return self.__class__(self._data.__getitem__(key)) + + def __setitem__(self, key: slice, value: Any) -> None: + check_item_key_is_1d_contiguous(key) + self._data.__setitem__(key, value) + + def __len__(self) -> int: + return self._data.size + + @abstractmethod + def __add__(self, other: Buffer) -> Self: + """Concatenate two buffers""" + ... + + def __eq__(self, other: object) -> bool: + # Another Buffer class can override this to choose a more efficient path + return isinstance(other, Buffer) and np.array_equal( + self.as_numpy_array(), other.as_numpy_array() + ) + + +class NDBuffer: + """An n-dimensional memory block + + We use NDBuffer throughout Zarr to represent a n-dimensional memory block. + + A NDBuffer is backed by a underlying ndarray-like instance that represents + the memory. The memory type is unspecified; can be regular host memory, + CUDA device memory, or something else. The only requirement is that the + ndarray-like instance can be copied/converted to a regular Numpy array + (host memory). + + Notes + ----- + The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer + is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, + in order to use Python's type system to differentiate between the contiguous + Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the + two classes separate. + + Parameters + ---------- + array : ndarray_like + ndarray-like object that is convertible to a regular Numpy array. + """ + + def __init__(self, array: NDArrayLike) -> None: + self._data = array + + @classmethod + @abstractmethod + def create( + cls, + *, + shape: Iterable[int], + dtype: npt.DTypeLike, + order: Literal["C", "F"] = "C", + fill_value: Any | None = None, + ) -> Self: + """Create a new buffer and its underlying ndarray-like object + + Parameters + ---------- + shape + The shape of the buffer and its underlying ndarray-like object + dtype + The datatype of the buffer and its underlying ndarray-like object + order + Whether to store multi-dimensional data in row-major (C-style) or + column-major (Fortran-style) order in memory. + fill_value + If not None, fill the new buffer with a scalar value. + + Returns + ------- + New buffer representing a new ndarray_like object + + Notes + ----- + A subclass can overwrite this method to create a ndarray-like object + other then the default Numpy array. + """ + if cls is NDBuffer: + raise NotImplementedError( + "Cannot call abstract method on the abstract class 'NDBuffer'" + ) + return cls( + cast("NDArrayLike", None) + ) # This line will never be reached, but it satisfies the type checker + + @classmethod + def from_ndarray_like(cls, ndarray_like: NDArrayLike) -> Self: + """Create a new buffer of a ndarray-like object + + Parameters + ---------- + ndarray_like + ndarray-like object + + Returns + ------- + New buffer representing `ndarray_like` + """ + return cls(ndarray_like) + + @classmethod + @abstractmethod + def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: + """Create a new buffer of Numpy array-like object + + Parameters + ---------- + array_like + Object that can be coerced into a Numpy array + + Returns + ------- + New buffer representing `array_like` + """ + if cls is NDBuffer: + raise NotImplementedError( + "Cannot call abstract method on the abstract class 'NDBuffer'" + ) + return cls( + cast("NDArrayLike", None) + ) # This line will never be reached, but it satisfies the type checker + + def as_ndarray_like(self) -> NDArrayLike: + """Returns the underlying array (host or device memory) of this buffer + + This will never copy data. + + Returns + ------- + The underlying array such as a NumPy or CuPy array. + """ + return self._data + + @abstractmethod + def as_numpy_array(self) -> npt.NDArray[Any]: + """Returns the buffer as a NumPy array (host memory). + + Warnings + -------- + Might have to copy data, consider using `.as_ndarray_like()` instead. + + Returns + ------- + NumPy array of this buffer (might be a data copy) + """ + ... + + def as_scalar(self) -> ScalarType: + """Returns the buffer as a scalar value""" + if self._data.size != 1: + raise ValueError("Buffer does not contain a single scalar value") + return cast("ScalarType", self.as_numpy_array()[()]) + + @property + def dtype(self) -> np.dtype[Any]: + return self._data.dtype + + @property + def shape(self) -> tuple[int, ...]: + return self._data.shape + + @property + def byteorder(self) -> Endian: + from zarr.codecs.bytes import Endian + + if self.dtype.byteorder == "<": + return Endian.little + elif self.dtype.byteorder == ">": + return Endian.big + else: + return Endian(sys.byteorder) + + def reshape(self, newshape: ChunkCoords | Literal[-1]) -> Self: + return self.__class__(self._data.reshape(newshape)) + + def squeeze(self, axis: tuple[int, ...]) -> Self: + newshape = tuple(a for i, a in enumerate(self.shape) if i not in axis) + return self.__class__(self._data.reshape(newshape)) + + def astype(self, dtype: npt.DTypeLike, order: Literal["K", "A", "C", "F"] = "K") -> Self: + return self.__class__(self._data.astype(dtype=dtype, order=order)) + + @abstractmethod + def __getitem__(self, key: Any) -> Self: ... + + @abstractmethod + def __setitem__(self, key: Any, value: Any) -> None: ... + + def __len__(self) -> int: + return self._data.__len__() + + def __repr__(self) -> str: + return f"" + + def all_equal(self, other: Any, equal_nan: bool = True) -> bool: + """Compare to `other` using np.array_equal.""" + if other is None: + # Handle None fill_value for Zarr V2 + return False + # use array_equal to obtain equal_nan=True functionality + # Since fill-value is a scalar, isn't there a faster path than allocating a new array for fill value + # every single time we have to write data? + _data, other = np.broadcast_arrays(self._data, other) + return np.array_equal( + self._data, + other, + equal_nan=equal_nan + if self._data.dtype.kind not in ("U", "S", "T", "O", "V") + else False, + ) + + def fill(self, value: Any) -> None: + self._data.fill(value) + + def copy(self) -> Self: + return self.__class__(self._data.copy()) + + def transpose(self, axes: SupportsIndex | Sequence[SupportsIndex] | None) -> Self: + return self.__class__(self._data.transpose(axes)) + + +class BufferPrototype(NamedTuple): + """Prototype of the Buffer and NDBuffer class + + The protocol must be pickable. + + Attributes + ---------- + buffer + The Buffer class to use when Zarr needs to create new Buffer. + nd_buffer + The NDBuffer class to use when Zarr needs to create new NDBuffer. + """ + + buffer: type[Buffer] + nd_buffer: type[NDBuffer] + + +# The default buffer prototype used throughout the Zarr codebase. +def default_buffer_prototype() -> BufferPrototype: + from zarr.registry import ( + get_buffer_class, + get_ndbuffer_class, + ) + + return BufferPrototype(buffer=get_buffer_class(), nd_buffer=get_ndbuffer_class()) diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py new file mode 100644 index 0000000000..3140d75111 --- /dev/null +++ b/src/zarr/core/buffer/cpu.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Any, + Literal, +) + +import numpy as np +import numpy.typing as npt + +from zarr.core.buffer import core +from zarr.registry import ( + register_buffer, + register_ndbuffer, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable + from typing import Self + + from zarr.core.buffer.core import ArrayLike, NDArrayLike + from zarr.core.common import BytesLike + + +class Buffer(core.Buffer): + """A flat contiguous memory block + + We use Buffer throughout Zarr to represent a contiguous block of memory. + + A Buffer is backed by a underlying array-like instance that represents + the memory. The memory type is unspecified; can be regular host memory, + CUDA device memory, or something else. The only requirement is that the + array-like instance can be copied/converted to a regular Numpy array + (host memory). + + Notes + ----- + This buffer is untyped, so all indexing and sizes are in bytes. + + Parameters + ---------- + array_like + array-like object that must be 1-dim, contiguous, and byte dtype. + """ + + def __init__(self, array_like: ArrayLike) -> None: + super().__init__(array_like) + + @classmethod + def create_zero_length(cls) -> Self: + return cls(np.array([], dtype="B")) + + @classmethod + def from_buffer(cls, buffer: core.Buffer) -> Self: + """Create a new buffer of an existing Buffer + + This is useful if you want to ensure that an existing buffer is + of the correct subclass of Buffer. E.g., MemoryStore uses this + to return a buffer instance of the subclass specified by its + BufferPrototype argument. + + Typically, this only copies data if the data has to be moved between + memory types, such as from host to device memory. + + Parameters + ---------- + buffer + buffer object. + + Returns + ------- + A new buffer representing the content of the input buffer + + Notes + ----- + Subclasses of `Buffer` must override this method to implement + more optimal conversions that avoid copies where possible + """ + return cls.from_array_like(buffer.as_numpy_array()) + + @classmethod + def from_bytes(cls, bytes_like: BytesLike) -> Self: + """Create a new buffer of a bytes-like object (host memory) + + Parameters + ---------- + bytes_like + bytes-like object + + Returns + ------- + New buffer representing `bytes_like` + """ + return cls.from_array_like(np.frombuffer(bytes_like, dtype="B")) + + def as_numpy_array(self) -> npt.NDArray[Any]: + """Returns the buffer as a NumPy array (host memory). + + Notes + ----- + Might have to copy data, consider using `.as_array_like()` instead. + + Returns + ------- + NumPy array of this buffer (might be a data copy) + """ + return np.asanyarray(self._data) + + def __add__(self, other: core.Buffer) -> Self: + """Concatenate two buffers""" + + other_array = other.as_array_like() + assert other_array.dtype == np.dtype("B") + return self.__class__( + np.concatenate((np.asanyarray(self._data), np.asanyarray(other_array))) + ) + + +class NDBuffer(core.NDBuffer): + """An n-dimensional memory block + + We use NDBuffer throughout Zarr to represent a n-dimensional memory block. + + A NDBuffer is backed by a underlying ndarray-like instance that represents + the memory. The memory type is unspecified; can be regular host memory, + CUDA device memory, or something else. The only requirement is that the + ndarray-like instance can be copied/converted to a regular Numpy array + (host memory). + + Notes + ----- + The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer + is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, + in order to use Python's type system to differentiate between the contiguous + Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the + two classes separate. + + Parameters + ---------- + array + ndarray-like object that is convertible to a regular Numpy array. + """ + + def __init__(self, array: NDArrayLike) -> None: + super().__init__(array) + + @classmethod + def create( + cls, + *, + shape: Iterable[int], + dtype: npt.DTypeLike, + order: Literal["C", "F"] = "C", + fill_value: Any | None = None, + ) -> Self: + # np.zeros is much faster than np.full, and therefore using it when possible is better. + if fill_value is None or (isinstance(fill_value, int) and fill_value == 0): + return cls(np.zeros(shape=tuple(shape), dtype=dtype, order=order)) + else: + return cls(np.full(shape=tuple(shape), fill_value=fill_value, dtype=dtype, order=order)) + + @classmethod + def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: + return cls.from_ndarray_like(np.asanyarray(array_like)) + + def as_numpy_array(self) -> npt.NDArray[Any]: + """Returns the buffer as a NumPy array (host memory). + + Warnings + -------- + Might have to copy data, consider using `.as_ndarray_like()` instead. + + Returns + ------- + NumPy array of this buffer (might be a data copy) + """ + return np.asanyarray(self._data) + + def __getitem__(self, key: Any) -> Self: + return self.__class__(np.asanyarray(self._data.__getitem__(key))) + + def __setitem__(self, key: Any, value: Any) -> None: + if isinstance(value, NDBuffer): + value = value._data + self._data.__setitem__(key, value) + + +def as_numpy_array_wrapper( + func: Callable[[npt.NDArray[Any]], bytes], buf: core.Buffer, prototype: core.BufferPrototype +) -> core.Buffer: + """Converts the input of `func` to a numpy array and the output back to `Buffer`. + + This function is useful when calling a `func` that only support host memory such + as `GZip.decode` and `Blosc.decode`. In this case, use this wrapper to convert + the input `buf` to a Numpy array and convert the result back into a `Buffer`. + + Parameters + ---------- + func + The callable that will be called with the converted `buf` as input. + `func` must return bytes, which will be converted into a `Buffer` + before returned. + buf + The buffer that will be converted to a Numpy array before given as + input to `func`. + prototype + The prototype of the output buffer. + + Returns + ------- + The result of `func` converted to a `Buffer` + """ + return prototype.buffer.from_bytes(func(buf.as_numpy_array())) + + +# CPU buffer prototype using numpy arrays +buffer_prototype = core.BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) +# default_buffer_prototype = buffer_prototype + + +# The numpy prototype used for E.g. when reading the shard index +def numpy_buffer_prototype() -> core.BufferPrototype: + return core.BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) + + +register_buffer(Buffer, qualname="zarr.buffer.cpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.buffer.cpu.NDBuffer") + + +# backwards compatibility +register_buffer(Buffer, qualname="zarr.core.buffer.cpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.core.buffer.cpu.NDBuffer") diff --git a/src/zarr/core/buffer/gpu.py b/src/zarr/core/buffer/gpu.py new file mode 100644 index 0000000000..7ea6d53fe3 --- /dev/null +++ b/src/zarr/core/buffer/gpu.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import warnings +from typing import ( + TYPE_CHECKING, + Any, + Literal, + cast, +) + +import numpy as np +import numpy.typing as npt + +from zarr.core.buffer import core +from zarr.core.buffer.core import ArrayLike, BufferPrototype, NDArrayLike +from zarr.registry import ( + register_buffer, + register_ndbuffer, +) + +if TYPE_CHECKING: + from collections.abc import Iterable + from typing import Self + + from zarr.core.common import BytesLike + +try: + import cupy as cp +except ImportError: + cp = None + + +class Buffer(core.Buffer): + """A flat contiguous memory block on the GPU + + We use Buffer throughout Zarr to represent a contiguous block of memory. + + A Buffer is backed by a underlying array-like instance that represents + the memory. The memory type is unspecified; can be regular host memory, + CUDA device memory, or something else. The only requirement is that the + array-like instance can be copied/converted to a regular Numpy array + (host memory). + + Notes + ----- + This buffer is untyped, so all indexing and sizes are in bytes. + + Parameters + ---------- + array_like + array-like object that must be 1-dim, contiguous, and byte dtype. + """ + + def __init__(self, array_like: ArrayLike) -> None: + if cp is None: + raise ImportError( + "Cannot use zarr.buffer.gpu.Buffer without cupy. Please install cupy." + ) + + if array_like.ndim != 1: + raise ValueError("array_like: only 1-dim allowed") + if array_like.dtype != np.dtype("B"): + raise ValueError("array_like: only byte dtype allowed") + + if not hasattr(array_like, "__cuda_array_interface__"): + # Slow copy based path for arrays that don't support the __cuda_array_interface__ + # TODO: Add a fast zero-copy path for arrays that support the dlpack protocol + msg = ( + "Creating a zarr.buffer.gpu.Buffer with an array that does not support the " + "__cuda_array_interface__ for zero-copy transfers, " + "falling back to slow copy based path" + ) + warnings.warn( + msg, + stacklevel=2, + ) + self._data = cp.asarray(array_like) + + @classmethod + def create_zero_length(cls) -> Self: + """Create an empty buffer with length zero + + Returns + ------- + New empty 0-length buffer + """ + return cls(cp.array([], dtype="B")) + + @classmethod + def from_buffer(cls, buffer: core.Buffer) -> Self: + """Create an GPU Buffer given an arbitrary Buffer + This will try to be zero-copy if `buffer` is already on the + GPU and will trigger a copy if not. + + Returns + ------- + New GPU Buffer constructed from `buffer` + """ + return cls(buffer.as_array_like()) + + @classmethod + def from_bytes(cls, bytes_like: BytesLike) -> Self: + return cls.from_array_like(cp.frombuffer(bytes_like, dtype="B")) + + def as_numpy_array(self) -> npt.NDArray[Any]: + return cast("npt.NDArray[Any]", cp.asnumpy(self._data)) + + def __add__(self, other: core.Buffer) -> Self: + other_array = other.as_array_like() + assert other_array.dtype == np.dtype("B") + gpu_other = Buffer(other_array) + gpu_other_array = gpu_other.as_array_like() + return self.__class__( + cp.concatenate((cp.asanyarray(self._data), cp.asanyarray(gpu_other_array))) + ) + + +class NDBuffer(core.NDBuffer): + """A n-dimensional memory block on the GPU + + We use NDBuffer throughout Zarr to represent a n-dimensional memory block. + + A NDBuffer is backed by a underlying ndarray-like instance that represents + the memory. The memory type is unspecified; can be regular host memory, + CUDA device memory, or something else. The only requirement is that the + ndarray-like instance can be copied/converted to a regular Numpy array + (host memory). + + Notes + ----- + The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer + is a special case of NDBuffer where dim=1, stride=1, and dtype="B". However, + in order to use Python's type system to differentiate between the contiguous + Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the + two classes separate. + + Parameters + ---------- + array + ndarray-like object that is convertible to a regular Numpy array. + """ + + def __init__(self, array: NDArrayLike) -> None: + if cp is None: + raise ImportError( + "Cannot use zarr.buffer.gpu.NDBuffer without cupy. Please install cupy." + ) + + # assert array.ndim > 0 + assert array.dtype != object + self._data = array + + if not hasattr(array, "__cuda_array_interface__"): + # Slow copy based path for arrays that don't support the __cuda_array_interface__ + # TODO: Add a fast zero-copy path for arrays that support the dlpack protocol + msg = ( + "Creating a zarr.buffer.gpu.NDBuffer with an array that does not support the " + "__cuda_array_interface__ for zero-copy transfers, " + "falling back to slow copy based path" + ) + warnings.warn( + msg, + stacklevel=2, + ) + self._data = cp.asarray(array) + + @classmethod + def create( + cls, + *, + shape: Iterable[int], + dtype: npt.DTypeLike, + order: Literal["C", "F"] = "C", + fill_value: Any | None = None, + ) -> Self: + ret = cls(cp.empty(shape=tuple(shape), dtype=dtype, order=order)) + if fill_value is not None: + ret.fill(fill_value) + return ret + + @classmethod + def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: + """Create a new buffer of Numpy array-like object + + Parameters + ---------- + array_like + Object that can be coerced into a Numpy array + + Returns + ------- + New buffer representing `array_like` + """ + return cls(cp.asarray(array_like)) + + def as_numpy_array(self) -> npt.NDArray[Any]: + """Returns the buffer as a NumPy array (host memory). + + Warnings + -------- + Might have to copy data, consider using `.as_ndarray_like()` instead. + + Returns + ------- + NumPy array of this buffer (might be a data copy) + """ + return cast("npt.NDArray[Any]", cp.asnumpy(self._data)) + + def __getitem__(self, key: Any) -> Self: + return self.__class__(self._data.__getitem__(key)) + + def __setitem__(self, key: Any, value: Any) -> None: + if isinstance(value, NDBuffer): + value = value._data + elif isinstance(value, core.NDBuffer): + gpu_value = NDBuffer(value.as_ndarray_like()) + value = gpu_value._data + self._data.__setitem__(key, value) + + +buffer_prototype = BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) + +register_buffer(Buffer, qualname="zarr.buffer.gpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.buffer.gpu.NDBuffer") + +# backwards compatibility +register_buffer(Buffer, qualname="zarr.core.buffer.gpu.Buffer") +register_ndbuffer(NDBuffer, qualname="zarr.core.buffer.gpu.NDBuffer") diff --git a/src/zarr/core/chunk_grids.py b/src/zarr/core/chunk_grids.py new file mode 100644 index 0000000000..4bf03c89de --- /dev/null +++ b/src/zarr/core/chunk_grids.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +import itertools +import math +import numbers +import operator +import warnings +from abc import abstractmethod +from dataclasses import dataclass +from functools import reduce +from typing import TYPE_CHECKING, Any, Literal + +import numpy as np + +from zarr.abc.metadata import Metadata +from zarr.core.common import ( + JSON, + ChunkCoords, + ChunkCoordsLike, + ShapeLike, + parse_named_configuration, + parse_shapelike, +) +from zarr.core.indexing import ceildiv + +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import Self + + from zarr.core.array import ShardsLike + + +def _guess_chunks( + shape: ShapeLike, + typesize: int, + *, + increment_bytes: int = 256 * 1024, + min_bytes: int = 128 * 1024, + max_bytes: int = 64 * 1024 * 1024, +) -> ChunkCoords: + """ + Iteratively guess an appropriate chunk layout for an array, given its shape and + the size of each element in bytes, and size constraints expressed in bytes. This logic is + adapted from h5py. + + Parameters + ---------- + shape : ChunkCoords + The chunk shape. + typesize : int + The size, in bytes, of each element of the chunk. + increment_bytes : int = 256 * 1024 + The number of bytes used to increment or decrement the target chunk size in bytes. + min_bytes : int = 128 * 1024 + The soft lower bound on the final chunk size in bytes. + max_bytes : int = 64 * 1024 * 1024 + The hard upper bound on the final chunk size in bytes. + + Returns + ------- + ChunkCoords + + """ + if isinstance(shape, int): + shape = (shape,) + + if typesize == 0: + return shape + + ndims = len(shape) + # require chunks to have non-zero length for all dimensions + chunks = np.maximum(np.array(shape, dtype="=f8"), 1) + + # Determine the optimal chunk size in bytes using a PyTables expression. + # This is kept as a float. + dset_size = np.prod(chunks) * typesize + target_size = increment_bytes * (2 ** np.log10(dset_size / (1024.0 * 1024))) + + if target_size > max_bytes: + target_size = max_bytes + elif target_size < min_bytes: + target_size = min_bytes + + idx = 0 + while True: + # Repeatedly loop over the axes, dividing them by 2. Stop when: + # 1a. We're smaller than the target chunk size, OR + # 1b. We're within 50% of the target chunk size, AND + # 2. The chunk is smaller than the maximum chunk size + + chunk_bytes = np.prod(chunks) * typesize + + if ( + chunk_bytes < target_size or abs(chunk_bytes - target_size) / target_size < 0.5 + ) and chunk_bytes < max_bytes: + break + + if np.prod(chunks) == 1: + break # Element size larger than max_bytes + + chunks[idx % ndims] = math.ceil(chunks[idx % ndims] / 2.0) + idx += 1 + + return tuple(int(x) for x in chunks) + + +def normalize_chunks(chunks: Any, shape: tuple[int, ...], typesize: int) -> tuple[int, ...]: + """Convenience function to normalize the `chunks` argument for an array + with the given `shape`.""" + + # N.B., expect shape already normalized + + # handle auto-chunking + if chunks is None or chunks is True: + return _guess_chunks(shape, typesize) + + # handle no chunking + if chunks is False: + return shape + + # handle 1D convenience form + if isinstance(chunks, numbers.Integral): + chunks = tuple(int(chunks) for _ in shape) + + # handle dask-style chunks (iterable of iterables) + if all(isinstance(c, (tuple | list)) for c in chunks): + # take first chunk size for each dimension + chunks = tuple( + c[0] for c in chunks + ) # TODO: check/error/warn for irregular chunks (e.g. if c[0] != c[1:-1]) + + # handle bad dimensionality + if len(chunks) > len(shape): + raise ValueError("too many dimensions in chunks") + + # handle underspecified chunks + if len(chunks) < len(shape): + # assume chunks across remaining dimensions + chunks += shape[len(chunks) :] + + # handle None or -1 in chunks + if -1 in chunks or None in chunks: + chunks = tuple( + s if c == -1 or c is None else int(c) for s, c in zip(shape, chunks, strict=False) + ) + + if not all(isinstance(c, numbers.Integral) for c in chunks): + raise TypeError("non integer value in chunks") + + return tuple(int(c) for c in chunks) + + +@dataclass(frozen=True) +class ChunkGrid(Metadata): + @classmethod + def from_dict(cls, data: dict[str, JSON] | ChunkGrid) -> ChunkGrid: + if isinstance(data, ChunkGrid): + return data + + name_parsed, _ = parse_named_configuration(data) + if name_parsed == "regular": + return RegularChunkGrid._from_dict(data) + raise ValueError(f"Unknown chunk grid. Got {name_parsed}.") + + @abstractmethod + def all_chunk_coords(self, array_shape: ChunkCoords) -> Iterator[ChunkCoords]: + pass + + @abstractmethod + def get_nchunks(self, array_shape: ChunkCoords) -> int: + pass + + +@dataclass(frozen=True) +class RegularChunkGrid(ChunkGrid): + chunk_shape: ChunkCoords + + def __init__(self, *, chunk_shape: ChunkCoordsLike) -> None: + chunk_shape_parsed = parse_shapelike(chunk_shape) + + object.__setattr__(self, "chunk_shape", chunk_shape_parsed) + + @classmethod + def _from_dict(cls, data: dict[str, JSON]) -> Self: + _, configuration_parsed = parse_named_configuration(data, "regular") + + return cls(**configuration_parsed) # type: ignore[arg-type] + + def to_dict(self) -> dict[str, JSON]: + return {"name": "regular", "configuration": {"chunk_shape": tuple(self.chunk_shape)}} + + def all_chunk_coords(self, array_shape: ChunkCoords) -> Iterator[ChunkCoords]: + return itertools.product( + *(range(ceildiv(s, c)) for s, c in zip(array_shape, self.chunk_shape, strict=False)) + ) + + def get_nchunks(self, array_shape: ChunkCoords) -> int: + return reduce( + operator.mul, + itertools.starmap(ceildiv, zip(array_shape, self.chunk_shape, strict=True)), + 1, + ) + + +def _auto_partition( + *, + array_shape: tuple[int, ...], + chunk_shape: tuple[int, ...] | Literal["auto"], + shard_shape: ShardsLike | None, + item_size: int, +) -> tuple[tuple[int, ...] | None, tuple[int, ...]]: + """ + Automatically determine the shard shape and chunk shape for an array, given the shape and dtype of the array. + If `shard_shape` is `None` and the chunk_shape is "auto", the chunks will be set heuristically based + on the dtype and shape of the array. + If `shard_shape` is "auto", then the shard shape will be set heuristically from the dtype and shape + of the array; if the `chunk_shape` is also "auto", then the chunks will be set heuristically as well, + given the dtype and shard shape. Otherwise, the chunks will be returned as-is. + """ + if shard_shape is None: + _shards_out: None | tuple[int, ...] = None + if chunk_shape == "auto": + _chunks_out = _guess_chunks(array_shape, item_size) + else: + _chunks_out = chunk_shape + else: + if chunk_shape == "auto": + # aim for a 1MiB chunk + _chunks_out = _guess_chunks(array_shape, item_size, max_bytes=1024) + else: + _chunks_out = chunk_shape + + if shard_shape == "auto": + warnings.warn( + "Automatic shard shape inference is experimental and may change without notice.", + UserWarning, + stacklevel=2, + ) + _shards_out = () + for a_shape, c_shape in zip(array_shape, _chunks_out, strict=True): + # TODO: make a better heuristic than this. + # for each axis, if there are more than 8 chunks along that axis, then put + # 2 chunks in each shard for that axis. + if a_shape // c_shape > 8: + _shards_out += (c_shape * 2,) + else: + _shards_out += (c_shape,) + elif isinstance(shard_shape, dict): + _shards_out = tuple(shard_shape["shape"]) + else: + _shards_out = shard_shape + + return _shards_out, _chunks_out diff --git a/src/zarr/core/chunk_key_encodings.py b/src/zarr/core/chunk_key_encodings.py new file mode 100644 index 0000000000..91dfc90365 --- /dev/null +++ b/src/zarr/core/chunk_key_encodings.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from abc import abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal, TypeAlias, TypedDict, cast + +if TYPE_CHECKING: + from typing import NotRequired + +from zarr.abc.metadata import Metadata +from zarr.core.common import ( + JSON, + ChunkCoords, + parse_named_configuration, +) + +SeparatorLiteral = Literal[".", "/"] + + +def parse_separator(data: JSON) -> SeparatorLiteral: + if data not in (".", "/"): + raise ValueError(f"Expected an '.' or '/' separator. Got {data} instead.") + return cast("SeparatorLiteral", data) + + +class ChunkKeyEncodingParams(TypedDict): + name: Literal["v2", "default"] + separator: NotRequired[SeparatorLiteral] + + +@dataclass(frozen=True) +class ChunkKeyEncoding(Metadata): + name: str + separator: SeparatorLiteral = "." + + def __init__(self, *, separator: SeparatorLiteral) -> None: + separator_parsed = parse_separator(separator) + + object.__setattr__(self, "separator", separator_parsed) + + @classmethod + def from_dict(cls, data: dict[str, JSON] | ChunkKeyEncodingLike) -> ChunkKeyEncoding: + if isinstance(data, ChunkKeyEncoding): + return data + + # handle ChunkKeyEncodingParams + if "name" in data and "separator" in data: + data = {"name": data["name"], "configuration": {"separator": data["separator"]}} + + # TODO: remove this cast when we are statically typing the JSON metadata completely. + data = cast("dict[str, JSON]", data) + + # configuration is optional for chunk key encodings + name_parsed, config_parsed = parse_named_configuration(data, require_configuration=False) + if name_parsed == "default": + if config_parsed is None: + # for default, normalize missing configuration to use the "/" separator. + config_parsed = {"separator": "/"} + return DefaultChunkKeyEncoding(**config_parsed) # type: ignore[arg-type] + if name_parsed == "v2": + if config_parsed is None: + # for v2, normalize missing configuration to use the "." separator. + config_parsed = {"separator": "."} + return V2ChunkKeyEncoding(**config_parsed) # type: ignore[arg-type] + msg = f"Unknown chunk key encoding. Got {name_parsed}, expected one of ('v2', 'default')." + raise ValueError(msg) + + def to_dict(self) -> dict[str, JSON]: + return {"name": self.name, "configuration": {"separator": self.separator}} + + @abstractmethod + def decode_chunk_key(self, chunk_key: str) -> ChunkCoords: + pass + + @abstractmethod + def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: + pass + + +ChunkKeyEncodingLike: TypeAlias = ChunkKeyEncodingParams | ChunkKeyEncoding + + +@dataclass(frozen=True) +class DefaultChunkKeyEncoding(ChunkKeyEncoding): + name: Literal["default"] = "default" + + def decode_chunk_key(self, chunk_key: str) -> ChunkCoords: + if chunk_key == "c": + return () + return tuple(map(int, chunk_key[1:].split(self.separator))) + + def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: + return self.separator.join(map(str, ("c",) + chunk_coords)) + + +@dataclass(frozen=True) +class V2ChunkKeyEncoding(ChunkKeyEncoding): + name: Literal["v2"] = "v2" + + def decode_chunk_key(self, chunk_key: str) -> ChunkCoords: + return tuple(map(int, chunk_key.split(self.separator))) + + def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: + chunk_identifier = self.separator.join(map(str, chunk_coords)) + return "0" if chunk_identifier == "" else chunk_identifier diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py new file mode 100644 index 0000000000..23c27e40c6 --- /dev/null +++ b/src/zarr/core/codec_pipeline.py @@ -0,0 +1,552 @@ +from __future__ import annotations + +from dataclasses import dataclass +from itertools import islice, pairwise +from typing import TYPE_CHECKING, Any, TypeVar +from warnings import warn + +from zarr.abc.codec import ( + ArrayArrayCodec, + ArrayBytesCodec, + ArrayBytesCodecPartialDecodeMixin, + ArrayBytesCodecPartialEncodeMixin, + BytesBytesCodec, + Codec, + CodecPipeline, +) +from zarr.core.common import ChunkCoords, concurrent_map +from zarr.core.config import config +from zarr.core.indexing import SelectorTuple, is_scalar +from zarr.registry import register_pipeline + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + from typing import Self + + from zarr.abc.store import ByteGetter, ByteSetter + from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer, BufferPrototype, NDBuffer + from zarr.core.chunk_grids import ChunkGrid + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + +T = TypeVar("T") +U = TypeVar("U") + + +def _unzip2(iterable: Iterable[tuple[T, U]]) -> tuple[list[T], list[U]]: + out0: list[T] = [] + out1: list[U] = [] + for item0, item1 in iterable: + out0.append(item0) + out1.append(item1) + return (out0, out1) + + +def batched(iterable: Iterable[T], n: int) -> Iterable[tuple[T, ...]]: + if n < 1: + raise ValueError("n must be at least one") + it = iter(iterable) + while batch := tuple(islice(it, n)): + yield batch + + +def resolve_batched(codec: Codec, chunk_specs: Iterable[ArraySpec]) -> Iterable[ArraySpec]: + return [codec.resolve_metadata(chunk_spec) for chunk_spec in chunk_specs] + + +def fill_value_or_default(chunk_spec: ArraySpec) -> Any: + fill_value = chunk_spec.fill_value + if fill_value is None: + # Zarr V2 allowed `fill_value` to be null in the metadata. + # Zarr V3 requires it to be set. This has already been + # validated when decoding the metadata, but we support reading + # Zarr V2 data and need to support the case where fill_value + # is None. + return chunk_spec.dtype.default_scalar() + else: + return fill_value + + +@dataclass(frozen=True) +class BatchedCodecPipeline(CodecPipeline): + """Default codec pipeline. + + This batched codec pipeline divides the chunk batches into batches of a configurable + batch size ("mini-batch"). Fetching, decoding, encoding and storing are performed in + lock step for each mini-batch. Multiple mini-batches are processing concurrently. + """ + + array_array_codecs: tuple[ArrayArrayCodec, ...] + array_bytes_codec: ArrayBytesCodec + bytes_bytes_codecs: tuple[BytesBytesCodec, ...] + batch_size: int + + def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: + return type(self).from_codecs(c.evolve_from_array_spec(array_spec=array_spec) for c in self) + + @classmethod + def from_codecs(cls, codecs: Iterable[Codec], *, batch_size: int | None = None) -> Self: + array_array_codecs, array_bytes_codec, bytes_bytes_codecs = codecs_from_list(codecs) + + return cls( + array_array_codecs=array_array_codecs, + array_bytes_codec=array_bytes_codec, + bytes_bytes_codecs=bytes_bytes_codecs, + batch_size=batch_size or config.get("codec_pipeline.batch_size"), + ) + + @property + def supports_partial_decode(self) -> bool: + """Determines whether the codec pipeline supports partial decoding. + + Currently, only codec pipelines with a single ArrayBytesCodec that supports + partial decoding can support partial decoding. This limitation is due to the fact + that ArrayArrayCodecs can change the slice selection leading to non-contiguous + slices and BytesBytesCodecs can change the chunk bytes in a way that slice + selections cannot be attributed to byte ranges anymore which renders partial + decoding infeasible. + + This limitation may softened in the future.""" + return (len(self.array_array_codecs) + len(self.bytes_bytes_codecs)) == 0 and isinstance( + self.array_bytes_codec, ArrayBytesCodecPartialDecodeMixin + ) + + @property + def supports_partial_encode(self) -> bool: + """Determines whether the codec pipeline supports partial encoding. + + Currently, only codec pipelines with a single ArrayBytesCodec that supports + partial encoding can support partial encoding. This limitation is due to the fact + that ArrayArrayCodecs can change the slice selection leading to non-contiguous + slices and BytesBytesCodecs can change the chunk bytes in a way that slice + selections cannot be attributed to byte ranges anymore which renders partial + encoding infeasible. + + This limitation may softened in the future.""" + return (len(self.array_array_codecs) + len(self.bytes_bytes_codecs)) == 0 and isinstance( + self.array_bytes_codec, ArrayBytesCodecPartialEncodeMixin + ) + + def __iter__(self) -> Iterator[Codec]: + yield from self.array_array_codecs + yield self.array_bytes_codec + yield from self.bytes_bytes_codecs + + def validate( + self, *, shape: ChunkCoords, dtype: ZDType[TBaseDType, TBaseScalar], chunk_grid: ChunkGrid + ) -> None: + for codec in self: + codec.validate(shape=shape, dtype=dtype, chunk_grid=chunk_grid) + + def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int: + for codec in self: + byte_length = codec.compute_encoded_size(byte_length, array_spec) + array_spec = codec.resolve_metadata(array_spec) + return byte_length + + def _codecs_with_resolved_metadata_batched( + self, chunk_specs: Iterable[ArraySpec] + ) -> tuple[ + list[tuple[ArrayArrayCodec, list[ArraySpec]]], + tuple[ArrayBytesCodec, list[ArraySpec]], + list[tuple[BytesBytesCodec, list[ArraySpec]]], + ]: + aa_codecs_with_spec: list[tuple[ArrayArrayCodec, list[ArraySpec]]] = [] + chunk_specs = list(chunk_specs) + for aa_codec in self.array_array_codecs: + aa_codecs_with_spec.append((aa_codec, chunk_specs)) + chunk_specs = [aa_codec.resolve_metadata(chunk_spec) for chunk_spec in chunk_specs] + + ab_codec_with_spec = (self.array_bytes_codec, chunk_specs) + chunk_specs = [ + self.array_bytes_codec.resolve_metadata(chunk_spec) for chunk_spec in chunk_specs + ] + + bb_codecs_with_spec: list[tuple[BytesBytesCodec, list[ArraySpec]]] = [] + for bb_codec in self.bytes_bytes_codecs: + bb_codecs_with_spec.append((bb_codec, chunk_specs)) + chunk_specs = [bb_codec.resolve_metadata(chunk_spec) for chunk_spec in chunk_specs] + + return (aa_codecs_with_spec, ab_codec_with_spec, bb_codecs_with_spec) + + async def decode_batch( + self, + chunk_bytes_and_specs: Iterable[tuple[Buffer | None, ArraySpec]], + ) -> Iterable[NDBuffer | None]: + chunk_bytes_batch: Iterable[Buffer | None] + chunk_bytes_batch, chunk_specs = _unzip2(chunk_bytes_and_specs) + ( + aa_codecs_with_spec, + ab_codec_with_spec, + bb_codecs_with_spec, + ) = self._codecs_with_resolved_metadata_batched(chunk_specs) + + for bb_codec, chunk_spec_batch in bb_codecs_with_spec[::-1]: + chunk_bytes_batch = await bb_codec.decode( + zip(chunk_bytes_batch, chunk_spec_batch, strict=False) + ) + + ab_codec, chunk_spec_batch = ab_codec_with_spec + chunk_array_batch = await ab_codec.decode( + zip(chunk_bytes_batch, chunk_spec_batch, strict=False) + ) + + for aa_codec, chunk_spec_batch in aa_codecs_with_spec[::-1]: + chunk_array_batch = await aa_codec.decode( + zip(chunk_array_batch, chunk_spec_batch, strict=False) + ) + + return chunk_array_batch + + async def decode_partial_batch( + self, + batch_info: Iterable[tuple[ByteGetter, SelectorTuple, ArraySpec]], + ) -> Iterable[NDBuffer | None]: + assert self.supports_partial_decode + assert isinstance(self.array_bytes_codec, ArrayBytesCodecPartialDecodeMixin) + return await self.array_bytes_codec.decode_partial(batch_info) + + async def encode_batch( + self, + chunk_arrays_and_specs: Iterable[tuple[NDBuffer | None, ArraySpec]], + ) -> Iterable[Buffer | None]: + chunk_array_batch: Iterable[NDBuffer | None] + chunk_specs: Iterable[ArraySpec] + chunk_array_batch, chunk_specs = _unzip2(chunk_arrays_and_specs) + + for aa_codec in self.array_array_codecs: + chunk_array_batch = await aa_codec.encode( + zip(chunk_array_batch, chunk_specs, strict=False) + ) + chunk_specs = resolve_batched(aa_codec, chunk_specs) + + chunk_bytes_batch = await self.array_bytes_codec.encode( + zip(chunk_array_batch, chunk_specs, strict=False) + ) + chunk_specs = resolve_batched(self.array_bytes_codec, chunk_specs) + + for bb_codec in self.bytes_bytes_codecs: + chunk_bytes_batch = await bb_codec.encode( + zip(chunk_bytes_batch, chunk_specs, strict=False) + ) + chunk_specs = resolve_batched(bb_codec, chunk_specs) + + return chunk_bytes_batch + + async def encode_partial_batch( + self, + batch_info: Iterable[tuple[ByteSetter, NDBuffer, SelectorTuple, ArraySpec]], + ) -> None: + assert self.supports_partial_encode + assert isinstance(self.array_bytes_codec, ArrayBytesCodecPartialEncodeMixin) + await self.array_bytes_codec.encode_partial(batch_info) + + async def read_batch( + self, + batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], + out: NDBuffer, + drop_axes: tuple[int, ...] = (), + ) -> None: + if self.supports_partial_decode: + chunk_array_batch = await self.decode_partial_batch( + [ + (byte_getter, chunk_selection, chunk_spec) + for byte_getter, chunk_spec, chunk_selection, *_ in batch_info + ] + ) + for chunk_array, (_, chunk_spec, _, out_selection, _) in zip( + chunk_array_batch, batch_info, strict=False + ): + if chunk_array is not None: + out[out_selection] = chunk_array + else: + out[out_selection] = fill_value_or_default(chunk_spec) + else: + chunk_bytes_batch = await concurrent_map( + [(byte_getter, array_spec.prototype) for byte_getter, array_spec, *_ in batch_info], + lambda byte_getter, prototype: byte_getter.get(prototype), + config.get("async.concurrency"), + ) + chunk_array_batch = await self.decode_batch( + [ + (chunk_bytes, chunk_spec) + for chunk_bytes, (_, chunk_spec, *_) in zip( + chunk_bytes_batch, batch_info, strict=False + ) + ], + ) + for chunk_array, (_, chunk_spec, chunk_selection, out_selection, _) in zip( + chunk_array_batch, batch_info, strict=False + ): + if chunk_array is not None: + tmp = chunk_array[chunk_selection] + if drop_axes != (): + tmp = tmp.squeeze(axis=drop_axes) + out[out_selection] = tmp + else: + out[out_selection] = fill_value_or_default(chunk_spec) + + def _merge_chunk_array( + self, + existing_chunk_array: NDBuffer | None, + value: NDBuffer, + out_selection: SelectorTuple, + chunk_spec: ArraySpec, + chunk_selection: SelectorTuple, + is_complete_chunk: bool, + drop_axes: tuple[int, ...], + ) -> NDBuffer: + if chunk_selection == () or is_scalar( + value.as_ndarray_like(), chunk_spec.dtype.to_native_dtype() + ): + chunk_value = value + else: + chunk_value = value[out_selection] + # handle missing singleton dimensions + if drop_axes != (): + item = tuple( + None # equivalent to np.newaxis + if idx in drop_axes + else slice(None) + for idx in range(chunk_spec.ndim) + ) + chunk_value = chunk_value[item] + if is_complete_chunk and chunk_value.shape == chunk_spec.shape: + # TODO: For the last chunk, we could have is_complete_chunk=True + # that is smaller than the chunk_spec.shape but this throws + # an error in the _decode_single + return chunk_value + if existing_chunk_array is None: + chunk_array = chunk_spec.prototype.nd_buffer.create( + shape=chunk_spec.shape, + dtype=chunk_spec.dtype.to_native_dtype(), + order=chunk_spec.order, + fill_value=fill_value_or_default(chunk_spec), + ) + else: + chunk_array = existing_chunk_array.copy() # make a writable copy + chunk_array[chunk_selection] = chunk_value + return chunk_array + + async def write_batch( + self, + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], + value: NDBuffer, + drop_axes: tuple[int, ...] = (), + ) -> None: + if self.supports_partial_encode: + # Pass scalar values as is + if len(value.shape) == 0: + await self.encode_partial_batch( + [ + (byte_setter, value, chunk_selection, chunk_spec) + for byte_setter, chunk_spec, chunk_selection, out_selection, _ in batch_info + ], + ) + else: + await self.encode_partial_batch( + [ + (byte_setter, value[out_selection], chunk_selection, chunk_spec) + for byte_setter, chunk_spec, chunk_selection, out_selection, _ in batch_info + ], + ) + + else: + # Read existing bytes if not total slice + async def _read_key( + byte_setter: ByteSetter | None, prototype: BufferPrototype + ) -> Buffer | None: + if byte_setter is None: + return None + return await byte_setter.get(prototype=prototype) + + chunk_bytes_batch: Iterable[Buffer | None] + chunk_bytes_batch = await concurrent_map( + [ + ( + None if is_complete_chunk else byte_setter, + chunk_spec.prototype, + ) + for byte_setter, chunk_spec, chunk_selection, _, is_complete_chunk in batch_info + ], + _read_key, + config.get("async.concurrency"), + ) + chunk_array_decoded = await self.decode_batch( + [ + (chunk_bytes, chunk_spec) + for chunk_bytes, (_, chunk_spec, *_) in zip( + chunk_bytes_batch, batch_info, strict=False + ) + ], + ) + + chunk_array_merged = [ + self._merge_chunk_array( + chunk_array, + value, + out_selection, + chunk_spec, + chunk_selection, + is_complete_chunk, + drop_axes, + ) + for chunk_array, ( + _, + chunk_spec, + chunk_selection, + out_selection, + is_complete_chunk, + ) in zip(chunk_array_decoded, batch_info, strict=False) + ] + chunk_array_batch: list[NDBuffer | None] = [] + for chunk_array, (_, chunk_spec, *_) in zip( + chunk_array_merged, batch_info, strict=False + ): + if chunk_array is None: + chunk_array_batch.append(None) # type: ignore[unreachable] + else: + if not chunk_spec.config.write_empty_chunks and chunk_array.all_equal( + fill_value_or_default(chunk_spec) + ): + chunk_array_batch.append(None) + else: + chunk_array_batch.append(chunk_array) + + chunk_bytes_batch = await self.encode_batch( + [ + (chunk_array, chunk_spec) + for chunk_array, (_, chunk_spec, *_) in zip( + chunk_array_batch, batch_info, strict=False + ) + ], + ) + + async def _write_key(byte_setter: ByteSetter, chunk_bytes: Buffer | None) -> None: + if chunk_bytes is None: + await byte_setter.delete() + else: + await byte_setter.set(chunk_bytes) + + await concurrent_map( + [ + (byte_setter, chunk_bytes) + for chunk_bytes, (byte_setter, *_) in zip( + chunk_bytes_batch, batch_info, strict=False + ) + ], + _write_key, + config.get("async.concurrency"), + ) + + async def decode( + self, + chunk_bytes_and_specs: Iterable[tuple[Buffer | None, ArraySpec]], + ) -> Iterable[NDBuffer | None]: + output: list[NDBuffer | None] = [] + for batch_info in batched(chunk_bytes_and_specs, self.batch_size): + output.extend(await self.decode_batch(batch_info)) + return output + + async def encode( + self, + chunk_arrays_and_specs: Iterable[tuple[NDBuffer | None, ArraySpec]], + ) -> Iterable[Buffer | None]: + output: list[Buffer | None] = [] + for single_batch_info in batched(chunk_arrays_and_specs, self.batch_size): + output.extend(await self.encode_batch(single_batch_info)) + return output + + async def read( + self, + batch_info: Iterable[tuple[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], + out: NDBuffer, + drop_axes: tuple[int, ...] = (), + ) -> None: + await concurrent_map( + [ + (single_batch_info, out, drop_axes) + for single_batch_info in batched(batch_info, self.batch_size) + ], + self.read_batch, + config.get("async.concurrency"), + ) + + async def write( + self, + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], + value: NDBuffer, + drop_axes: tuple[int, ...] = (), + ) -> None: + await concurrent_map( + [ + (single_batch_info, value, drop_axes) + for single_batch_info in batched(batch_info, self.batch_size) + ], + self.write_batch, + config.get("async.concurrency"), + ) + + +def codecs_from_list( + codecs: Iterable[Codec], +) -> tuple[tuple[ArrayArrayCodec, ...], ArrayBytesCodec, tuple[BytesBytesCodec, ...]]: + from zarr.codecs.sharding import ShardingCodec + + array_array: tuple[ArrayArrayCodec, ...] = () + array_bytes_maybe: ArrayBytesCodec | None = None + bytes_bytes: tuple[BytesBytesCodec, ...] = () + + if any(isinstance(codec, ShardingCodec) for codec in codecs) and len(tuple(codecs)) > 1: + warn( + "Combining a `sharding_indexed` codec disables partial reads and " + "writes, which may lead to inefficient performance.", + stacklevel=3, + ) + + for prev_codec, cur_codec in pairwise((None, *codecs)): + if isinstance(cur_codec, ArrayArrayCodec): + if isinstance(prev_codec, ArrayBytesCodec | BytesBytesCodec): + msg = ( + f"Invalid codec order. ArrayArrayCodec {cur_codec}" + "must be preceded by another ArrayArrayCodec. " + f"Got {type(prev_codec)} instead." + ) + raise TypeError(msg) + array_array += (cur_codec,) + + elif isinstance(cur_codec, ArrayBytesCodec): + if isinstance(prev_codec, BytesBytesCodec): + msg = ( + f"Invalid codec order. ArrayBytes codec {cur_codec}" + f" must be preceded by an ArrayArrayCodec. Got {type(prev_codec)} instead." + ) + raise TypeError(msg) + + if array_bytes_maybe is not None: + msg = ( + f"Got two instances of ArrayBytesCodec: {array_bytes_maybe} and {cur_codec}. " + "Only one array-to-bytes codec is allowed." + ) + raise ValueError(msg) + + array_bytes_maybe = cur_codec + + elif isinstance(cur_codec, BytesBytesCodec): + if isinstance(prev_codec, ArrayArrayCodec): + msg = ( + f"Invalid codec order. BytesBytesCodec {cur_codec}" + "must be preceded by either another BytesBytesCodec, or an ArrayBytesCodec. " + f"Got {type(prev_codec)} instead." + ) + bytes_bytes += (cur_codec,) + else: + raise TypeError + + if array_bytes_maybe is None: + raise ValueError("Required ArrayBytesCodec was not found.") + else: + return array_array, array_bytes_maybe, bytes_bytes + + +register_pipeline(BatchedCodecPipeline) diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py new file mode 100644 index 0000000000..2ba5914ea5 --- /dev/null +++ b/src/zarr/core/common.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import asyncio +import functools +import operator +import warnings +from collections.abc import Iterable, Mapping, Sequence +from enum import Enum +from itertools import starmap +from typing import ( + TYPE_CHECKING, + Any, + Generic, + Literal, + TypedDict, + TypeVar, + cast, + overload, +) + +from zarr.core.config import config as zarr_config + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Iterator + + +ZARR_JSON = "zarr.json" +ZARRAY_JSON = ".zarray" +ZGROUP_JSON = ".zgroup" +ZATTRS_JSON = ".zattrs" +ZMETADATA_V2_JSON = ".zmetadata" + +BytesLike = bytes | bytearray | memoryview +ShapeLike = tuple[int, ...] | int +ChunkCoords = tuple[int, ...] +ChunkCoordsLike = Iterable[int] +ZarrFormat = Literal[2, 3] +NodeType = Literal["array", "group"] +JSON = str | int | float | Mapping[str, "JSON"] | Sequence["JSON"] | None +MemoryOrder = Literal["C", "F"] +AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"] +DimensionNames = Iterable[str | None] | None + +TName = TypeVar("TName", bound=str) +TConfig = TypeVar("TConfig", bound=Mapping[str, object]) + + +class NamedConfig(TypedDict, Generic[TName, TConfig]): + name: TName + configuration: TConfig + + +def product(tup: ChunkCoords) -> int: + return functools.reduce(operator.mul, tup, 1) + + +T = TypeVar("T", bound=tuple[Any, ...]) +V = TypeVar("V") + + +async def concurrent_map( + items: Iterable[T], + func: Callable[..., Awaitable[V]], + limit: int | None = None, +) -> list[V]: + if limit is None: + return await asyncio.gather(*list(starmap(func, items))) + + else: + sem = asyncio.Semaphore(limit) + + async def run(item: tuple[Any]) -> V: + async with sem: + return await func(*item) + + return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items]) + + +E = TypeVar("E", bound=Enum) + + +def enum_names(enum: type[E]) -> Iterator[str]: + for item in enum: + yield item.name + + +def parse_enum(data: object, cls: type[E]) -> E: + if isinstance(data, cls): + return data + if not isinstance(data, str): + raise TypeError(f"Expected str, got {type(data)}") + if data in enum_names(cls): + return cls(data) + raise ValueError(f"Value must be one of {list(enum_names(cls))!r}. Got {data} instead.") + + +def parse_name(data: JSON, expected: str | None = None) -> str: + if isinstance(data, str): + if expected is None or data == expected: + return data + raise ValueError(f"Expected '{expected}'. Got {data} instead.") + else: + raise TypeError(f"Expected a string, got an instance of {type(data)}.") + + +def parse_configuration(data: JSON) -> JSON: + if not isinstance(data, dict): + raise TypeError(f"Expected dict, got {type(data)}") + return data + + +@overload +def parse_named_configuration( + data: JSON, expected_name: str | None = None +) -> tuple[str, dict[str, JSON]]: ... + + +@overload +def parse_named_configuration( + data: JSON, expected_name: str | None = None, *, require_configuration: bool = True +) -> tuple[str, dict[str, JSON] | None]: ... + + +def parse_named_configuration( + data: JSON, expected_name: str | None = None, *, require_configuration: bool = True +) -> tuple[str, JSON | None]: + if not isinstance(data, dict): + raise TypeError(f"Expected dict, got {type(data)}") + if "name" not in data: + raise ValueError(f"Named configuration does not have a 'name' key. Got {data}.") + name_parsed = parse_name(data["name"], expected_name) + if "configuration" in data: + configuration_parsed = parse_configuration(data["configuration"]) + elif require_configuration: + raise ValueError(f"Named configuration does not have a 'configuration' key. Got {data}.") + else: + configuration_parsed = None + return name_parsed, configuration_parsed + + +def parse_shapelike(data: int | Iterable[int]) -> tuple[int, ...]: + if isinstance(data, int): + if data < 0: + raise ValueError(f"Expected a non-negative integer. Got {data} instead") + return (data,) + try: + data_tuple = tuple(data) + except TypeError as e: + msg = f"Expected an integer or an iterable of integers. Got {data} instead." + raise TypeError(msg) from e + + if not all(isinstance(v, int) for v in data_tuple): + msg = f"Expected an iterable of integers. Got {data} instead." + raise TypeError(msg) + if not all(v > -1 for v in data_tuple): + msg = f"Expected all values to be non-negative. Got {data} instead." + raise ValueError(msg) + return data_tuple + + +def parse_fill_value(data: Any) -> Any: + # todo: real validation + return data + + +def parse_order(data: Any) -> Literal["C", "F"]: + if data in ("C", "F"): + return cast("Literal['C', 'F']", data) + raise ValueError(f"Expected one of ('C', 'F'), got {data} instead.") + + +def parse_bool(data: Any) -> bool: + if isinstance(data, bool): + return data + raise ValueError(f"Expected bool, got {data} instead.") + + +def _warn_write_empty_chunks_kwarg() -> None: + # TODO: link to docs page on array configuration in this message + msg = ( + "The `write_empty_chunks` keyword argument is deprecated and will be removed in future versions. " + "To control whether empty chunks are written to storage, either use the `config` keyword " + "argument, as in `config={'write_empty_chunks: True}`," + "or change the global 'array.write_empty_chunks' configuration variable." + ) + warnings.warn(msg, RuntimeWarning, stacklevel=2) + + +def _warn_order_kwarg() -> None: + # TODO: link to docs page on array configuration in this message + msg = ( + "The `order` keyword argument has no effect for Zarr format 3 arrays. " + "To control the memory layout of the array, either use the `config` keyword " + "argument, as in `config={'order: 'C'}`," + "or change the global 'array.order' configuration variable." + ) + warnings.warn(msg, RuntimeWarning, stacklevel=2) + + +def _default_zarr_format() -> ZarrFormat: + """Return the default zarr_version""" + return cast("ZarrFormat", int(zarr_config.get("default_zarr_format", 3))) diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py new file mode 100644 index 0000000000..05d048ef74 --- /dev/null +++ b/src/zarr/core/config.py @@ -0,0 +1,156 @@ +""" +The config module is responsible for managing the configuration of zarr and is based on the Donfig python library. +For selecting custom implementations of codecs, pipelines, buffers and ndbuffers, first register the implementations +in the registry and then select them in the config. + +Example: + An implementation of the bytes codec in a class ``your.module.NewBytesCodec`` requires the value of ``codecs.bytes`` + to be ``your.module.NewBytesCodec``. Donfig can be configured programmatically, by environment variables, or from + YAML files in standard locations. + + .. code-block:: python + + from your.module import NewBytesCodec + from zarr.core.config import register_codec, config + + register_codec("bytes", NewBytesCodec) + config.set({"codecs.bytes": "your.module.NewBytesCodec"}) + + Instead of setting the value programmatically with ``config.set``, you can also set the value with an environment + variable. The environment variable ``ZARR_CODECS__BYTES`` can be set to ``your.module.NewBytesCodec``. The double + underscore ``__`` is used to indicate nested access. + + .. code-block:: bash + + export ZARR_CODECS__BYTES="your.module.NewBytesCodec" + +For more information, see the Donfig documentation at https://github.com/pytroll/donfig. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Literal, cast + +from donfig import Config as DConfig + +if TYPE_CHECKING: + from donfig.config_obj import ConfigSet + + from zarr.core.dtype.wrapper import ZDType + + +class BadConfigError(ValueError): + _msg = "bad Config: %r" + + +# These values are used for rough categorization of data types +# we use this for choosing a default encoding scheme based on the data type. Specifically, +# these categories are keys in a configuration dictionary. +# it is not a part of the ZDType class because these categories are more of an implementation detail +# of our config system rather than a useful attribute of any particular data type. +DTypeCategory = Literal["variable-length-string", "default"] + + +class Config(DConfig): # type: ignore[misc] + """The Config will collect configuration from config files and environment variables + + Example environment variables: + Grabs environment variables of the form "ZARR_FOO__BAR_BAZ=123" and + turns these into config variables of the form ``{"foo": {"bar-baz": 123}}`` + It transforms the key and value in the following way: + + - Lower-cases the key text + - Treats ``__`` (double-underscore) as nested access + - Calls ``ast.literal_eval`` on the value + + """ + + def reset(self) -> None: + self.clear() + self.refresh() + + def enable_gpu(self) -> ConfigSet: + """ + Configure Zarr to use GPUs where possible. + """ + return self.set( + {"buffer": "zarr.buffer.gpu.Buffer", "ndbuffer": "zarr.buffer.gpu.NDBuffer"} + ) + + +# The default configuration for zarr +config = Config( + "zarr", + defaults=[ + { + "default_zarr_format": 3, + "array": { + "order": "C", + "write_empty_chunks": False, + "v2_default_compressor": { + "default": {"id": "zstd", "level": 0, "checksum": False}, + "variable-length-string": {"id": "zstd", "level": 0, "checksum": False}, + }, + "v2_default_filters": { + "default": None, + "variable-length-string": [{"id": "vlen-utf8"}], + }, + "v3_default_filters": {"default": [], "variable-length-string": []}, + "v3_default_serializer": { + "default": {"name": "bytes", "configuration": {"endian": "little"}}, + "variable-length-string": {"name": "vlen-utf8"}, + }, + "v3_default_compressors": { + "default": [ + {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, + ], + "variable-length-string": [ + {"name": "zstd", "configuration": {"level": 0, "checksum": False}} + ], + }, + }, + "async": {"concurrency": 10, "timeout": None}, + "threading": {"max_workers": None}, + "json_indent": 2, + "codec_pipeline": { + "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", + "batch_size": 1, + }, + "codecs": { + "blosc": "zarr.codecs.blosc.BloscCodec", + "gzip": "zarr.codecs.gzip.GzipCodec", + "zstd": "zarr.codecs.zstd.ZstdCodec", + "bytes": "zarr.codecs.bytes.BytesCodec", + "endian": "zarr.codecs.bytes.BytesCodec", # compatibility with earlier versions of ZEP1 + "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", + "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", + "transpose": "zarr.codecs.transpose.TransposeCodec", + "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", + "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", + }, + "buffer": "zarr.buffer.cpu.Buffer", + "ndbuffer": "zarr.buffer.cpu.NDBuffer", + } + ], +) + + +def parse_indexing_order(data: Any) -> Literal["C", "F"]: + if data in ("C", "F"): + return cast("Literal['C', 'F']", data) + msg = f"Expected one of ('C', 'F'), got {data} instead." + raise ValueError(msg) + + +def categorize_data_type(dtype: ZDType[Any, Any]) -> DTypeCategory: + """ + Classify a ZDType. The return value is a string which belongs to the type ``DTypeCategory``. + + This is used by the config system to determine how to encode arrays with the associated data type + when the user has not specified a particular serialization scheme. + """ + from zarr.core.dtype import VariableLengthUTF8 + + if isinstance(dtype, VariableLengthUTF8): + return "variable-length-string" + return "default" diff --git a/src/zarr/core/dtype/__init__.py b/src/zarr/core/dtype/__init__.py new file mode 100644 index 0000000000..735690d4bc --- /dev/null +++ b/src/zarr/core/dtype/__init__.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Final, TypeAlias + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeJSON, +) +from zarr.core.dtype.npy.bool import Bool +from zarr.core.dtype.npy.bytes import NullTerminatedBytes, RawBytes, VariableLengthBytes +from zarr.core.dtype.npy.complex import Complex64, Complex128 +from zarr.core.dtype.npy.float import Float16, Float32, Float64 +from zarr.core.dtype.npy.int import Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64 +from zarr.core.dtype.npy.structured import ( + Structured, +) +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64 + +if TYPE_CHECKING: + from zarr.core.common import ZarrFormat + +from collections.abc import Mapping + +import numpy as np +import numpy.typing as npt + +from zarr.core.common import JSON +from zarr.core.dtype.npy.string import ( + FixedLengthUTF32, + VariableLengthUTF8, +) +from zarr.core.dtype.registry import DataTypeRegistry +from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + +__all__ = [ + "Bool", + "Complex64", + "Complex128", + "DataTypeRegistry", + "DataTypeValidationError", + "DateTime64", + "FixedLengthUTF32", + "Float16", + "Float32", + "Float64", + "Int8", + "Int16", + "Int32", + "Int64", + "NullTerminatedBytes", + "RawBytes", + "Structured", + "TBaseDType", + "TBaseScalar", + "TimeDelta64", + "TimeDelta64", + "UInt8", + "UInt16", + "UInt32", + "UInt64", + "VariableLengthUTF8", + "ZDType", + "data_type_registry", + "parse_data_type", +] + +data_type_registry = DataTypeRegistry() + +IntegerDType = Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 +INTEGER_DTYPE: Final = Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64 + +FloatDType = Float16 | Float32 | Float64 +FLOAT_DTYPE: Final = Float16, Float32, Float64 + +ComplexFloatDType = Complex64 | Complex128 +COMPLEX_FLOAT_DTYPE: Final = Complex64, Complex128 + +StringDType = FixedLengthUTF32 | VariableLengthUTF8 +STRING_DTYPE: Final = FixedLengthUTF32, VariableLengthUTF8 + +TimeDType = DateTime64 | TimeDelta64 +TIME_DTYPE: Final = DateTime64, TimeDelta64 + +BytesDType = RawBytes | NullTerminatedBytes | VariableLengthBytes +BYTES_DTYPE: Final = RawBytes, NullTerminatedBytes, VariableLengthBytes + +AnyDType = ( + Bool + | IntegerDType + | FloatDType + | ComplexFloatDType + | StringDType + | BytesDType + | Structured + | TimeDType + | VariableLengthBytes +) +# mypy has trouble inferring the type of variablelengthstring dtype, because its class definition +# depends on the installed numpy version. That's why the type: ignore statement is needed here. +ANY_DTYPE: Final = ( + Bool, + *INTEGER_DTYPE, + *FLOAT_DTYPE, + *COMPLEX_FLOAT_DTYPE, + *STRING_DTYPE, + *BYTES_DTYPE, + Structured, + *TIME_DTYPE, + VariableLengthBytes, +) + +# This type models inputs that can be coerced to a ZDType +ZDTypeLike: TypeAlias = npt.DTypeLike | ZDType[TBaseDType, TBaseScalar] | Mapping[str, JSON] | str + +for dtype in ANY_DTYPE: + # mypy does not know that all the elements of ANY_DTYPE are subclasses of ZDType + data_type_registry.register(dtype._zarr_v3_name, dtype) # type: ignore[arg-type] + + +# TODO: find a better name for this function +def get_data_type_from_native_dtype(dtype: npt.DTypeLike) -> ZDType[TBaseDType, TBaseScalar]: + """ + Get a data type wrapper (an instance of ``ZDType``) from a native data type, e.g. a numpy dtype. + """ + if not isinstance(dtype, np.dtype): + na_dtype: np.dtype[np.generic] + if isinstance(dtype, list): + # this is a valid _VoidDTypeLike check + na_dtype = np.dtype([tuple(d) for d in dtype]) + else: + na_dtype = np.dtype(dtype) + else: + na_dtype = dtype + return data_type_registry.match_dtype(dtype=na_dtype) + + +def get_data_type_from_json( + dtype_spec: DTypeJSON, *, zarr_format: ZarrFormat +) -> ZDType[TBaseDType, TBaseScalar]: + """ + Given a JSON representation of a data type and a Zarr format version, + attempt to create a ZDType instance from the registered ZDType classes. + """ + return data_type_registry.match_json(dtype_spec, zarr_format=zarr_format) + + +def parse_data_type( + dtype_spec: ZDTypeLike, + *, + zarr_format: ZarrFormat, +) -> ZDType[TBaseDType, TBaseScalar]: + """ + Interpret the input as a ZDType instance. + """ + if isinstance(dtype_spec, ZDType): + return dtype_spec + # dict and zarr_format 3 means that we have a JSON object representation of the dtype + if zarr_format == 3 and isinstance(dtype_spec, Mapping): + return get_data_type_from_json(dtype_spec, zarr_format=3) + # otherwise, we have either a numpy dtype string, or a zarr v3 dtype string, and in either case + # we can create a numpy dtype from it, and do the dtype inference from that + return get_data_type_from_native_dtype(dtype_spec) # type: ignore[arg-type] diff --git a/src/zarr/core/dtype/common.py b/src/zarr/core/dtype/common.py new file mode 100644 index 0000000000..6f61b6775e --- /dev/null +++ b/src/zarr/core/dtype/common.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import warnings +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import ( + ClassVar, + Final, + Generic, + Literal, + TypedDict, + TypeGuard, + TypeVar, +) + +from zarr.core.common import NamedConfig + +EndiannessStr = Literal["little", "big"] +ENDIANNESS_STR: Final = "little", "big" + +SpecialFloatStrings = Literal["NaN", "Infinity", "-Infinity"] +SPECIAL_FLOAT_STRINGS: Final = ("NaN", "Infinity", "-Infinity") + +JSONFloatV2 = float | SpecialFloatStrings +JSONFloatV3 = float | SpecialFloatStrings | str + +ObjectCodecID = Literal["vlen-utf8", "vlen-bytes", "vlen-array", "pickle", "json2", "msgpack2"] +# These are the ids of the known object codecs for zarr v2. +OBJECT_CODEC_IDS: Final = ("vlen-utf8", "vlen-bytes", "vlen-array", "pickle", "json2", "msgpack2") + +# This is a wider type than our standard JSON type because we need +# to work with typeddict objects which are assignable to Mapping[str, object] +DTypeJSON = str | int | float | Sequence["DTypeJSON"] | None | Mapping[str, object] + +# The DTypeJSON_V2 type exists because ZDType.from_json takes a single argument, which must contain +# all the information necessary to decode the data type. Zarr v2 supports multiple distinct +# data types that all used the "|O" data type identifier. These data types can only be +# discriminated on the basis of their "object codec", i.e. a special data type specific +# compressor or filter. So to figure out what data type a zarr v2 array has, we need the +# data type identifier from metadata, as well as an object codec id if the data type identifier +# is "|O". +# So we will pack the name of the dtype alongside the name of the object codec id, if applicable, +# in a single dict, and pass that to the data type inference logic. +# These type variables have a very wide bound because the individual zdtype +# classes can perform a very specific type check. + +# This is the JSON representation of a structured dtype in zarr v2 +StructuredName_V2 = Sequence["str | StructuredName_V2"] + +# This models the type of the name a dtype might have in zarr v2 array metadata +DTypeName_V2 = StructuredName_V2 | str + +TDTypeNameV2_co = TypeVar("TDTypeNameV2_co", bound=DTypeName_V2, covariant=True) +TObjectCodecID_co = TypeVar("TObjectCodecID_co", bound=None | str, covariant=True) + + +class DTypeConfig_V2(TypedDict, Generic[TDTypeNameV2_co, TObjectCodecID_co]): + name: TDTypeNameV2_co + object_codec_id: TObjectCodecID_co + + +DTypeSpec_V2 = DTypeConfig_V2[DTypeName_V2, None | str] + + +def check_structured_dtype_v2_inner(data: object) -> TypeGuard[StructuredName_V2]: + """ + A type guard for the inner elements of a structured dtype. This is a recursive check because + the type is itself recursive. + + This check ensures that all the elements are 2-element sequences beginning with a string + and ending with either another string or another 2-element sequence beginning with a string and + ending with another instance of that type. + """ + if isinstance(data, (str, Mapping)): + return False + if not isinstance(data, Sequence): + return False + if len(data) != 2: + return False + if not (isinstance(data[0], str)): + return False + if isinstance(data[-1], str): + return True + elif isinstance(data[-1], Sequence): + return check_structured_dtype_v2_inner(data[-1]) + return False + + +def check_structured_dtype_name_v2(data: Sequence[object]) -> TypeGuard[StructuredName_V2]: + return all(check_structured_dtype_v2_inner(d) for d in data) + + +def check_dtype_name_v2(data: object) -> TypeGuard[DTypeName_V2]: + """ + Type guard for narrowing the type of a python object to an valid zarr v2 dtype name. + """ + if isinstance(data, str): + return True + elif isinstance(data, Sequence): + return check_structured_dtype_name_v2(data) + return False + + +def check_dtype_spec_v2(data: object) -> TypeGuard[DTypeSpec_V2]: + """ + Type guard for narrowing a python object to an instance of DTypeSpec_V2 + """ + if not isinstance(data, Mapping): + return False + if set(data.keys()) != {"name", "object_codec_id"}: + return False + if not check_dtype_name_v2(data["name"]): + return False + return isinstance(data["object_codec_id"], str | None) + + +# By comparison, The JSON representation of a dtype in zarr v3 is much simpler. +# It's either a string, or a structured dict +DTypeSpec_V3 = str | NamedConfig[str, Mapping[str, object]] + + +def check_dtype_spec_v3(data: object) -> TypeGuard[DTypeSpec_V3]: + """ + Type guard for narrowing the type of a python object to an instance of + DTypeSpec_V3, i.e either a string or a dict with a "name" field that's a string and a + "configuration" field that's a mapping with string keys. + """ + if isinstance(data, str) or ( # noqa: SIM103 + isinstance(data, Mapping) + and set(data.keys()) == {"name", "configuration"} + and isinstance(data["configuration"], Mapping) + and all(isinstance(k, str) for k in data["configuration"]) + ): + return True + return False + + +def unpack_dtype_json(data: DTypeSpec_V2 | DTypeSpec_V3) -> DTypeJSON: + """ + Return the array metadata form of the dtype JSON representation. For the Zarr V3 form of dtype + metadata, this is a no-op. For the Zarr V2 form of dtype metadata, this unpacks the dtype name. + """ + if isinstance(data, Mapping) and set(data.keys()) == {"name", "object_codec_id"}: + return data["name"] + return data + + +class DataTypeValidationError(ValueError): ... + + +class ScalarTypeValidationError(ValueError): ... + + +@dataclass(frozen=True) +class HasLength: + """ + A mix-in class for data types with a length attribute, such as fixed-size collections + of unicode strings, or bytes. + """ + + length: int + + +@dataclass(frozen=True) +class HasEndianness: + """ + A mix-in class for data types with an endianness attribute + """ + + endianness: EndiannessStr = "little" + + +@dataclass(frozen=True) +class HasItemSize: + """ + A mix-in class for data types with an item size attribute. + This mix-in bears a property ``item_size``, which denotes the size of each element of the data + type, in bytes. + """ + + @property + def item_size(self) -> int: + raise NotImplementedError + + +@dataclass(frozen=True) +class HasObjectCodec: + """ + A mix-in class for data types that require an object codec id. + This class bears the property ``object_codec_id``, which is the string name of an object + codec that is required to encode and decode the data type. + + In zarr-python 2.x certain data types like variable-length strings or variable-length arrays + used the catch-all numpy "object" data type for their in-memory representation. But these data + types cannot be stored as numpy object data types, because the object data type does not define + a fixed memory layout. So these data types required a special codec, called an "object codec", + that effectively defined a compact representation for the data type, which was used to encode + and decode the data type. + + Zarr-python 2.x would not allow the creation of arrays with the "object" data type if an object + codec was not specified, and thus the name of the object codec is effectively part of the data + type model. + """ + + object_codec_id: ClassVar[str] + + +class UnstableSpecificationWarning(FutureWarning): ... + + +def v3_unstable_dtype_warning(dtype: object) -> None: + """ + Emit this warning when a data type does not have a stable zarr v3 spec + """ + msg = ( + f"The data type ({dtype}) does not have a Zarr V3 specification. " + "That means that the representation of array saved with this data type may change without " + "warning in a future version of Zarr Python. " + "Arrays stored with this data type may be unreadable by other Zarr libraries. " + "Use this data type at your own risk! " + "Check https://github.com/zarr-developers/zarr-extensions/tree/main/data-types for the " + "status of data type specifications for Zarr V3." + ) + warnings.warn(msg, category=UnstableSpecificationWarning, stacklevel=2) diff --git a/.gitmodules b/src/zarr/core/dtype/npy/__init__.py similarity index 100% rename from .gitmodules rename to src/zarr/core/dtype/npy/__init__.py diff --git a/src/zarr/core/dtype/npy/bool.py b/src/zarr/core/dtype/npy/bool.py new file mode 100644 index 0000000000..d8d52468bf --- /dev/null +++ b/src/zarr/core/dtype/npy/bool.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, overload + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +@dataclass(frozen=True, kw_only=True, slots=True) +class Bool(ZDType[np.dtypes.BoolDType, np.bool_], HasItemSize): + """ + Wrapper for numpy boolean dtype. + + Attributes + ---------- + name : str + The name of the dtype. + dtype_cls : ClassVar[type[np.dtypes.BoolDType]] + The numpy dtype class. + """ + + _zarr_v3_name: ClassVar[Literal["bool"]] = "bool" + _zarr_v2_name: ClassVar[Literal["|b1"]] = "|b1" + dtype_cls = np.dtypes.BoolDType + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + """ + Create a Bool from a np.dtype('bool') instance. + """ + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self: Self) -> np.dtypes.BoolDType: + """ + Create a NumPy boolean dtype instance from this ZDType + """ + return self.dtype_cls() + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[Literal["|b1"], None]]: + """ + Check that the input is a valid JSON representation of a Bool. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] == cls._zarr_v2_name + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["bool"]]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_name!r}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls: type[Self], data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|b1"], None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["bool"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|b1"], None] | Literal["bool"]: + if zarr_format == 2: + return {"name": self._zarr_v2_name, "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> bool: + # Anything can become a bool + return True + + def cast_scalar(self, data: object) -> np.bool_: + if self._check_scalar(data): + return np.bool_(data) + msg = f"Cannot convert object with type {type(data)} to a numpy boolean." + raise TypeError(msg) + + def default_scalar(self) -> np.bool_: + """ + Get the default value for the boolean dtype. + + Returns + ------- + np.bool_ + The default value. + """ + return np.False_ + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> bool: + """ + Convert a scalar to a python bool. + + Parameters + ---------- + data : object + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + bool + The JSON-serializable format. + """ + return bool(data) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.bool_: + """ + Read a JSON-serializable value as a numpy boolean scalar. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + np.bool_ + The numpy boolean scalar. + """ + if self._check_scalar(data): + return np.bool_(data) + raise TypeError(f"Invalid type: {data}. Expected a boolean.") # pragma: no cover + + @property + def item_size(self) -> int: + return 1 diff --git a/src/zarr/core/dtype/npy/bytes.py b/src/zarr/core/dtype/npy/bytes.py new file mode 100644 index 0000000000..e363c75053 --- /dev/null +++ b/src/zarr/core/dtype/npy/bytes.py @@ -0,0 +1,369 @@ +from __future__ import annotations + +import base64 +import re +from dataclasses import dataclass +from typing import Any, ClassVar, Literal, Self, TypedDict, TypeGuard, cast, overload + +import numpy as np + +from zarr.core.common import JSON, NamedConfig, ZarrFormat +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasItemSize, + HasLength, + HasObjectCodec, + check_dtype_spec_v2, + v3_unstable_dtype_warning, +) +from zarr.core.dtype.npy.common import check_json_str +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +BytesLike = np.bytes_ | str | bytes | int + + +class FixedLengthBytesConfig(TypedDict): + length_bytes: int + + +NullTerminatedBytesJSONV3 = NamedConfig[Literal["null_terminated_bytes"], FixedLengthBytesConfig] +RawBytesJSONV3 = NamedConfig[Literal["raw_bytes"], FixedLengthBytesConfig] + + +@dataclass(frozen=True, kw_only=True) +class NullTerminatedBytes(ZDType[np.dtypes.BytesDType[int], np.bytes_], HasLength, HasItemSize): + dtype_cls = np.dtypes.BytesDType + _zarr_v3_name: ClassVar[Literal["null_terminated_bytes"]] = "null_terminated_bytes" + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(length=dtype.itemsize) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.BytesDType[int]: + return self.dtype_cls(self.length) + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid representation of a numpy S dtype. We expect + something like ``{"name": "|S10", "object_codec_id": None}`` + """ + return ( + check_dtype_spec_v2(data) + and isinstance(data["name"], str) + and re.match(r"^\|S\d+$", data["name"]) is not None + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[NullTerminatedBytesJSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and "length_bytes" in data["configuration"] + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls(length=int(name[2:])) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string like '|S1', '|S2', etc" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls(length=data["configuration"]["length_bytes"]) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> NullTerminatedBytesJSONV3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[str, None] | NullTerminatedBytesJSONV3: + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return { + "name": self._zarr_v3_name, + "configuration": {"length_bytes": self.length}, + } + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[BytesLike]: + # this is generous for backwards compatibility + return isinstance(data, BytesLike) + + def _cast_scalar_unchecked(self, data: BytesLike) -> np.bytes_: + # We explicitly truncate the result because of the following numpy behavior: + # >>> x = np.dtype('S3').type('hello world') + # >>> x + # np.bytes_(b'hello world') + # >>> x.dtype + # dtype('S11') + + if isinstance(data, int): + return self.to_native_dtype().type(str(data)[: self.length]) + else: + return self.to_native_dtype().type(data[: self.length]) + + def cast_scalar(self, data: object) -> np.bytes_: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy bytes scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.bytes_: + return np.bytes_(b"") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + as_bytes = self.cast_scalar(data) + return base64.standard_b64encode(as_bytes).decode("ascii") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.bytes_: + if check_json_str(data): + return self.to_native_dtype().type(base64.standard_b64decode(data.encode("ascii"))) + raise TypeError( + f"Invalid type: {data}. Expected a base64-encoded string." + ) # pragma: no cover + + @property + def item_size(self) -> int: + return self.length + + +@dataclass(frozen=True, kw_only=True) +class RawBytes(ZDType[np.dtypes.VoidDType[int], np.void], HasLength, HasItemSize): + # np.dtypes.VoidDType is specified in an odd way in numpy + # it cannot be used to create instances of the dtype + # so we have to tell mypy to ignore this here + dtype_cls = np.dtypes.VoidDType # type: ignore[assignment] + _zarr_v3_name: ClassVar[Literal["raw_bytes"]] = "raw_bytes" + + @classmethod + def _check_native_dtype( + cls: type[Self], dtype: TBaseDType + ) -> TypeGuard[np.dtypes.VoidDType[Any]]: + """ + Numpy void dtype comes in two forms: + * If the ``fields`` attribute is ``None``, then the dtype represents N raw bytes. + * If the ``fields`` attribute is not ``None``, then the dtype represents a structured dtype, + + In this check we ensure that ``fields`` is ``None``. + + Parameters + ---------- + dtype : TDType + The dtype to check. + + Returns + ------- + Bool + True if the dtype matches, False otherwise. + """ + return cls.dtype_cls is type(dtype) and dtype.fields is None # type: ignore[has-type] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(length=dtype.itemsize) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + + def to_native_dtype(self) -> np.dtypes.VoidDType[int]: + # Numpy does not allow creating a void type + # by invoking np.dtypes.VoidDType directly + return cast("np.dtypes.VoidDType[int]", np.dtype(f"V{self.length}")) + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid representation of a numpy S dtype. We expect + something like ``{"name": "|V10", "object_codec_id": None}`` + """ + return ( + check_dtype_spec_v2(data) + and isinstance(data["name"], str) + and re.match(r"^\|V\d+$", data["name"]) is not None + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[RawBytesJSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"length_bytes"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls(length=int(name[2:])) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string like '|V1', '|V2', etc" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls(length=data["configuration"]["length_bytes"]) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> RawBytesJSONV3: ... + + def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | RawBytesJSONV3: + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return {"name": self._zarr_v3_name, "configuration": {"length_bytes": self.length}} + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> bool: + return isinstance(data, np.bytes_ | str | bytes | np.void) + + def _cast_scalar_unchecked(self, data: object) -> np.void: + native_dtype = self.to_native_dtype() + # Without the second argument, numpy will return a void scalar for dtype V1. + # The second argument ensures that, if native_dtype is something like V10, + # the result will actually be a V10 scalar. + return native_dtype.type(data, native_dtype) + + def cast_scalar(self, data: object) -> np.void: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy void scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.void: + return self.to_native_dtype().type(("\x00" * self.length).encode("ascii")) + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return base64.standard_b64encode(self.cast_scalar(data).tobytes()).decode("ascii") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void: + if check_json_str(data): + return self.to_native_dtype().type(base64.standard_b64decode(data)) + raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover + + @property + def item_size(self) -> int: + return self.length + + +@dataclass(frozen=True, kw_only=True) +class VariableLengthBytes(ZDType[np.dtypes.ObjectDType, bytes], HasObjectCodec): + dtype_cls = np.dtypes.ObjectDType + _zarr_v3_name: ClassVar[Literal["variable_length_bytes"]] = "variable_length_bytes" + object_codec_id: ClassVar[Literal["vlen-bytes"]] = "vlen-bytes" + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.ObjectDType: + return self.dtype_cls() + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[Literal["|O"], Literal["vlen-bytes"]]]: + """ + Check that the input is a valid JSON representation of a numpy O dtype, and that the + object codec id is appropriate for variable-length UTF-8 strings. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] == "|O" + and data["object_codec_id"] == cls.object_codec_id + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["variable_length_bytes"]]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string '|O' and an object_codec_id of {cls.object_codec_id}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json( + self, zarr_format: Literal[2] + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-bytes"]]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["variable_length_bytes"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-bytes"]] | Literal["variable_length_bytes"]: + if zarr_format == 2: + return {"name": "|O", "object_codec_id": self.object_codec_id} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def default_scalar(self) -> bytes: + return b"" + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return base64.standard_b64encode(data).decode("ascii") # type: ignore[arg-type] + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> bytes: + if check_json_str(data): + return base64.standard_b64decode(data.encode("ascii")) + raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[BytesLike]: + return isinstance(data, BytesLike) + + def _cast_scalar_unchecked(self, data: BytesLike) -> bytes: + if isinstance(data, str): + return bytes(data, encoding="utf-8") + return bytes(data) + + def cast_scalar(self, data: object) -> bytes: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to bytes." + raise TypeError(msg) diff --git a/src/zarr/core/dtype/npy/common.py b/src/zarr/core/dtype/npy/common.py new file mode 100644 index 0000000000..264561f25c --- /dev/null +++ b/src/zarr/core/dtype/npy/common.py @@ -0,0 +1,503 @@ +from __future__ import annotations + +import base64 +import struct +import sys +from collections.abc import Sequence +from typing import ( + TYPE_CHECKING, + Any, + Final, + Literal, + SupportsComplex, + SupportsFloat, + SupportsIndex, + SupportsInt, + TypeGuard, + TypeVar, +) + +import numpy as np + +from zarr.core.dtype.common import ( + ENDIANNESS_STR, + SPECIAL_FLOAT_STRINGS, + EndiannessStr, + JSONFloatV2, + JSONFloatV3, +) + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + +IntLike = SupportsInt | SupportsIndex | bytes | str +FloatLike = SupportsIndex | SupportsFloat | bytes | str +ComplexLike = SupportsFloat | SupportsIndex | SupportsComplex | bytes | str | None +DateTimeUnit = Literal[ + "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" +] +DATETIME_UNIT: Final = ( + "Y", + "M", + "W", + "D", + "h", + "m", + "s", + "ms", + "us", + "μs", + "ns", + "ps", + "fs", + "as", + "generic", +) + +NumpyEndiannessStr = Literal[">", "<", "="] +NUMPY_ENDIANNESS_STR: Final = ">", "<", "=" + +TFloatDType_co = TypeVar( + "TFloatDType_co", + bound=np.dtypes.Float16DType | np.dtypes.Float32DType | np.dtypes.Float64DType, + covariant=True, +) +TFloatScalar_co = TypeVar( + "TFloatScalar_co", bound=np.float16 | np.float32 | np.float64, covariant=True +) + +TComplexDType_co = TypeVar( + "TComplexDType_co", bound=np.dtypes.Complex64DType | np.dtypes.Complex128DType, covariant=True +) +TComplexScalar_co = TypeVar("TComplexScalar_co", bound=np.complex64 | np.complex128, covariant=True) + + +def endianness_from_numpy_str(endianness: NumpyEndiannessStr) -> EndiannessStr: + """ + Convert a numpy endianness string literal to a human-readable literal value. + + Parameters + ---------- + endianness : Literal[">", "<", "="] + The numpy string representation of the endianness. + + Returns + ------- + Endianness + The human-readable representation of the endianness. + + Raises + ------ + ValueError + If the endianness is invalid. + """ + match endianness: + case "=": + # Use the local system endianness + return sys.byteorder + case "<": + return "little" + case ">": + return "big" + raise ValueError(f"Invalid endianness: {endianness!r}. Expected one of {NUMPY_ENDIANNESS_STR}") + + +def endianness_to_numpy_str(endianness: EndiannessStr) -> NumpyEndiannessStr: + """ + Convert an endianness literal to its numpy string representation. + + Parameters + ---------- + endianness : Endianness + The endianness to convert. + + Returns + ------- + Literal[">", "<"] + The numpy string representation of the endianness. + + Raises + ------ + ValueError + If the endianness is invalid. + """ + match endianness: + case "little": + return "<" + case "big": + return ">" + raise ValueError( + f"Invalid endianness: {endianness!r}. Expected one of {ENDIANNESS_STR} or None" + ) + + +def get_endianness_from_numpy_dtype(dtype: np.dtype[np.generic]) -> EndiannessStr: + """ + Gets the endianness from a numpy dtype that has an endianness. This function will + raise a ValueError if the numpy data type does not have a concrete endianness. + """ + endianness = dtype.byteorder + if dtype.byteorder in NUMPY_ENDIANNESS_STR: + return endianness_from_numpy_str(endianness) # type: ignore [arg-type] + raise ValueError(f"The dtype {dtype} has an unsupported endianness: {endianness}") + + +def float_from_json_v2(data: JSONFloatV2) -> float: + """ + Convert a JSON float to a float (Zarr v2). + + Parameters + ---------- + data : JSONFloat + The JSON float to convert. + + Returns + ------- + float + The float value. + """ + match data: + case "NaN": + return float("nan") + case "Infinity": + return float("inf") + case "-Infinity": + return float("-inf") + case _: + return float(data) + + +def float_from_json_v3(data: JSONFloatV3) -> float: + """ + Convert a JSON float to a float (v3). + + Parameters + ---------- + data : JSONFloat + The JSON float to convert. + + Returns + ------- + float + The float value. + + Notes + ----- + Zarr V3 allows floats to be stored as hex strings. To quote the spec: + "...for float32, "NaN" is equivalent to "0x7fc00000". + This representation is the only way to specify a NaN value other than the specific NaN value + denoted by "NaN"." + """ + + if isinstance(data, str): + if data in SPECIAL_FLOAT_STRINGS: + return float_from_json_v2(data) # type: ignore[arg-type] + if not data.startswith("0x"): + msg = ( + f"Invalid float value: {data!r}. Expected a string starting with the hex prefix" + " '0x', or one of 'NaN', 'Infinity', or '-Infinity'." + ) + raise ValueError(msg) + if len(data[2:]) == 4: + dtype_code = ">e" + elif len(data[2:]) == 8: + dtype_code = ">f" + elif len(data[2:]) == 16: + dtype_code = ">d" + else: + msg = ( + f"Invalid hexadecimal float value: {data!r}. " + "Expected the '0x' prefix to be followed by 4, 8, or 16 numeral characters" + ) + raise ValueError(msg) + return float(struct.unpack(dtype_code, bytes.fromhex(data[2:]))[0]) + return float_from_json_v2(data) + + +def bytes_from_json(data: str, *, zarr_format: ZarrFormat) -> bytes: + """ + Convert a JSON string to bytes + + Parameters + ---------- + data : str + The JSON string to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + bytes + The bytes. + """ + if zarr_format == 2: + return base64.b64decode(data.encode("ascii")) + # TODO: differentiate these as needed. This is a spec question. + if zarr_format == 3: + return base64.b64decode(data.encode("ascii")) + raise ValueError(f"Invalid zarr format: {zarr_format}. Expected 2 or 3.") # pragma: no cover + + +def bytes_to_json(data: bytes, zarr_format: ZarrFormat) -> str: + """ + Convert bytes to JSON. + + Parameters + ---------- + data : bytes + The bytes to store. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The bytes encoded as ascii using the base64 alphabet. + """ + # TODO: decide if we are going to make this implementation zarr format-specific + return base64.b64encode(data).decode("ascii") + + +def float_to_json_v2(data: float | np.floating[Any]) -> JSONFloatV2: + """ + Convert a float to JSON (v2). + + Parameters + ---------- + data : float or np.floating + The float value to convert. + + Returns + ------- + JSONFloat + The JSON representation of the float. + """ + if np.isnan(data): + return "NaN" + elif np.isinf(data): + return "Infinity" if data > 0 else "-Infinity" + return float(data) + + +def float_to_json_v3(data: float | np.floating[Any]) -> JSONFloatV3: + """ + Convert a float to JSON (v3). + + Parameters + ---------- + data : float or np.floating + The float value to convert. + + Returns + ------- + JSONFloat + The JSON representation of the float. + """ + # v3 can in principle handle distinct NaN values, but numpy does not represent these explicitly + # so we just reuse the v2 routine here + return float_to_json_v2(data) + + +def complex_float_to_json_v3( + data: complex | np.complexfloating[Any, Any], +) -> tuple[JSONFloatV3, JSONFloatV3]: + """ + Convert a complex number to JSON as defined by the Zarr V3 spec. + + Parameters + ---------- + data : complex or np.complexfloating + The complex value to convert. + + Returns + ------- + tuple[JSONFloat, JSONFloat] + The JSON representation of the complex number. + """ + return float_to_json_v3(data.real), float_to_json_v3(data.imag) + + +def complex_float_to_json_v2( + data: complex | np.complexfloating[Any, Any], +) -> tuple[JSONFloatV2, JSONFloatV2]: + """ + Convert a complex number to JSON as defined by the Zarr V2 spec. + + Parameters + ---------- + data : complex | np.complexfloating + The complex value to convert. + + Returns + ------- + tuple[JSONFloat, JSONFloat] + The JSON representation of the complex number. + """ + return float_to_json_v2(data.real), float_to_json_v2(data.imag) + + +def complex_float_from_json_v2(data: tuple[JSONFloatV2, JSONFloatV2]) -> complex: + """ + Convert a JSON complex float to a complex number (v2). + + Parameters + ---------- + data : tuple[JSONFloat, JSONFloat] + The JSON complex float to convert. + + Returns + ------- + np.complexfloating + The complex number. + """ + return complex(float_from_json_v2(data[0]), float_from_json_v2(data[1])) + + +def complex_float_from_json_v3(data: tuple[JSONFloatV3, JSONFloatV3]) -> complex: + """ + Convert a JSON complex float to a complex number (v3). + + Parameters + ---------- + data : tuple[JSONFloat, JSONFloat] + The JSON complex float to convert. + + Returns + ------- + np.complexfloating + The complex number. + """ + return complex(float_from_json_v3(data[0]), float_from_json_v3(data[1])) + + +def check_json_float_v2(data: JSON) -> TypeGuard[JSONFloatV2]: + """ + Check if a JSON value represents a float (v2). + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a float, False otherwise. + """ + if data == "NaN" or data == "Infinity" or data == "-Infinity": + return True + return isinstance(data, float | int) + + +def check_json_float_v3(data: JSON) -> TypeGuard[JSONFloatV3]: + """ + Check if a JSON value represents a float (v3). + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a float, False otherwise. + """ + return check_json_float_v2(data) or (isinstance(data, str) and data.startswith("0x")) + + +def check_json_complex_float_v2(data: JSON) -> TypeGuard[tuple[JSONFloatV2, JSONFloatV2]]: + """ + Check if a JSON value represents a complex float, as per the behavior of zarr-python 2.x + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a complex float, False otherwise. + """ + return ( + not isinstance(data, str) + and isinstance(data, Sequence) + and len(data) == 2 + and check_json_float_v2(data[0]) + and check_json_float_v2(data[1]) + ) + + +def check_json_complex_float_v3(data: JSON) -> TypeGuard[tuple[JSONFloatV3, JSONFloatV3]]: + """ + Check if a JSON value represents a complex float, as per the zarr v3 spec + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a complex float, False otherwise. + """ + return ( + not isinstance(data, str) + and isinstance(data, Sequence) + and len(data) == 2 + and check_json_float_v3(data[0]) + and check_json_float_v3(data[1]) + ) + + +def check_json_int(data: JSON) -> TypeGuard[int]: + """ + Check if a JSON value is an integer. + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is an integer, False otherwise. + """ + return bool(isinstance(data, int)) + + +def check_json_str(data: JSON) -> TypeGuard[str]: + """ + Check if a JSON value is a string. + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a string, False otherwise. + """ + return bool(isinstance(data, str)) + + +def check_json_bool(data: JSON) -> TypeGuard[bool]: + """ + Check if a JSON value is a boolean. + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a boolean, False otherwise. + """ + return isinstance(data, bool) diff --git a/src/zarr/core/dtype/npy/complex.py b/src/zarr/core/dtype/npy/complex.py new file mode 100644 index 0000000000..38e506f1bc --- /dev/null +++ b/src/zarr/core/dtype/npy/complex.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Self, + TypeGuard, + overload, +) + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + ComplexLike, + TComplexDType_co, + TComplexScalar_co, + check_json_complex_float_v2, + check_json_complex_float_v3, + complex_float_from_json_v2, + complex_float_from_json_v3, + complex_float_to_json_v2, + complex_float_to_json_v3, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +@dataclass(frozen=True) +class BaseComplex(ZDType[TComplexDType_co, TComplexScalar_co], HasEndianness, HasItemSize): + # This attribute holds the possible zarr v2 JSON names for the data type + _zarr_v2_names: ClassVar[tuple[str, ...]] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> TComplexDType_co: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of this data type. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] in cls._zarr_v2_names + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[str]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> str: ... + + def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | str: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[ComplexLike]: + return isinstance(data, ComplexLike) + + def _cast_scalar_unchecked(self, data: ComplexLike) -> TComplexScalar_co: + return self.to_native_dtype().type(data) # type: ignore[return-value] + + def cast_scalar(self, data: object) -> TComplexScalar_co: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy float scalar." + raise TypeError(msg) + + def default_scalar(self) -> TComplexScalar_co: + """ + Get the default value, which is 0 cast to this dtype + + Returns + ------- + Int scalar + The default value. + """ + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> TComplexScalar_co: + """ + Read a JSON-serializable value as a numpy float. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + TScalar_co + The numpy float. + """ + if zarr_format == 2: + if check_json_complex_float_v2(data): + return self._cast_scalar_unchecked(complex_float_from_json_v2(data)) + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + elif zarr_format == 3: + if check_json_complex_float_v3(data): + return self._cast_scalar_unchecked(complex_float_from_json_v3(data)) + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> JSON: + """ + Convert an object to a JSON-serializable float. + + Parameters + ---------- + data : _BaseScalar + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + JSON + The JSON-serializable form of the complex number, which is a list of two floats, + each of which is encoding according to a zarr-format-specific encoding. + """ + if zarr_format == 2: + return complex_float_to_json_v2(self.cast_scalar(data)) + elif zarr_format == 3: + return complex_float_to_json_v3(self.cast_scalar(data)) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +@dataclass(frozen=True, kw_only=True) +class Complex64(BaseComplex[np.dtypes.Complex64DType, np.complex64]): + dtype_cls = np.dtypes.Complex64DType + _zarr_v3_name: ClassVar[Literal["complex64"]] = "complex64" + _zarr_v2_names: ClassVar[tuple[str, ...]] = (">c8", " int: + return 8 + + +@dataclass(frozen=True, kw_only=True) +class Complex128(BaseComplex[np.dtypes.Complex128DType, np.complex128], HasEndianness): + dtype_cls = np.dtypes.Complex128DType + _zarr_v3_name: ClassVar[Literal["complex128"]] = "complex128" + _zarr_v2_names: ClassVar[tuple[str, ...]] = (">c16", " int: + return 16 diff --git a/src/zarr/core/dtype/npy/float.py b/src/zarr/core/dtype/npy/float.py new file mode 100644 index 0000000000..7b7243993f --- /dev/null +++ b/src/zarr/core/dtype/npy/float.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, overload + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + ScalarTypeValidationError, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + FloatLike, + TFloatDType_co, + TFloatScalar_co, + check_json_float_v2, + check_json_float_v3, + endianness_to_numpy_str, + float_from_json_v2, + float_from_json_v3, + float_to_json_v2, + float_to_json_v3, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +@dataclass(frozen=True) +class BaseFloat(ZDType[TFloatDType_co, TFloatScalar_co], HasEndianness, HasItemSize): + # This attribute holds the possible zarr v2 JSON names for the data type + _zarr_v2_names: ClassVar[tuple[str, ...]] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> TFloatDType_co: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) # type: ignore[return-value] + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of this data type. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] in cls._zarr_v2_names + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[str]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> str: ... + + def to_json(self, zarr_format: ZarrFormat) -> DTypeConfig_V2[str, None] | str: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[FloatLike]: + return isinstance(data, FloatLike) + + def _cast_scalar_unchecked(self, data: FloatLike) -> TFloatScalar_co: + return self.to_native_dtype().type(data) # type: ignore[return-value] + + def cast_scalar(self, data: object) -> TFloatScalar_co: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy float scalar." + raise ScalarTypeValidationError(msg) + + def default_scalar(self) -> TFloatScalar_co: + """ + Get the default value, which is 0 cast to this dtype + + Returns + ------- + Int scalar + The default value. + """ + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> TFloatScalar_co: + """ + Read a JSON-serializable value as a numpy float. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + TScalar_co + The numpy float. + """ + if zarr_format == 2: + if check_json_float_v2(data): + return self._cast_scalar_unchecked(float_from_json_v2(data)) + else: + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + elif zarr_format == 3: + if check_json_float_v3(data): + return self._cast_scalar_unchecked(float_from_json_v3(data)) + else: + raise TypeError( + f"Invalid type: {data}. Expected a float or a special string encoding of a float." + ) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> float | str: + """ + Convert an object to a JSON-serializable float. + + Parameters + ---------- + data : _BaseScalar + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + JSON + The JSON-serializable form of the float, which is potentially a number or a string. + See the zarr specifications for details on the JSON encoding for floats. + """ + if zarr_format == 2: + return float_to_json_v2(self.cast_scalar(data)) + elif zarr_format == 3: + return float_to_json_v3(self.cast_scalar(data)) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +@dataclass(frozen=True, kw_only=True) +class Float16(BaseFloat[np.dtypes.Float16DType, np.float16]): + dtype_cls = np.dtypes.Float16DType + _zarr_v3_name = "float16" + _zarr_v2_names: ClassVar[tuple[Literal[">f2"], Literal["f2", " int: + return 2 + + +@dataclass(frozen=True, kw_only=True) +class Float32(BaseFloat[np.dtypes.Float32DType, np.float32]): + dtype_cls = np.dtypes.Float32DType + _zarr_v3_name = "float32" + _zarr_v2_names: ClassVar[tuple[Literal[">f4"], Literal["f4", " int: + return 4 + + +@dataclass(frozen=True, kw_only=True) +class Float64(BaseFloat[np.dtypes.Float64DType, np.float64]): + dtype_cls = np.dtypes.Float64DType + _zarr_v3_name = "float64" + _zarr_v2_names: ClassVar[tuple[Literal[">f8"], Literal["f8", " int: + return 8 diff --git a/src/zarr/core/dtype/npy/int.py b/src/zarr/core/dtype/npy/int.py new file mode 100644 index 0000000000..79d3ce2d47 --- /dev/null +++ b/src/zarr/core/dtype/npy/int.py @@ -0,0 +1,686 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Self, + SupportsIndex, + SupportsInt, + TypeGuard, + TypeVar, + overload, +) + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + check_json_int, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + +_NumpyIntDType = ( + np.dtypes.Int8DType + | np.dtypes.Int16DType + | np.dtypes.Int32DType + | np.dtypes.Int64DType + | np.dtypes.UInt8DType + | np.dtypes.UInt16DType + | np.dtypes.UInt32DType + | np.dtypes.UInt64DType +) +_NumpyIntScalar = ( + np.int8 | np.int16 | np.int32 | np.int64 | np.uint8 | np.uint16 | np.uint32 | np.uint64 +) +TIntDType_co = TypeVar("TIntDType_co", bound=_NumpyIntDType, covariant=True) +TIntScalar_co = TypeVar("TIntScalar_co", bound=_NumpyIntScalar, covariant=True) +IntLike = SupportsInt | SupportsIndex | bytes | str + + +@dataclass(frozen=True) +class BaseInt(ZDType[TIntDType_co, TIntScalar_co], HasItemSize): + # This attribute holds the possible zarr V2 JSON names for the data type + _zarr_v2_names: ClassVar[tuple[str, ...]] + + @classmethod + def _check_json_v2(cls, data: object) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of this data type. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] in cls._zarr_v2_names + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: object) -> TypeGuard[str]: + """ + Check that a JSON value is consistent with the zarr v3 spec for this data type. + """ + return data == cls._zarr_v3_name + + def _check_scalar(self, data: object) -> TypeGuard[IntLike]: + """ + Check that a python object is IntLike + """ + return isinstance(data, IntLike) + + def _cast_scalar_unchecked(self, data: IntLike) -> TIntScalar_co: + """ + Create an integer without any type checking of the input. + """ + return self.to_native_dtype().type(data) # type: ignore[return-value] + + def cast_scalar(self, data: object) -> TIntScalar_co: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy integer." + raise TypeError(msg) + + def default_scalar(self) -> TIntScalar_co: + """ + Get the default value, which is 0 cast to this dtype + + Returns + ------- + Int scalar + The default value. + """ + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> TIntScalar_co: + """ + Read a JSON-serializable value as a numpy int scalar. + + Parameters + ---------- + data : JSON + The JSON-serializable value. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + TScalar_co + The numpy scalar. + """ + if check_json_int(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Invalid type: {data}. Expected an integer.") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> int: + """ + Convert an object to JSON-serializable scalar. + + Parameters + ---------- + data : _BaseScalar + The value to convert. + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + int + The JSON-serializable form of the scalar. + """ + return int(self.cast_scalar(data)) + + +@dataclass(frozen=True, kw_only=True) +class Int8(BaseInt[np.dtypes.Int8DType, np.int8]): + dtype_cls = np.dtypes.Int8DType + _zarr_v3_name: ClassVar[Literal["int8"]] = "int8" + _zarr_v2_names: ClassVar[tuple[Literal["|i1"]]] = ("|i1",) + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + """ + Create a Int8 from a np.dtype('int8') instance. + """ + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self: Self) -> np.dtypes.Int8DType: + return self.dtype_cls() + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_names[0]!r}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|i1"], None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["int8"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|i1"], None] | Literal["int8"]: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self._zarr_v2_names[0], "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @property + def item_size(self) -> int: + return 1 + + +@dataclass(frozen=True, kw_only=True) +class UInt8(BaseInt[np.dtypes.UInt8DType, np.uint8]): + dtype_cls = np.dtypes.UInt8DType + _zarr_v3_name: ClassVar[Literal["uint8"]] = "uint8" + _zarr_v2_names: ClassVar[tuple[Literal["|u1"]]] = ("|u1",) + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + """ + Create a Bool from a np.dtype('uint8') instance. + """ + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self: Self) -> np.dtypes.UInt8DType: + return self.dtype_cls() + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v2_names[0]!r}" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal["|u1"], None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["uint8"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|u1"], None] | Literal["uint8"]: + """ + Convert the wrapped data type to a JSON-serializable form. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + str + The JSON-serializable representation of the wrapped data type + """ + if zarr_format == 2: + return {"name": self._zarr_v2_names[0], "object_codec_id": None} + elif zarr_format == 3: + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @property + def item_size(self) -> int: + return 1 + + +@dataclass(frozen=True, kw_only=True) +class Int16(BaseInt[np.dtypes.Int16DType, np.int16], HasEndianness): + dtype_cls = np.dtypes.Int16DType + _zarr_v3_name: ClassVar[Literal["int16"]] = "int16" + _zarr_v2_names: ClassVar[tuple[Literal[">i2"], Literal["i2", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.Int16DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names!r}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i2", " Literal["int16"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">i2", " int: + return 2 + + +@dataclass(frozen=True, kw_only=True) +class UInt16(BaseInt[np.dtypes.UInt16DType, np.uint16], HasEndianness): + dtype_cls = np.dtypes.UInt16DType + _zarr_v3_name: ClassVar[Literal["uint16"]] = "uint16" + _zarr_v2_names: ClassVar[tuple[Literal[">u2"], Literal["u2", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.UInt16DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of UInt16. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of UInt16. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u2", " Literal["uint16"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">u2", " int: + return 2 + + +@dataclass(frozen=True, kw_only=True) +class Int32(BaseInt[np.dtypes.Int32DType, np.int32], HasEndianness): + dtype_cls = np.dtypes.Int32DType + _zarr_v3_name: ClassVar[Literal["int32"]] = "int32" + _zarr_v2_names: ClassVar[tuple[Literal[">i4"], Literal["i4", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.Int32DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i4", " Literal["int32"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">i4", " int: + return 4 + + +@dataclass(frozen=True, kw_only=True) +class UInt32(BaseInt[np.dtypes.UInt32DType, np.uint32], HasEndianness): + dtype_cls = np.dtypes.UInt32DType + _zarr_v3_name: ClassVar[Literal["uint32"]] = "uint32" + _zarr_v2_names: ClassVar[tuple[Literal[">u4"], Literal["u4", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.UInt32DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u4", " Literal["uint32"]: ... + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">u4", " int: + return 4 + + +@dataclass(frozen=True, kw_only=True) +class Int64(BaseInt[np.dtypes.Int64DType, np.int64], HasEndianness): + dtype_cls = np.dtypes.Int64DType + _zarr_v3_name: ClassVar[Literal["int64"]] = "int64" + _zarr_v2_names: ClassVar[tuple[Literal[">i8"], Literal["i8", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.Int64DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">i8", " Literal["int64"]: ... + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">i8", " int: + return 8 + + +@dataclass(frozen=True, kw_only=True) +class UInt64(BaseInt[np.dtypes.UInt64DType, np.uint64], HasEndianness): + dtype_cls = np.dtypes.UInt64DType + _zarr_v3_name: ClassVar[Literal["uint64"]] = "uint64" + _zarr_v2_names: ClassVar[tuple[Literal[">u8"], Literal["u8", " np.dtypes.UInt64DType: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls().newbyteorder(byte_order) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Going via numpy ensures that we get the endianness correct without + # annoying string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected one of the strings {cls._zarr_v2_names}." + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[Literal[">u8", " Literal["uint64"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal[">u8", " Self: + if cls._check_native_dtype(dtype): + return cls(endianness=get_endianness_from_numpy_dtype(dtype)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + @property + def item_size(self) -> int: + return 8 diff --git a/src/zarr/core/dtype/npy/string.py b/src/zarr/core/dtype/npy/string.py new file mode 100644 index 0000000000..4a1114617a --- /dev/null +++ b/src/zarr/core/dtype/npy/string.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Protocol, + Self, + TypedDict, + TypeGuard, + overload, + runtime_checkable, +) + +import numpy as np + +from zarr.core.common import NamedConfig +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + HasLength, + HasObjectCodec, + check_dtype_spec_v2, + v3_unstable_dtype_warning, +) +from zarr.core.dtype.npy.common import ( + check_json_str, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TDType_co, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + from zarr.core.dtype.wrapper import TBaseDType + +_NUMPY_SUPPORTS_VLEN_STRING = hasattr(np.dtypes, "StringDType") + + +@runtime_checkable +class SupportsStr(Protocol): + def __str__(self) -> str: ... + + +class LengthBytesConfig(TypedDict): + length_bytes: int + + +# TODO: Fix this terrible name +FixedLengthUTF32JSONV3 = NamedConfig[Literal["fixed_length_utf32"], LengthBytesConfig] + + +@dataclass(frozen=True, kw_only=True) +class FixedLengthUTF32( + ZDType[np.dtypes.StrDType[int], np.str_], HasEndianness, HasLength, HasItemSize +): + dtype_cls = np.dtypes.StrDType + _zarr_v3_name: ClassVar[Literal["fixed_length_utf32"]] = "fixed_length_utf32" + code_point_bytes: ClassVar[int] = 4 # utf32 is 4 bytes per code point + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + endianness = get_endianness_from_numpy_dtype(dtype) + return cls( + length=dtype.itemsize // (cls.code_point_bytes), + endianness=endianness, + ) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> np.dtypes.StrDType[int]: + byte_order = endianness_to_numpy_str(self.endianness) + return self.dtype_cls(self.length).newbyteorder(byte_order) + + @classmethod + def _check_json_v2(cls, data: DTypeJSON) -> TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that the input is a valid JSON representation of a numpy U dtype. + """ + return ( + check_dtype_spec_v2(data) + and isinstance(data["name"], str) + and re.match(r"^[><]U\d+$", data["name"]) is not None + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[FixedLengthUTF32JSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and "configuration" in data + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"length_bytes"} + and isinstance(data["configuration"]["length_bytes"], int) + ) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> FixedLengthUTF32JSONV3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[str, None] | FixedLengthUTF32JSONV3: + if zarr_format == 2: + return {"name": self.to_native_dtype().str, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return { + "name": self._zarr_v3_name, + "configuration": {"length_bytes": self.length * self.code_point_bytes}, + } + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + # Construct the numpy dtype instead of string parsing. + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + raise DataTypeValidationError( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string representation of a numpy U dtype." + ) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls(length=data["configuration"]["length_bytes"] // cls.code_point_bytes) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + def default_scalar(self) -> np.str_: + return np.str_("") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return str(data) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.str_: + if check_json_str(data): + return self.to_native_dtype().type(data) + raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[str | np.str_ | bytes | int]: + # this is generous for backwards compatibility + return isinstance(data, str | np.str_ | bytes | int) + + def cast_scalar(self, data: object) -> np.str_: + if self._check_scalar(data): + # We explicitly truncate before casting because of the following numpy behavior: + # >>> x = np.dtype('U3').type('hello world') + # >>> x + # np.str_('hello world') + # >>> x.dtype + # dtype('U11') + + if isinstance(data, int): + return self.to_native_dtype().type(str(data)[: self.length]) + else: + return self.to_native_dtype().type(data[: self.length]) + raise TypeError( + f"Cannot convert object with type {type(data)} to a numpy unicode string scalar." + ) + + @property + def item_size(self) -> int: + return self.length * self.code_point_bytes + + +def check_vlen_string_json_scalar(data: object) -> TypeGuard[int | str | float]: + """ + This function checks the type of JSON-encoded variable length strings. It is generous for + backwards compatibility, as zarr-python v2 would use ints for variable length strings + fill values + """ + return isinstance(data, int | str | float) + + +# VariableLengthUTF8 is defined in two places, conditioned on the version of numpy. +# If numpy 2 is installed, then VariableLengthUTF8 is defined with the numpy variable length +# string dtype as the native dtype. Otherwise, VariableLengthUTF8 is defined with the numpy object +# dtype as the native dtype. +class UTF8Base(ZDType[TDType_co, str], HasObjectCodec): + """ + A base class for the variable length UTF-8 string data type. This class should not be used + as data type, but as a base class for other variable length string data types. + """ + + _zarr_v3_name: ClassVar[Literal["variable_length_utf8"]] = "variable_length_utf8" + object_codec_id: ClassVar[Literal["vlen-utf8"]] = "vlen-utf8" + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + return cls() + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]]]: + """ + Check that the input is a valid JSON representation of a numpy O dtype, and that the + object codec id is appropriate for variable-length UTF-8 strings. + """ + return ( + check_dtype_spec_v2(data) + and data["name"] == "|O" + and data["object_codec_id"] == cls.object_codec_id + ) + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[Literal["variable_length_utf8"]]: + return data == cls._zarr_v3_name + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + return cls() + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected the string '|O'" + ) + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + return cls() + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected {cls._zarr_v3_name}." + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json( + self, zarr_format: Literal[2] + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]]: ... + @overload + def to_json(self, zarr_format: Literal[3]) -> Literal["variable_length_utf8"]: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[Literal["|O"], Literal["vlen-utf8"]] | Literal["variable_length_utf8"]: + if zarr_format == 2: + return {"name": "|O", "object_codec_id": self.object_codec_id} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + return self._zarr_v3_name + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def default_scalar(self) -> str: + return "" + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Invalid type: {data}. Expected a string.") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> str: + if not check_vlen_string_json_scalar(data): + raise TypeError(f"Invalid type: {data}. Expected a string or number.") + return str(data) + + def _check_scalar(self, data: object) -> TypeGuard[SupportsStr]: + return isinstance(data, SupportsStr) + + def _cast_scalar_unchecked(self, data: SupportsStr) -> str: + return str(data) + + def cast_scalar(self, data: object) -> str: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Cannot convert object with type {type(data)} to a python string.") + + +if _NUMPY_SUPPORTS_VLEN_STRING: + + @dataclass(frozen=True, kw_only=True) + class VariableLengthUTF8(UTF8Base[np.dtypes.StringDType]): # type: ignore[type-var] + dtype_cls = np.dtypes.StringDType + + def to_native_dtype(self) -> np.dtypes.StringDType: + return self.dtype_cls() + +else: + # Numpy pre-2 does not have a variable length string dtype, so we use the Object dtype instead. + @dataclass(frozen=True, kw_only=True) + class VariableLengthUTF8(UTF8Base[np.dtypes.ObjectDType]): # type: ignore[no-redef] + dtype_cls = np.dtypes.ObjectDType + + def to_native_dtype(self) -> np.dtypes.ObjectDType: + return self.dtype_cls() diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py new file mode 100644 index 0000000000..07e3000826 --- /dev/null +++ b/src/zarr/core/dtype/npy/structured.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal, Self, TypeGuard, cast, overload + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + DTypeSpec_V3, + HasItemSize, + StructuredName_V2, + check_dtype_spec_v2, + check_structured_dtype_name_v2, + v3_unstable_dtype_warning, +) +from zarr.core.dtype.npy.common import ( + bytes_from_json, + bytes_to_json, + check_json_str, +) +from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + +if TYPE_CHECKING: + from collections.abc import Sequence + + from zarr.core.common import JSON, NamedConfig, ZarrFormat + +StructuredScalarLike = list[object] | tuple[object, ...] | bytes | int + + +@dataclass(frozen=True, kw_only=True) +class Structured(ZDType[np.dtypes.VoidDType[int], np.void], HasItemSize): + dtype_cls = np.dtypes.VoidDType # type: ignore[assignment] + _zarr_v3_name = "structured" + fields: tuple[tuple[str, ZDType[TBaseDType, TBaseScalar]], ...] + + @classmethod + def _check_native_dtype(cls, dtype: TBaseDType) -> TypeGuard[np.dtypes.VoidDType[int]]: + """ + Check that this dtype is a numpy structured dtype + + Parameters + ---------- + dtype : np.dtypes.DTypeLike + The dtype to check. + + Returns + ------- + TypeGuard[np.dtypes.VoidDType] + True if the dtype matches, False otherwise. + """ + return isinstance(dtype, cls.dtype_cls) and dtype.fields is not None # type: ignore[has-type] + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + from zarr.core.dtype import get_data_type_from_native_dtype + + fields: list[tuple[str, ZDType[TBaseDType, TBaseScalar]]] = [] + if cls._check_native_dtype(dtype): + # fields of a structured numpy dtype are either 2-tuples or 3-tuples. we only + # care about the first element in either case. + for key, (dtype_instance, *_) in dtype.fields.items(): # type: ignore[union-attr] + dtype_wrapped = get_data_type_from_native_dtype(dtype_instance) + fields.append((key, dtype_wrapped)) + + return cls(fields=tuple(fields)) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + + def to_native_dtype(self) -> np.dtypes.VoidDType[int]: + return cast( + "np.dtypes.VoidDType[int]", + np.dtype([(key, dtype.to_native_dtype()) for (key, dtype) in self.fields]), + ) + + @classmethod + def _check_json_v2( + cls, + data: DTypeJSON, + ) -> TypeGuard[DTypeConfig_V2[StructuredName_V2, None]]: + return ( + check_dtype_spec_v2(data) + and not isinstance(data["name"], str) + and check_structured_dtype_name_v2(data["name"]) + and data["object_codec_id"] is None + ) + + @classmethod + def _check_json_v3( + cls, data: DTypeJSON + ) -> TypeGuard[NamedConfig[Literal["structured"], dict[str, Sequence[tuple[str, DTypeJSON]]]]]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"fields"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + # avoid circular import + from zarr.core.dtype import get_data_type_from_json + + if cls._check_json_v2(data): + # structured dtypes are constructed directly from a list of lists + # note that we do not handle the object codec here! this will prevent structured + # dtypes from containing object dtypes. + return cls( + fields=tuple( # type: ignore[misc] + ( # type: ignore[misc] + f_name, + get_data_type_from_json( + {"name": f_dtype, "object_codec_id": None}, zarr_format=2 + ), + ) + for f_name, f_dtype in data["name"] + ) + ) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a JSON array of arrays" + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + # avoid circular import + from zarr.core.dtype import get_data_type_from_json + + if cls._check_json_v3(data): + config = data["configuration"] + meta_fields = config["fields"] + return cls( + fields=tuple( + (f_name, get_data_type_from_json(f_dtype, zarr_format=3)) + for f_name, f_dtype in meta_fields + ) + ) + msg = f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a JSON object with the key {cls._zarr_v3_name!r}" + raise DataTypeValidationError(msg) + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[StructuredName_V2, None]: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> DTypeSpec_V3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[StructuredName_V2, None] | DTypeSpec_V3: + if zarr_format == 2: + fields = [ + [f_name, f_dtype.to_json(zarr_format=zarr_format)["name"]] + for f_name, f_dtype in self.fields + ] + return {"name": fields, "object_codec_id": None} + elif zarr_format == 3: + v3_unstable_dtype_warning(self) + fields = [ + [f_name, f_dtype.to_json(zarr_format=zarr_format)] # type: ignore[list-item] + for f_name, f_dtype in self.fields + ] + base_dict = { + "name": self._zarr_v3_name, + "configuration": {"fields": fields}, + } + return cast("DTypeSpec_V3", base_dict) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def _check_scalar(self, data: object) -> TypeGuard[StructuredScalarLike]: + # TODO: implement something more precise here! + return isinstance(data, (bytes, list, tuple, int, np.void)) + + def _cast_scalar_unchecked(self, data: StructuredScalarLike) -> np.void: + na_dtype = self.to_native_dtype() + if isinstance(data, bytes): + res = np.frombuffer(data, dtype=na_dtype)[0] + elif isinstance(data, list | tuple): + res = np.array([tuple(data)], dtype=na_dtype)[0] + else: + res = np.array([data], dtype=na_dtype)[0] + return cast("np.void", res) + + def cast_scalar(self, data: object) -> np.void: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy structured scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.void: + return self._cast_scalar_unchecked(0) + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void: + if check_json_str(data): + as_bytes = bytes_from_json(data, zarr_format=zarr_format) + dtype = self.to_native_dtype() + return cast("np.void", np.array([as_bytes]).view(dtype)[0]) + raise TypeError(f"Invalid type: {data}. Expected a string.") + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> str: + return bytes_to_json(self.cast_scalar(data).tobytes(), zarr_format) + + @property + def item_size(self) -> int: + # Lets have numpy do the arithmetic here + return self.to_native_dtype().itemsize diff --git a/src/zarr/core/dtype/npy/time.py b/src/zarr/core/dtype/npy/time.py new file mode 100644 index 0000000000..1f9080475c --- /dev/null +++ b/src/zarr/core/dtype/npy/time.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import ( + TYPE_CHECKING, + ClassVar, + Literal, + Self, + TypedDict, + TypeGuard, + TypeVar, + cast, + get_args, + overload, +) + +import numpy as np + +from zarr.core.common import NamedConfig +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeConfig_V2, + DTypeJSON, + HasEndianness, + HasItemSize, + check_dtype_spec_v2, +) +from zarr.core.dtype.npy.common import ( + DATETIME_UNIT, + DateTimeUnit, + check_json_int, + endianness_to_numpy_str, + get_endianness_from_numpy_dtype, +) +from zarr.core.dtype.wrapper import TBaseDType, ZDType + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + +_DTypeName = Literal["datetime64", "timedelta64"] +TimeDeltaLike = str | int | bytes | np.timedelta64 | timedelta | None +DateTimeLike = str | int | bytes | np.datetime64 | datetime | None + + +def datetime_from_int(data: int, *, unit: DateTimeUnit, scale_factor: int) -> np.datetime64: + """ + Convert an integer to a datetime64. + + Parameters + ---------- + data : int + The integer to convert. + unit : DateTimeUnit + The unit of the datetime64. + scale_factor : int + The scale factor of the datetime64. + + Returns + ------- + np.datetime64 + The datetime64 value. + """ + dtype_name = f"datetime64[{scale_factor}{unit}]" + return cast("np.datetime64", np.int64(data).view(dtype_name)) + + +def datetimelike_to_int(data: np.datetime64 | np.timedelta64) -> int: + """ + Convert a datetime64 or a timedelta64 to an integer. + + Parameters + ---------- + data : np.datetime64 | np.timedelta64 + The value to convert. + + Returns + ------- + int + An integer representation of the scalar. + """ + return data.view(np.int64).item() + + +def check_json_time(data: JSON) -> TypeGuard[Literal["NaT"] | int]: + """ + Type guard to check if the input JSON data is the literal string "NaT" + or an integer. + """ + return check_json_int(data) or data == "NaT" + + +BaseTimeDType_co = TypeVar( + "BaseTimeDType_co", + bound=np.dtypes.TimeDelta64DType | np.dtypes.DateTime64DType, + covariant=True, +) +BaseTimeScalar_co = TypeVar( + "BaseTimeScalar_co", bound=np.timedelta64 | np.datetime64, covariant=True +) + + +class TimeConfig(TypedDict): + unit: DateTimeUnit + scale_factor: int + + +DateTime64JSONV3 = NamedConfig[Literal["numpy.datetime64"], TimeConfig] +TimeDelta64JSONV3 = NamedConfig[Literal["numpy.timedelta64"], TimeConfig] + + +@dataclass(frozen=True, kw_only=True, slots=True) +class TimeDTypeBase(ZDType[BaseTimeDType_co, BaseTimeScalar_co], HasEndianness, HasItemSize): + _zarr_v2_names: ClassVar[tuple[str, ...]] + # this attribute exists so that we can programmatically create a numpy dtype instance + # because the particular numpy dtype we are wrapping does not allow direct construction via + # cls.dtype_cls() + _numpy_name: ClassVar[_DTypeName] + scale_factor: int + unit: DateTimeUnit + + def __post_init__(self) -> None: + if self.scale_factor < 1: + raise ValueError(f"scale_factor must be > 0, got {self.scale_factor}.") + if self.scale_factor >= 2**31: + raise ValueError(f"scale_factor must be < 2147483648, got {self.scale_factor}.") + if self.unit not in get_args(DateTimeUnit): + raise ValueError(f"unit must be one of {get_args(DateTimeUnit)}, got {self.unit!r}.") + + @classmethod + def from_native_dtype(cls, dtype: TBaseDType) -> Self: + if cls._check_native_dtype(dtype): + unit, scale_factor = np.datetime_data(dtype.name) + unit = cast("DateTimeUnit", unit) + return cls( + unit=unit, + scale_factor=scale_factor, + endianness=get_endianness_from_numpy_dtype(dtype), + ) + raise DataTypeValidationError( + f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" + ) + + def to_native_dtype(self) -> BaseTimeDType_co: + # Numpy does not allow creating datetime64 or timedelta64 via + # np.dtypes.{dtype_name}() + # so we use np.dtype with a formatted string. + dtype_string = f"{self._numpy_name}[{self.scale_factor}{self.unit}]" + return np.dtype(dtype_string).newbyteorder(endianness_to_numpy_str(self.endianness)) # type: ignore[return-value] + + @overload # type: ignore[override] + def to_json(self, zarr_format: Literal[2]) -> DTypeConfig_V2[str, None]: ... + @overload + def to_json(self, zarr_format: Literal[3]) -> DateTime64JSONV3 | TimeDelta64JSONV3: ... + + def to_json( + self, zarr_format: ZarrFormat + ) -> DTypeConfig_V2[str, None] | DateTime64JSONV3 | TimeDelta64JSONV3: + if zarr_format == 2: + name = self.to_native_dtype().str + return {"name": name, "object_codec_id": None} + elif zarr_format == 3: + return cast( + "DateTime64JSONV3 | TimeDelta64JSONV3", + { + "name": self._zarr_v3_name, + "configuration": {"unit": self.unit, "scale_factor": self.scale_factor}, + }, + ) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> int: + return datetimelike_to_int(data) # type: ignore[arg-type] + + @property + def item_size(self) -> int: + return 8 + + +@dataclass(frozen=True, kw_only=True, slots=True) +class TimeDelta64(TimeDTypeBase[np.dtypes.TimeDelta64DType, np.timedelta64], HasEndianness): + """ + A wrapper for the ``TimeDelta64`` data type defined in numpy. + Scalars of this type can be created by performing arithmetic with ``DateTime64`` scalars. + Like ``DateTime64``, ``TimeDelta64`` is parametrized by a unit, but unlike ``DateTime64``, the + unit for ``TimeDelta64`` is optional. + """ + + # mypy infers the type of np.dtypes.TimeDelta64DType to be + # "Callable[[Literal['Y', 'M', 'W', 'D'] | Literal['h', 'm', 's', 'ms', 'us', 'ns', 'ps', 'fs', 'as']], Never]" + dtype_cls = np.dtypes.TimeDelta64DType # type: ignore[assignment] + _zarr_v3_name: ClassVar[Literal["numpy.timedelta64"]] = "numpy.timedelta64" + _zarr_v2_names = (">m8", " TypeGuard[DTypeConfig_V2[str, None]]: + if not check_dtype_spec_v2(data): + return False + name = data["name"] + # match m[M], etc + # consider making this a standalone function + if not isinstance(name, str): + return False + if not name.startswith(cls._zarr_v2_names): + return False + if len(name) == 3: + # no unit, and + # we already checked that this string is either m8 + return True + else: + return name[4:-1].endswith(DATETIME_UNIT) and name[-1] == "]" + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[DateTime64JSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"unit", "scale_factor"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string " + f"representation of an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + unit = data["configuration"]["unit"] + scale_factor = data["configuration"]["scale_factor"] + return cls(unit=unit, scale_factor=scale_factor) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a dict " + f"with a 'name' key with the value 'numpy.timedelta64', " + "and a 'configuration' key with a value of a dict with a 'unit' key and a " + "'scale_factor' key" + ) + raise DataTypeValidationError(msg) + + def _check_scalar(self, data: object) -> TypeGuard[TimeDeltaLike]: + if data is None: + return True + return isinstance(data, str | int | bytes | np.timedelta64 | timedelta) + + def _cast_scalar_unchecked(self, data: TimeDeltaLike) -> np.timedelta64: + return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") + + def cast_scalar(self, data: object) -> np.timedelta64: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy timedelta64 scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.timedelta64: + return np.timedelta64("NaT") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.timedelta64: + if check_json_time(data): + return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") + raise TypeError(f"Invalid type: {data}. Expected an integer.") # pragma: no cover + + +@dataclass(frozen=True, kw_only=True, slots=True) +class DateTime64(TimeDTypeBase[np.dtypes.DateTime64DType, np.datetime64], HasEndianness): + dtype_cls = np.dtypes.DateTime64DType # type: ignore[assignment] + _zarr_v3_name: ClassVar[Literal["numpy.datetime64"]] = "numpy.datetime64" + _zarr_v2_names = (">M8", " TypeGuard[DTypeConfig_V2[str, None]]: + """ + Check that JSON input is a string representation of a NumPy datetime64 data type, like "M8[10s]". This function can be used as a type guard to narrow the type of unknown JSON + input. + """ + if not check_dtype_spec_v2(data): + return False + name = data["name"] + if not isinstance(name, str): + return False + if not name.startswith(cls._zarr_v2_names): + return False + if len(name) == 3: + # no unit, and + # we already checked that this string is either M8 + return True + else: + return name[4:-1].endswith(DATETIME_UNIT) and name[-1] == "]" + + @classmethod + def _check_json_v3(cls, data: DTypeJSON) -> TypeGuard[DateTime64JSONV3]: + return ( + isinstance(data, dict) + and set(data.keys()) == {"name", "configuration"} + and data["name"] == cls._zarr_v3_name + and isinstance(data["configuration"], dict) + and set(data["configuration"].keys()) == {"unit", "scale_factor"} + ) + + @classmethod + def _from_json_v2(cls, data: DTypeJSON) -> Self: + if cls._check_json_v2(data): + name = data["name"] + return cls.from_native_dtype(np.dtype(name)) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a string " + f"representation of an instance of {cls.dtype_cls}" # type: ignore[has-type] + ) + raise DataTypeValidationError(msg) + + @classmethod + def _from_json_v3(cls, data: DTypeJSON) -> Self: + if cls._check_json_v3(data): + unit = data["configuration"]["unit"] + scale_factor = data["configuration"]["scale_factor"] + return cls(unit=unit, scale_factor=scale_factor) + msg = ( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}, expected a dict " + f"with a 'name' key with the value 'numpy.datetime64', " + "and a 'configuration' key with a value of a dict with a 'unit' key and a " + "'scale_factor' key" + ) + raise DataTypeValidationError(msg) + + def _check_scalar(self, data: object) -> TypeGuard[DateTimeLike]: + if data is None: + return True + return isinstance(data, str | int | bytes | np.datetime64 | datetime) + + def _cast_scalar_unchecked(self, data: DateTimeLike) -> np.datetime64: + return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") + + def cast_scalar(self, data: object) -> np.datetime64: + if self._check_scalar(data): + return self._cast_scalar_unchecked(data) + msg = f"Cannot convert object with type {type(data)} to a numpy datetime scalar." + raise TypeError(msg) + + def default_scalar(self) -> np.datetime64: + return np.datetime64("NaT") + + def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.datetime64: + if check_json_time(data): + return self._cast_scalar_unchecked(data) + raise TypeError(f"Invalid type: {data}. Expected an integer.") # pragma: no cover diff --git a/src/zarr/core/dtype/registry.py b/src/zarr/core/dtype/registry.py new file mode 100644 index 0000000000..1d2a97a90a --- /dev/null +++ b/src/zarr/core/dtype/registry.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import contextlib +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Self + +import numpy as np + +from zarr.core.dtype.common import ( + DataTypeValidationError, + DTypeJSON, +) + +if TYPE_CHECKING: + from importlib.metadata import EntryPoint + + from zarr.core.common import ZarrFormat + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + + +# This class is different from the other registry classes, which inherit from +# dict. IMO it's simpler to just do a dataclass. But long-term we should +# have just 1 registry class in use. +@dataclass(frozen=True, kw_only=True) +class DataTypeRegistry: + contents: dict[str, type[ZDType[TBaseDType, TBaseScalar]]] = field( + default_factory=dict, init=False + ) + + lazy_load_list: list[EntryPoint] = field(default_factory=list, init=False) + + def lazy_load(self) -> None: + for e in self.lazy_load_list: + self.register(e.load()._zarr_v3_name, e.load()) + + self.lazy_load_list.clear() + + def register(self: Self, key: str, cls: type[ZDType[TBaseDType, TBaseScalar]]) -> None: + # don't register the same dtype twice + if key not in self.contents or self.contents[key] != cls: + self.contents[key] = cls + + def unregister(self, key: str) -> None: + """Unregister a data type by its key.""" + if key in self.contents: + del self.contents[key] + else: + raise KeyError(f"Data type '{key}' not found in registry.") + + def get(self, key: str) -> type[ZDType[TBaseDType, TBaseScalar]]: + return self.contents[key] + + def match_dtype(self, dtype: TBaseDType) -> ZDType[TBaseDType, TBaseScalar]: + if dtype == np.dtype("O"): + msg = ( + f"Zarr data type resolution from {dtype} failed. " + 'Attempted to resolve a zarr data type from a numpy "Object" data type, which is ' + 'ambiguous, as multiple zarr data types can be represented by the numpy "Object" ' + "data type. " + "In this case you should construct your array by providing a specific Zarr data " + 'type. For a list of Zarr data types that are compatible with the numpy "Object"' + "data type, see https://github.com/zarr-developers/zarr-python/issues/3117" + ) + raise ValueError(msg) + matched: list[ZDType[TBaseDType, TBaseScalar]] = [] + for val in self.contents.values(): + with contextlib.suppress(DataTypeValidationError): + matched.append(val.from_native_dtype(dtype)) + if len(matched) == 1: + return matched[0] + elif len(matched) > 1: + msg = ( + f"Zarr data type resolution from {dtype} failed. " + f"Multiple data type wrappers found that match dtype '{dtype}': {matched}. " + "You should unregister one of these data types, or avoid Zarr data type inference " + "entirely by providing a specific Zarr data type when creating your array." + "For more information, see https://github.com/zarr-developers/zarr-python/issues/3117" + ) + raise ValueError(msg) + raise ValueError(f"No Zarr data type found that matches dtype '{dtype!r}'") + + def match_json( + self, data: DTypeJSON, *, zarr_format: ZarrFormat + ) -> ZDType[TBaseDType, TBaseScalar]: + for val in self.contents.values(): + try: + return val.from_json(data, zarr_format=zarr_format) + except DataTypeValidationError: + pass + raise ValueError(f"No Zarr data type found that matches {data!r}") diff --git a/src/zarr/core/dtype/wrapper.py b/src/zarr/core/dtype/wrapper.py new file mode 100644 index 0000000000..7be97fa4b4 --- /dev/null +++ b/src/zarr/core/dtype/wrapper.py @@ -0,0 +1,297 @@ +""" +Wrapper for native array data types. + +The ``ZDType`` class is an abstract base class for wrapping native array data types, e.g. NumPy dtypes. +``ZDType`` provides a common interface for working with data types in a way that is independent of the +underlying data type system. + +The wrapper class encapsulates a native data type. Instances of the class can be created from a +native data type instance, and a native data type instance can be created from an instance of the +wrapper class. + +The wrapper class is responsible for: +- Serializing and deserializing a native data type to Zarr V2 or Zarr V3 metadata. + This ensures that the data type can be properly stored and retrieved from array metadata. +- Serializing and deserializing scalar values to Zarr V2 or Zarr V3 metadata. This is important for + storing a fill value for an array in a manner that is valid for the data type. + +You can add support for a new data type in Zarr by subclassing ``ZDType`` wrapper class and adapt its methods +to support your native data type. The wrapper class must be added to a data type registry +(defined elsewhere) before array creation routines or array reading routines can use your new data +type. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + ClassVar, + Generic, + Literal, + Self, + TypeGuard, + TypeVar, + overload, +) + +import numpy as np + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + from zarr.core.dtype.common import DTypeJSON, DTypeSpec_V2, DTypeSpec_V3 + +# This the upper bound for the scalar types we support. It's numpy scalars + str, +# because the new variable-length string dtype in numpy does not have a corresponding scalar type +TBaseScalar = np.generic | str | bytes +# This is the bound for the dtypes that we support. If we support non-numpy dtypes, +# then this bound will need to be widened. +TBaseDType = np.dtype[np.generic] + +# These two type parameters are covariant because we want +# x : ZDType[BaseDType, BaseScalar] = ZDType[SubDType, SubScalar] +# to type check +TScalar_co = TypeVar("TScalar_co", bound=TBaseScalar, covariant=True) +TDType_co = TypeVar("TDType_co", bound=TBaseDType, covariant=True) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ZDType(ABC, Generic[TDType_co, TScalar_co]): + """ + Abstract base class for wrapping native array data types, e.g. numpy dtypes + + Attributes + ---------- + dtype_cls : ClassVar[type[TDType]] + The wrapped dtype class. This is a class variable. + _zarr_v3_name : ClassVar[str] + The name given to the data type by a Zarr v3 data type specification. This is a + class variable, and it should generally be unique across different data types. + """ + + # this class will create a native data type + # mypy currently disallows class variables to contain type parameters + # but it seems OK for us to use it here: + # https://github.com/python/typing/discussions/1424#discussioncomment-7989934 + dtype_cls: ClassVar[type[TDType_co]] # type: ignore[misc] + _zarr_v3_name: ClassVar[str] + + @classmethod + def _check_native_dtype(cls: type[Self], dtype: TBaseDType) -> TypeGuard[TDType_co]: + """ + Check that a native data type matches the dtype_cls class attribute. Used as a type guard. + + Parameters + ---------- + dtype : TDType + The dtype to check. + + Returns + ------- + Bool + True if the dtype matches, False otherwise. + """ + return type(dtype) is cls.dtype_cls + + @classmethod + @abstractmethod + def from_native_dtype(cls: type[Self], dtype: TBaseDType) -> Self: + """ + Create a ZDType instance from a native data type. The default implementation first performs + a type check via ``cls._check_native_dtype``. If that type check succeeds, the ZDType class + instance is created. + + This method is used when taking a user-provided native data type, like a NumPy data type, + and creating the corresponding ZDType instance from them. + + Parameters + ---------- + dtype : TDType + The native data type object to wrap. + + Returns + ------- + Self + The ZDType that wraps the native data type. + + Raises + ------ + TypeError + If the native data type is not consistent with the wrapped data type. + """ + ... + + @abstractmethod + def to_native_dtype(self: Self) -> TDType_co: + """ + Return an instance of the wrapped data type. This operation inverts ``from_native_dtype``. + + Returns + ------- + TDType + The native data type wrapped by this ZDType. + """ + ... + + @classmethod + @abstractmethod + def _from_json_v2(cls: type[Self], data: DTypeJSON) -> Self: ... + + @classmethod + @abstractmethod + def _from_json_v3(cls: type[Self], data: DTypeJSON) -> Self: ... + + @classmethod + def from_json(cls: type[Self], data: DTypeJSON, *, zarr_format: ZarrFormat) -> Self: + """ + Create an instance of this ZDType from JSON data. + + Parameters + ---------- + data : DTypeJSON + The JSON representation of the data type. The type annotation includes + Mapping[str, object] to accommodate typed dictionaries. + + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + Self + The wrapped data type. + """ + if zarr_format == 2: + return cls._from_json_v2(data) + if zarr_format == 3: + return cls._from_json_v3(data) + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + @overload + def to_json(self, zarr_format: Literal[2]) -> DTypeSpec_V2: ... + + @overload + def to_json(self, zarr_format: Literal[3]) -> DTypeSpec_V3: ... + + @abstractmethod + def to_json(self, zarr_format: ZarrFormat) -> DTypeSpec_V2 | DTypeSpec_V3: + """ + Serialize this ZDType to JSON. + + Parameters + ---------- + zarr_format : ZarrFormat + The zarr format version. + + Returns + ------- + DTypeJSON_V2 | DTypeJSON_V3 + The JSON-serializable representation of the wrapped data type + """ + ... + + @abstractmethod + def _check_scalar(self, data: object) -> bool: + """ + Check that an python object is a valid scalar value for the wrapped data type. + + Parameters + ---------- + data : object + A value to check. + + Returns + ------- + Bool + True if the object is valid, False otherwise. + """ + ... + + @abstractmethod + def cast_scalar(self, data: object) -> TScalar_co: + """ + Cast a python object to the wrapped scalar type. + The type of the provided scalar is first checked for compatibility. + If it's incompatible with the associated scalar type, a ``TypeError`` will be raised. + + Parameters + ---------- + data : object + The python object to cast. + + Returns + ------- + TScalar + The cast value. + """ + + @abstractmethod + def default_scalar(self) -> TScalar_co: + """ + Get the default scalar value for the wrapped data type. This is a method, rather than an + attribute, because the default value for some data types depends on parameters that are + not known until a concrete data type is wrapped. For example, data types parametrized by a + length like fixed-length strings or bytes will generate scalars consistent with that length. + + Returns + ------- + TScalar + The default value for this data type. + """ + ... + + @abstractmethod + def from_json_scalar(self: Self, data: JSON, *, zarr_format: ZarrFormat) -> TScalar_co: + """ + Read a JSON-serializable value as a scalar. + + Parameters + ---------- + data : JSON + A JSON representation of a scalar value. + zarr_format : ZarrFormat + The zarr format version. This is specified because the JSON serialization of scalars + differs between Zarr V2 and Zarr V3. + + Returns + ------- + TScalar + The deserialized scalar value. + """ + ... + + @abstractmethod + def to_json_scalar(self, data: object, *, zarr_format: ZarrFormat) -> JSON: + """ + Serialize a python object to the JSON representation of a scalar. The value will first be + cast to the scalar type associated with this ZDType, then serialized to JSON. + + Parameters + ---------- + data : object + The value to convert. + zarr_format : ZarrFormat + The zarr format version. This is specified because the JSON serialization of scalars + differs between Zarr V2 and Zarr V3. + + Returns + ------- + JSON + The JSON-serialized scalar. + """ + ... + + +def scalar_failed_type_check_msg( + cls_instance: ZDType[TBaseDType, TBaseScalar], bad_scalar: object +) -> str: + """ + Generate an error message reporting that a particular value failed a type check when attempting + to cast that value to a scalar. + """ + return ( + f"The value {bad_scalar!r} failed a type check. " + f"It cannot be safely cast to a scalar compatible with {cls_instance}. " + f"Consult the documentation for {cls_instance} to determine the possible values that can " + "be cast to scalars of the wrapped data type." + ) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py new file mode 100644 index 0000000000..4c8ced21f4 --- /dev/null +++ b/src/zarr/core/group.py @@ -0,0 +1,3711 @@ +from __future__ import annotations + +import asyncio +import itertools +import json +import logging +import warnings +from collections import defaultdict +from dataclasses import asdict, dataclass, field, fields, replace +from itertools import accumulate +from typing import TYPE_CHECKING, Literal, TypeVar, assert_never, cast, overload + +import numpy as np +import numpy.typing as npt +from typing_extensions import deprecated + +import zarr.api.asynchronous as async_api +from zarr._compat import _deprecate_positional_args +from zarr.abc.metadata import Metadata +from zarr.abc.store import Store, set_or_delete +from zarr.core._info import GroupInfo +from zarr.core.array import ( + Array, + AsyncArray, + CompressorLike, + CompressorsLike, + FiltersLike, + SerializerLike, + ShardsLike, + _build_parents, + _parse_deprecated_compressor, + create_array, +) +from zarr.core.attributes import Attributes +from zarr.core.buffer import default_buffer_prototype +from zarr.core.common import ( + JSON, + ZARR_JSON, + ZARRAY_JSON, + ZATTRS_JSON, + ZGROUP_JSON, + ZMETADATA_V2_JSON, + ChunkCoords, + DimensionNames, + NodeType, + ShapeLike, + ZarrFormat, + parse_shapelike, +) +from zarr.core.config import config +from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata +from zarr.core.sync import SyncMixin, sync +from zarr.errors import ContainsArrayError, ContainsGroupError, MetadataValidationError +from zarr.storage import StoreLike, StorePath +from zarr.storage._common import ensure_no_existing_node, make_store_path +from zarr.storage._utils import _join_paths, _normalize_path_keys, normalize_path + +if TYPE_CHECKING: + from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Coroutine, + Generator, + Iterable, + Iterator, + Mapping, + ) + from typing import Any + + from zarr.core.array_spec import ArrayConfig, ArrayConfigLike + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.chunk_key_encodings import ChunkKeyEncodingLike + from zarr.core.common import MemoryOrder + +logger = logging.getLogger("zarr.group") + +DefaultT = TypeVar("DefaultT") + + +def parse_zarr_format(data: Any) -> ZarrFormat: + """Parse the zarr_format field from metadata.""" + if data in (2, 3): + return cast("ZarrFormat", data) + msg = f"Invalid zarr_format. Expected one of 2 or 3. Got {data}." + raise ValueError(msg) + + +def parse_node_type(data: Any) -> NodeType: + """Parse the node_type field from metadata.""" + if data in ("array", "group"): + return cast("Literal['array', 'group']", data) + raise MetadataValidationError("node_type", "array or group", data) + + +# todo: convert None to empty dict +def parse_attributes(data: Any) -> dict[str, Any]: + """Parse the attributes field from metadata.""" + if data is None: + return {} + elif isinstance(data, dict) and all(isinstance(k, str) for k in data): + return data + msg = f"Expected dict with string keys. Got {type(data)} instead." + raise TypeError(msg) + + +@overload +def _parse_async_node(node: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]) -> Array: ... + + +@overload +def _parse_async_node(node: AsyncGroup) -> Group: ... + + +def _parse_async_node( + node: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup, +) -> Array | Group: + """Wrap an AsyncArray in an Array, or an AsyncGroup in a Group.""" + if isinstance(node, AsyncArray): + return Array(node) + elif isinstance(node, AsyncGroup): + return Group(node) + else: + raise TypeError(f"Unknown node type, got {type(node)}") + + +@dataclass(frozen=True) +class ConsolidatedMetadata: + """ + Consolidated Metadata for this Group. + + This stores the metadata of child nodes below this group. Any child groups + will have their consolidated metadata set appropriately. + """ + + metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] + kind: Literal["inline"] = "inline" + must_understand: Literal[False] = False + + def to_dict(self) -> dict[str, JSON]: + return { + "kind": self.kind, + "must_understand": self.must_understand, + "metadata": {k: v.to_dict() for k, v in self.flattened_metadata.items()}, + } + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> ConsolidatedMetadata: + data = dict(data) + + kind = data.get("kind") + if kind != "inline": + raise ValueError(f"Consolidated metadata kind='{kind}' is not supported.") + + raw_metadata = data.get("metadata") + if not isinstance(raw_metadata, dict): + raise TypeError(f"Unexpected type for 'metadata': {type(raw_metadata)}") + + metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} + if raw_metadata: + for k, v in raw_metadata.items(): + if not isinstance(v, dict): + raise TypeError( + f"Invalid value for metadata items. key='{k}', type='{type(v).__name__}'" + ) + + # zarr_format is present in v2 and v3. + zarr_format = parse_zarr_format(v["zarr_format"]) + + if zarr_format == 3: + node_type = parse_node_type(v.get("node_type", None)) + if node_type == "group": + metadata[k] = GroupMetadata.from_dict(v) + elif node_type == "array": + metadata[k] = ArrayV3Metadata.from_dict(v) + else: + assert_never(node_type) + elif zarr_format == 2: + if "shape" in v: + metadata[k] = ArrayV2Metadata.from_dict(v) + else: + metadata[k] = GroupMetadata.from_dict(v) + else: + assert_never(zarr_format) + + cls._flat_to_nested(metadata) + + return cls(metadata=metadata) + + @staticmethod + def _flat_to_nested( + metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + ) -> None: + """ + Convert a flat metadata representation to a nested one. + + Notes + ----- + Flat metadata is used when persisting the consolidated metadata. The keys + include the full path, not just the node name. The key prefixes can be + used to determine which nodes are children of which other nodes. + + Nested metadata is used in-memory. The outermost level will only have the + *immediate* children of the Group. All nested child groups will be stored + under the consolidated metadata of their immediate parent. + """ + # We have a flat mapping from {k: v} where the keys include the *full* + # path segment: + # { + # "/a/b": { group_metadata }, + # "/a/b/array-0": { array_metadata }, + # "/a/b/array-1": { array_metadata }, + # } + # + # We want to reorganize the metadata such that each Group contains the + # array metadata of its immediate children. + # In the example, the group at `/a/b` will have consolidated metadata + # for its children `array-0` and `array-1`. + # + # metadata = dict(metadata) + + keys = sorted(metadata, key=lambda k: k.count("/")) + grouped = { + k: list(v) for k, v in itertools.groupby(keys, key=lambda k: k.rsplit("/", 1)[0]) + } + + # we go top down and directly manipulate metadata. + for key, children_keys in grouped.items(): + # key is a key like "a", "a/b", "a/b/c" + # The basic idea is to find the immediate parent (so "", "a", or "a/b") + # and update that node's consolidated metadata to include the metadata + # in children_keys + *prefixes, name = key.split("/") + parent = metadata + + while prefixes: + # e.g. a/b/c has a parent "a/b". Walk through to get + # metadata["a"]["b"] + part = prefixes.pop(0) + # we can assume that parent[part] here is a group + # otherwise we wouldn't have a node with this `part` prefix. + # We can also assume that the parent node will have consolidated metadata, + # because we're walking top to bottom. + parent = parent[part].consolidated_metadata.metadata # type: ignore[union-attr] + + node = parent[name] + children_keys = list(children_keys) + + if isinstance(node, ArrayV2Metadata | ArrayV3Metadata): + # These are already present, either thanks to being an array in the + # root, or by being collected as a child in the else clause + continue + children_keys = list(children_keys) + # We pop from metadata, since we're *moving* this under group + children = { + child_key.split("/")[-1]: metadata.pop(child_key) + for child_key in children_keys + if child_key != key + } + parent[name] = replace( + node, consolidated_metadata=ConsolidatedMetadata(metadata=children) + ) + + @property + def flattened_metadata(self) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: + """ + Return the flattened representation of Consolidated Metadata. + + The returned dictionary will have a key for each child node in the hierarchy + under this group. Under the default (nested) representation available through + ``self.metadata``, the dictionary only contains keys for immediate children. + + The keys of the dictionary will include the full path to a child node from + the current group, where segments are joined by ``/``. + + Examples + -------- + >>> cm = ConsolidatedMetadata( + ... metadata={ + ... "group-0": GroupMetadata( + ... consolidated_metadata=ConsolidatedMetadata( + ... { + ... "group-0-0": GroupMetadata(), + ... } + ... ) + ... ), + ... "group-1": GroupMetadata(), + ... } + ... ) + {'group-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group'), + 'group-0/group-0-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group'), + 'group-1': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group')} + """ + metadata = {} + + def flatten( + key: str, group: GroupMetadata | ArrayV2Metadata | ArrayV3Metadata + ) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata]: + children: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {} + if isinstance(group, ArrayV2Metadata | ArrayV3Metadata): + children[key] = group + else: + if group.consolidated_metadata and group.consolidated_metadata.metadata is not None: + children[key] = replace( + group, consolidated_metadata=ConsolidatedMetadata(metadata={}) + ) + for name, val in group.consolidated_metadata.metadata.items(): + full_key = f"{key}/{name}" + if isinstance(val, GroupMetadata): + children.update(flatten(full_key, val)) + else: + children[full_key] = val + else: + children[key] = replace(group, consolidated_metadata=None) + return children + + for k, v in self.metadata.items(): + metadata.update(flatten(k, v)) + + return metadata + + +@dataclass(frozen=True) +class GroupMetadata(Metadata): + """ + Metadata for a Group. + """ + + attributes: dict[str, Any] = field(default_factory=dict) + zarr_format: ZarrFormat = 3 + consolidated_metadata: ConsolidatedMetadata | None = None + node_type: Literal["group"] = field(default="group", init=False) + + def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: + json_indent = config.get("json_indent") + if self.zarr_format == 3: + return { + ZARR_JSON: prototype.buffer.from_bytes( + json.dumps(self.to_dict(), indent=json_indent, allow_nan=False).encode() + ) + } + else: + items = { + ZGROUP_JSON: prototype.buffer.from_bytes( + json.dumps({"zarr_format": self.zarr_format}, indent=json_indent).encode() + ), + ZATTRS_JSON: prototype.buffer.from_bytes( + json.dumps(self.attributes, indent=json_indent, allow_nan=False).encode() + ), + } + if self.consolidated_metadata: + d = { + ZGROUP_JSON: {"zarr_format": self.zarr_format}, + ZATTRS_JSON: self.attributes, + } + consolidated_metadata = self.consolidated_metadata.to_dict()["metadata"] + assert isinstance(consolidated_metadata, dict) + for k, v in consolidated_metadata.items(): + attrs = v.pop("attributes", {}) + d[f"{k}/{ZATTRS_JSON}"] = attrs + if "shape" in v: + # it's an array + d[f"{k}/{ZARRAY_JSON}"] = v + else: + d[f"{k}/{ZGROUP_JSON}"] = { + "zarr_format": self.zarr_format, + "consolidated_metadata": { + "metadata": {}, + "must_understand": False, + "kind": "inline", + }, + } + + items[ZMETADATA_V2_JSON] = prototype.buffer.from_bytes( + json.dumps( + {"metadata": d, "zarr_consolidated_format": 1}, allow_nan=False + ).encode() + ) + + return items + + def __init__( + self, + attributes: dict[str, Any] | None = None, + zarr_format: ZarrFormat = 3, + consolidated_metadata: ConsolidatedMetadata | None = None, + ) -> None: + attributes_parsed = parse_attributes(attributes) + zarr_format_parsed = parse_zarr_format(zarr_format) + + object.__setattr__(self, "attributes", attributes_parsed) + object.__setattr__(self, "zarr_format", zarr_format_parsed) + object.__setattr__(self, "consolidated_metadata", consolidated_metadata) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> GroupMetadata: + data = dict(data) + assert data.pop("node_type", None) in ("group", None) + consolidated_metadata = data.pop("consolidated_metadata", None) + if consolidated_metadata: + data["consolidated_metadata"] = ConsolidatedMetadata.from_dict(consolidated_metadata) + + zarr_format = data.get("zarr_format") + if zarr_format == 2 or zarr_format is None: + # zarr v2 allowed arbitrary keys here. + # We don't want the GroupMetadata constructor to fail just because someone put an + # extra key in the metadata. + expected = {x.name for x in fields(cls)} + data = {k: v for k, v in data.items() if k in expected} + + return cls(**data) + + def to_dict(self) -> dict[str, Any]: + result = asdict(replace(self, consolidated_metadata=None)) + if self.consolidated_metadata: + result["consolidated_metadata"] = self.consolidated_metadata.to_dict() + return result + + +@dataclass(frozen=True) +class ImplicitGroupMarker(GroupMetadata): + """ + Marker for an implicit group. Instances of this class are only used in the context of group + creation as a placeholder to represent groups that should only be created if they do not + already exist in storage + """ + + +@dataclass(frozen=True) +class AsyncGroup: + """ + Asynchronous Group object. + """ + + metadata: GroupMetadata + store_path: StorePath + + # TODO: make this correct and work + # TODO: ensure that this can be bound properly to subclass of AsyncGroup + + @classmethod + async def from_store( + cls, + store: StoreLike, + *, + attributes: dict[str, Any] | None = None, + overwrite: bool = False, + zarr_format: ZarrFormat = 3, + ) -> AsyncGroup: + store_path = await make_store_path(store) + + if overwrite: + if store_path.store.supports_deletes: + await store_path.delete_dir() + else: + await ensure_no_existing_node(store_path, zarr_format=zarr_format) + else: + await ensure_no_existing_node(store_path, zarr_format=zarr_format) + attributes = attributes or {} + group = cls( + metadata=GroupMetadata(attributes=attributes, zarr_format=zarr_format), + store_path=store_path, + ) + await group._save_metadata(ensure_parents=True) + return group + + @classmethod + async def open( + cls, + store: StoreLike, + zarr_format: ZarrFormat | None = 3, + use_consolidated: bool | str | None = None, + ) -> AsyncGroup: + """Open a new AsyncGroup + + Parameters + ---------- + store : StoreLike + zarr_format : {2, 3}, optional + use_consolidated : bool or str, default None + Whether to use consolidated metadata. + + By default, consolidated metadata is used if it's present in the + store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file + for Zarr format 2) and the Store supports it. + + To explicitly require consolidated metadata, set ``use_consolidated=True``. + In this case, if the Store doesn't support consolidation or consolidated metadata is + not found, a ``ValueError`` exception is raised. + + To explicitly *not* use consolidated metadata, set ``use_consolidated=False``, + which will fall back to using the regular, non consolidated metadata. + + Zarr format 2 allowed configuring the key storing the consolidated metadata + (``.zmetadata`` by default). Specify the custom key as ``use_consolidated`` + to load consolidated metadata from a non-default key. + """ + store_path = await make_store_path(store) + if not store_path.store.supports_consolidated_metadata: + # Fail if consolidated metadata was requested but the Store doesn't support it + if use_consolidated: + store_name = type(store_path.store).__name__ + raise ValueError( + f"The Zarr store in use ({store_name}) doesn't support consolidated metadata." + ) + + # if use_consolidated was None (optional), the Store dictates it doesn't want consolidation + use_consolidated = False + + consolidated_key = ZMETADATA_V2_JSON + + if (zarr_format == 2 or zarr_format is None) and isinstance(use_consolidated, str): + consolidated_key = use_consolidated + + if zarr_format == 2: + paths = [store_path / ZGROUP_JSON, store_path / ZATTRS_JSON] + if use_consolidated or use_consolidated is None: + paths.append(store_path / consolidated_key) + + zgroup_bytes, zattrs_bytes, *rest = await asyncio.gather( + *[path.get() for path in paths] + ) + if zgroup_bytes is None: + raise FileNotFoundError(store_path) + + if use_consolidated or use_consolidated is None: + maybe_consolidated_metadata_bytes = rest[0] + + else: + maybe_consolidated_metadata_bytes = None + + elif zarr_format == 3: + zarr_json_bytes = await (store_path / ZARR_JSON).get() + if zarr_json_bytes is None: + raise FileNotFoundError(store_path) + elif zarr_format is None: + ( + zarr_json_bytes, + zgroup_bytes, + zattrs_bytes, + maybe_consolidated_metadata_bytes, + ) = await asyncio.gather( + (store_path / ZARR_JSON).get(), + (store_path / ZGROUP_JSON).get(), + (store_path / ZATTRS_JSON).get(), + (store_path / str(consolidated_key)).get(), + ) + if zarr_json_bytes is not None and zgroup_bytes is not None: + # warn and favor v3 + msg = f"Both zarr.json (Zarr format 3) and .zgroup (Zarr format 2) metadata objects exist at {store_path}. Zarr format 3 will be used." + warnings.warn(msg, stacklevel=1) + if zarr_json_bytes is None and zgroup_bytes is None: + raise FileNotFoundError( + f"could not find zarr.json or .zgroup objects in {store_path}" + ) + # set zarr_format based on which keys were found + if zarr_json_bytes is not None: + zarr_format = 3 + else: + zarr_format = 2 + else: + raise MetadataValidationError("zarr_format", "2, 3, or None", zarr_format) + + if zarr_format == 2: + # this is checked above, asserting here for mypy + assert zgroup_bytes is not None + + if use_consolidated and maybe_consolidated_metadata_bytes is None: + # the user requested consolidated metadata, but it was missing + raise ValueError(consolidated_key) + + elif use_consolidated is False: + # the user explicitly opted out of consolidated_metadata. + # Discard anything we might have read. + maybe_consolidated_metadata_bytes = None + + return cls._from_bytes_v2( + store_path, zgroup_bytes, zattrs_bytes, maybe_consolidated_metadata_bytes + ) + else: + # V3 groups are comprised of a zarr.json object + assert zarr_json_bytes is not None + if not isinstance(use_consolidated, bool | None): + raise TypeError("use_consolidated must be a bool or None for Zarr format 3.") + + return cls._from_bytes_v3( + store_path, + zarr_json_bytes, + use_consolidated=use_consolidated, + ) + + @classmethod + def _from_bytes_v2( + cls, + store_path: StorePath, + zgroup_bytes: Buffer, + zattrs_bytes: Buffer | None, + consolidated_metadata_bytes: Buffer | None, + ) -> AsyncGroup: + # V2 groups are comprised of a .zgroup and .zattrs objects + zgroup = json.loads(zgroup_bytes.to_bytes()) + zattrs = json.loads(zattrs_bytes.to_bytes()) if zattrs_bytes is not None else {} + group_metadata = {**zgroup, "attributes": zattrs} + + if consolidated_metadata_bytes is not None: + v2_consolidated_metadata = json.loads(consolidated_metadata_bytes.to_bytes()) + v2_consolidated_metadata = v2_consolidated_metadata["metadata"] + # We already read zattrs and zgroup. Should we ignore these? + v2_consolidated_metadata.pop(".zattrs", None) + v2_consolidated_metadata.pop(".zgroup", None) + + consolidated_metadata: defaultdict[str, dict[str, Any]] = defaultdict(dict) + + # keys like air/.zarray, air/.zattrs + for k, v in v2_consolidated_metadata.items(): + path, kind = k.rsplit("/.", 1) + + if kind == "zarray": + consolidated_metadata[path].update(v) + elif kind == "zattrs": + consolidated_metadata[path]["attributes"] = v + elif kind == "zgroup": + consolidated_metadata[path].update(v) + else: + raise ValueError(f"Invalid file type '{kind}' at path '{path}") + + group_metadata["consolidated_metadata"] = { + "metadata": dict(consolidated_metadata), + "kind": "inline", + "must_understand": False, + } + + return cls.from_dict(store_path, group_metadata) + + @classmethod + def _from_bytes_v3( + cls, + store_path: StorePath, + zarr_json_bytes: Buffer, + use_consolidated: bool | None, + ) -> AsyncGroup: + group_metadata = json.loads(zarr_json_bytes.to_bytes()) + if use_consolidated and group_metadata.get("consolidated_metadata") is None: + msg = f"Consolidated metadata requested with 'use_consolidated=True' but not found in '{store_path.path}'." + raise ValueError(msg) + + elif use_consolidated is False: + # Drop consolidated metadata if it's there. + group_metadata.pop("consolidated_metadata", None) + + return cls.from_dict(store_path, group_metadata) + + @classmethod + def from_dict( + cls, + store_path: StorePath, + data: dict[str, Any], + ) -> AsyncGroup: + return cls( + metadata=GroupMetadata.from_dict(data), + store_path=store_path, + ) + + async def setitem(self, key: str, value: Any) -> None: + """ + Fastpath for creating a new array + New arrays will be created with default array settings for the array type. + + Parameters + ---------- + key : str + Array name + value : array-like + Array data + """ + path = self.store_path / key + await async_api.save_array( + store=path, arr=value, zarr_format=self.metadata.zarr_format, overwrite=True + ) + + async def getitem( + self, + key: str, + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup: + """ + Get a subarray or subgroup from the group. + + Parameters + ---------- + key : str + Array or group name + + Returns + ------- + AsyncArray or AsyncGroup + """ + store_path = self.store_path / key + logger.debug("key=%s, store_path=%s", key, store_path) + + # Consolidated metadata lets us avoid some I/O operations so try that first. + if self.metadata.consolidated_metadata is not None: + return self._getitem_consolidated(store_path, key, prefix=self.name) + try: + return await get_node( + store=store_path.store, path=store_path.path, zarr_format=self.metadata.zarr_format + ) + except FileNotFoundError as e: + raise KeyError(key) from e + + def _getitem_consolidated( + self, store_path: StorePath, key: str, prefix: str + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup: + # getitem, in the special case where we have consolidated metadata. + # Note that this is a regular def (non async) function. + # This shouldn't do any additional I/O. + + # the caller needs to verify this! + assert self.metadata.consolidated_metadata is not None + + # we support nested getitems like group/subgroup/array + indexers = key.split("/") + indexers.reverse() + metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata = self.metadata + + while indexers: + indexer = indexers.pop() + if isinstance(metadata, ArrayV2Metadata | ArrayV3Metadata): + # we've indexed into an array with group["array/subarray"]. Invalid. + raise KeyError(key) + if metadata.consolidated_metadata is None: + # we've indexed into a group without consolidated metadata. + # This isn't normal; typically, consolidated metadata + # will include explicit markers for when there are no child + # nodes as metadata={}. + # We have some freedom in exactly how we interpret this case. + # For now, we treat None as the same as {}, i.e. we don't + # have any children. + raise KeyError(key) + try: + metadata = metadata.consolidated_metadata.metadata[indexer] + except KeyError as e: + # The Group Metadata has consolidated metadata, but the key + # isn't present. We trust this to mean that the key isn't in + # the hierarchy, and *don't* fall back to checking the store. + msg = f"'{key}' not found in consolidated metadata." + raise KeyError(msg) from e + + # update store_path to ensure that AsyncArray/Group.name is correct + if prefix != "/": + key = "/".join([prefix.lstrip("/"), key]) + store_path = StorePath(store=store_path.store, path=key) + + if isinstance(metadata, GroupMetadata): + return AsyncGroup(metadata=metadata, store_path=store_path) + else: + return AsyncArray(metadata=metadata, store_path=store_path) + + async def delitem(self, key: str) -> None: + """Delete a group member. + + Parameters + ---------- + key : str + Array or group name + """ + store_path = self.store_path / key + + await store_path.delete_dir() + if self.metadata.consolidated_metadata: + self.metadata.consolidated_metadata.metadata.pop(key, None) + await self._save_metadata() + + async def get( + self, key: str, default: DefaultT | None = None + ) -> AsyncArray[Any] | AsyncGroup | DefaultT | None: + """Obtain a group member, returning default if not found. + + Parameters + ---------- + key : str + Group member name. + default : object + Default value to return if key is not found (default: None). + + Returns + ------- + object + Group member (AsyncArray or AsyncGroup) or default if not found. + """ + try: + return await self.getitem(key) + except KeyError: + return default + + async def _save_metadata(self, ensure_parents: bool = False) -> None: + to_save = self.metadata.to_buffer_dict(default_buffer_prototype()) + awaitables = [set_or_delete(self.store_path / key, value) for key, value in to_save.items()] + + if ensure_parents: + parents = _build_parents(self) + for parent in parents: + awaitables.extend( + [ + (parent.store_path / key).set_if_not_exists(value) + for key, value in parent.metadata.to_buffer_dict( + default_buffer_prototype() + ).items() + ] + ) + + await asyncio.gather(*awaitables) + + @property + def path(self) -> str: + """Storage path.""" + return self.store_path.path + + @property + def name(self) -> str: + """Group name following h5py convention.""" + if self.path: + # follow h5py convention: add leading slash + name = self.path + if name[0] != "/": + name = "/" + name + return name + return "/" + + @property + def basename(self) -> str: + """Final component of name.""" + return self.name.split("/")[-1] + + @property + def attrs(self) -> dict[str, Any]: + return self.metadata.attributes + + @property + def info(self) -> Any: + """ + Return a visual representation of the statically known information about a group. + + Note that this doesn't include dynamic information, like the number of child + Groups or Arrays. + + Returns + ------- + GroupInfo + + See Also + -------- + AsyncGroup.info_complete + All information about a group, including dynamic information + """ + + if self.metadata.consolidated_metadata: + members = list(self.metadata.consolidated_metadata.flattened_metadata.values()) + else: + members = None + return self._info(members=members) + + async def info_complete(self) -> Any: + """ + Return all the information for a group. + + This includes dynamic information like the number + of child Groups or Arrays. If this group doesn't contain consolidated + metadata then this will need to read from the backing Store. + + Returns + ------- + GroupInfo + + See Also + -------- + AsyncGroup.info + """ + members = [x[1].metadata async for x in self.members(max_depth=None)] + return self._info(members=members) + + def _info( + self, members: list[ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] | None = None + ) -> Any: + kwargs = {} + if members is not None: + kwargs["_count_members"] = len(members) + count_arrays = 0 + count_groups = 0 + for member in members: + if isinstance(member, GroupMetadata): + count_groups += 1 + else: + count_arrays += 1 + kwargs["_count_arrays"] = count_arrays + kwargs["_count_groups"] = count_groups + + return GroupInfo( + _name=self.store_path.path, + _read_only=self.read_only, + _store_type=type(self.store_path.store).__name__, + _zarr_format=self.metadata.zarr_format, + # maybe do a typeddict + **kwargs, # type: ignore[arg-type] + ) + + @property + def store(self) -> Store: + return self.store_path.store + + @property + def read_only(self) -> bool: + # Backwards compatibility for 2.x + return self.store_path.read_only + + @property + def synchronizer(self) -> None: + # Backwards compatibility for 2.x + # Not implemented in 3.x yet. + return None + + async def create_group( + self, + name: str, + *, + overwrite: bool = False, + attributes: dict[str, Any] | None = None, + ) -> AsyncGroup: + """Create a sub-group. + + Parameters + ---------- + name : str + Group name. + overwrite : bool, optional + If True, do not raise an error if the group already exists. + attributes : dict, optional + Group attributes. + + Returns + ------- + g : AsyncGroup + """ + attributes = attributes or {} + return await type(self).from_store( + self.store_path / name, + attributes=attributes, + overwrite=overwrite, + zarr_format=self.metadata.zarr_format, + ) + + async def require_group(self, name: str, overwrite: bool = False) -> AsyncGroup: + """Obtain a sub-group, creating one if it doesn't exist. + + Parameters + ---------- + name : str + Group name. + overwrite : bool, optional + Overwrite any existing group with given `name` if present. + + Returns + ------- + g : AsyncGroup + """ + if overwrite: + # TODO: check that overwrite=True errors if an array exists where the group is being created + grp = await self.create_group(name, overwrite=True) + else: + try: + item: ( + AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] + ) = await self.getitem(name) + if not isinstance(item, AsyncGroup): + raise TypeError( + f"Incompatible object ({item.__class__.__name__}) already exists" + ) + assert isinstance(item, AsyncGroup) # make mypy happy + grp = item + except KeyError: + grp = await self.create_group(name) + return grp + + async def require_groups(self, *names: str) -> tuple[AsyncGroup, ...]: + """Convenience method to require multiple groups in a single call. + + Parameters + ---------- + *names : str + Group names. + + Returns + ------- + Tuple[AsyncGroup, ...] + """ + if not names: + return () + return tuple(await asyncio.gather(*(self.require_group(name) for name in names))) + + async def create_array( + self, + name: str, + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + compressor: CompressorLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = 0, + order: MemoryOrder | None = None, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfig | ArrayConfigLike | None = None, + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array within this group. + + This method lightly wraps :func:`zarr.core.array.create_array`. + + Parameters + ---------- + name : str + The name of the array relative to the group. If ``path`` is ``None``, the array will be located + at the root of the store. + shape : ChunkCoords + Shape of the array. + dtype : npt.DTypeLike + Data type of the array. + chunks : ChunkCoords, optional + Chunk shape of the array. + If not specified, default are guessed based on the shape and dtype. + shards : ChunkCoords, optional + Shard shape of the array. The default value of ``None`` results in no sharding at all. + filters : Iterable[Codec], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v3_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v2_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no ``compressor`` is provided, a default compressor will be used. + in :mod:`zarr.core.config`. + Use ``None`` to omit the default compressor. + compressor : Codec, optional + Deprecated in favor of ``compressors``. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` + in :mod:`zarr.core.config`. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If no ``order`` is provided, a default order will be used. + This default can be changed by modifying the value of ``array.order`` in :mod:`zarr.core.config`. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncoding, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfig or ArrayConfigLike, optional + Runtime configuration for the array. + + Returns + ------- + AsyncArray + + """ + compressors = _parse_deprecated_compressor( + compressor, compressors, zarr_format=self.metadata.zarr_format + ) + return await create_array( + store=self.store_path, + name=name, + shape=shape, + dtype=dtype, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=self.metadata.zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + storage_options=storage_options, + overwrite=overwrite, + config=config, + ) + + @deprecated("Use AsyncGroup.create_array instead.") + async def create_dataset( + self, name: str, *, shape: ShapeLike, **kwargs: Any + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array. + + .. deprecated:: 3.0.0 + The h5py compatibility methods will be removed in 3.1.0. Use `AsyncGroup.create_array` instead. + + Arrays are known as "datasets" in HDF5 terminology. For compatibility + with h5py, Zarr groups also implement the :func:`zarr.AsyncGroup.require_dataset` method. + + Parameters + ---------- + name : str + Array name. + **kwargs : dict + Additional arguments passed to :func:`zarr.AsyncGroup.create_array`. + + Returns + ------- + a : AsyncArray + """ + data = kwargs.pop("data", None) + # create_dataset in zarr 2.x requires shape but not dtype if data is + # provided. Allow this configuration by inferring dtype from data if + # necessary and passing it to create_array + if "dtype" not in kwargs and data is not None: + kwargs["dtype"] = data.dtype + array = await self.create_array(name, shape=shape, **kwargs) + if data is not None: + await array.setitem(slice(None), data) + return array + + @deprecated("Use AsyncGroup.require_array instead.") + async def require_dataset( + self, + name: str, + *, + shape: ChunkCoords, + dtype: npt.DTypeLike = None, + exact: bool = False, + **kwargs: Any, + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Obtain an array, creating if it doesn't exist. + + .. deprecated:: 3.0.0 + The h5py compatibility methods will be removed in 3.1.0. Use `AsyncGroup.require_dataset` instead. + + Arrays are known as "datasets" in HDF5 terminology. For compatibility + with h5py, Zarr groups also implement the :func:`zarr.AsyncGroup.create_dataset` method. + + Other `kwargs` are as per :func:`zarr.AsyncGroup.create_dataset`. + + Parameters + ---------- + name : str + Array name. + shape : int or tuple of ints + Array shape. + dtype : str or dtype, optional + NumPy dtype. + exact : bool, optional + If True, require `dtype` to match exactly. If false, require + `dtype` can be cast from array dtype. + + Returns + ------- + a : AsyncArray + """ + return await self.require_array(name, shape=shape, dtype=dtype, exact=exact, **kwargs) + + async def require_array( + self, + name: str, + *, + shape: ShapeLike, + dtype: npt.DTypeLike = None, + exact: bool = False, + **kwargs: Any, + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Obtain an array, creating if it doesn't exist. + + Other `kwargs` are as per :func:`zarr.AsyncGroup.create_dataset`. + + Parameters + ---------- + name : str + Array name. + shape : int or tuple of ints + Array shape. + dtype : str or dtype, optional + NumPy dtype. + exact : bool, optional + If True, require `dtype` to match exactly. If false, require + `dtype` can be cast from array dtype. + + Returns + ------- + a : AsyncArray + """ + try: + ds = await self.getitem(name) + if not isinstance(ds, AsyncArray): + raise TypeError(f"Incompatible object ({ds.__class__.__name__}) already exists") + + shape = parse_shapelike(shape) + if shape != ds.shape: + raise TypeError(f"Incompatible shape ({ds.shape} vs {shape})") + + dtype = np.dtype(dtype) + if exact: + if ds.dtype != dtype: + raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") + else: + if not np.can_cast(ds.dtype, dtype): + raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") + except KeyError: + ds = await self.create_array(name, shape=shape, dtype=dtype, **kwargs) + + return ds + + async def update_attributes(self, new_attributes: dict[str, Any]) -> AsyncGroup: + """Update group attributes. + + Parameters + ---------- + new_attributes : dict + New attributes to set on the group. + + Returns + ------- + self : AsyncGroup + """ + self.metadata.attributes.update(new_attributes) + + # Write new metadata + await self._save_metadata() + + return self + + def __repr__(self) -> str: + return f"" + + async def nmembers( + self, + max_depth: int | None = 0, + ) -> int: + """Count the number of members in this group. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + + Returns + ------- + count : int + """ + # check if we can use consolidated metadata, which requires that we have non-None + # consolidated metadata at all points in the hierarchy. + if self.metadata.consolidated_metadata is not None: + return len(self.metadata.consolidated_metadata.flattened_metadata) + # TODO: consider using aioitertools.builtins.sum for this + # return await aioitertools.builtins.sum((1 async for _ in self.members()), start=0) + n = 0 + async for _ in self.members(max_depth=max_depth): + n += 1 + return n + + async def members( + self, + max_depth: int | None = 0, + *, + use_consolidated_for_children: bool = True, + ) -> AsyncGenerator[ + tuple[str, AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup], + None, + ]: + """ + Returns an AsyncGenerator over the arrays and groups contained in this group. + This method requires that `store_path.store` supports directory listing. + + The results are not guaranteed to be ordered. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Returns + ------- + path: + A string giving the path to the target, relative to the Group ``self``. + value: AsyncArray or AsyncGroup + The AsyncArray or AsyncGroup that is a child of ``self``. + """ + if max_depth is not None and max_depth < 0: + raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") + async for item in self._members( + max_depth=max_depth, use_consolidated_for_children=use_consolidated_for_children + ): + yield item + + def _members_consolidated( + self, max_depth: int | None, prefix: str = "" + ) -> Generator[ + tuple[str, AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup], + None, + ]: + consolidated_metadata = self.metadata.consolidated_metadata + + do_recursion = max_depth is None or max_depth > 0 + + # we kind of just want the top-level keys. + if consolidated_metadata is not None: + for key in consolidated_metadata.metadata: + obj = self._getitem_consolidated( + self.store_path, key, prefix=self.name + ) # Metadata -> Group/Array + key = f"{prefix}/{key}".lstrip("/") + yield key, obj + + if do_recursion and isinstance(obj, AsyncGroup): + if max_depth is None: + new_depth = None + else: + new_depth = max_depth - 1 + yield from obj._members_consolidated(new_depth, prefix=key) + + async def _members( + self, max_depth: int | None, *, use_consolidated_for_children: bool = True + ) -> AsyncGenerator[ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup], None + ]: + skip_keys: tuple[str, ...] + if self.metadata.zarr_format == 2: + skip_keys = (".zattrs", ".zgroup", ".zarray", ".zmetadata") + elif self.metadata.zarr_format == 3: + skip_keys = ("zarr.json",) + else: + raise ValueError(f"Unknown Zarr format: {self.metadata.zarr_format}") + + if self.metadata.consolidated_metadata is not None: + members = self._members_consolidated(max_depth=max_depth) + for member in members: + yield member + return + + if not self.store_path.store.supports_listing: + msg = ( + f"The store associated with this group ({type(self.store_path.store)}) " + "does not support listing, " + "specifically via the `list_dir` method. " + "This function requires a store that supports listing." + ) + + raise ValueError(msg) + # enforce a concurrency limit by passing a semaphore to all the recursive functions + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + async for member in _iter_members_deep( + self, + max_depth=max_depth, + skip_keys=skip_keys, + semaphore=semaphore, + use_consolidated_for_children=use_consolidated_for_children, + ): + yield member + + async def create_hierarchy( + self, + nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + *, + overwrite: bool = False, + ) -> AsyncIterator[ + tuple[str, AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]] + ]: + """ + Create a hierarchy of arrays or groups rooted at this group. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. + + Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error + because this group instance is the root group. + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, AsyncArray | AsyncGroup]. + """ + # check that all the nodes have the same zarr_format as Self + prefix = self.path + nodes_parsed = {} + for key, value in nodes.items(): + if value.zarr_format != self.metadata.zarr_format: + msg = ( + "The zarr_format of the nodes must be the same as the parent group. " + f"The node at {key} has zarr_format {value.zarr_format}, but the parent group" + f" has zarr_format {self.metadata.zarr_format}." + ) + raise ValueError(msg) + if normalize_path(key) == "": + msg = ( + "The input defines a root node, but a root node already exists, namely this Group instance." + "It is an error to use this method to create a root node. " + "Remove the root node from the input dict, or use a function like " + "create_rooted_hierarchy to create a rooted hierarchy." + ) + raise ValueError(msg) + else: + nodes_parsed[_join_paths([prefix, key])] = value + + async for key, node in create_hierarchy( + store=self.store, + nodes=nodes_parsed, + overwrite=overwrite, + ): + if prefix == "": + out_key = key + else: + out_key = key.removeprefix(prefix + "/") + yield out_key, node + + async def keys(self) -> AsyncGenerator[str, None]: + """Iterate over member names.""" + async for key, _ in self.members(): + yield key + + async def contains(self, member: str) -> bool: + """Check if a member exists in the group. + + Parameters + ---------- + member : str + Member name. + + Returns + ------- + bool + """ + # TODO: this can be made more efficient. + try: + await self.getitem(member) + except KeyError: + return False + else: + return True + + async def groups(self) -> AsyncGenerator[tuple[str, AsyncGroup], None]: + """Iterate over subgroups.""" + async for name, value in self.members(): + if isinstance(value, AsyncGroup): + yield name, value + + async def group_keys(self) -> AsyncGenerator[str, None]: + """Iterate over group names.""" + async for key, _ in self.groups(): + yield key + + async def group_values(self) -> AsyncGenerator[AsyncGroup, None]: + """Iterate over group values.""" + async for _, group in self.groups(): + yield group + + async def arrays( + self, + ) -> AsyncGenerator[ + tuple[str, AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]], None + ]: + """Iterate over arrays.""" + async for key, value in self.members(): + if isinstance(value, AsyncArray): + yield key, value + + async def array_keys(self) -> AsyncGenerator[str, None]: + """Iterate over array names.""" + async for key, _ in self.arrays(): + yield key + + async def array_values( + self, + ) -> AsyncGenerator[AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], None]: + """Iterate over array values.""" + async for _, array in self.arrays(): + yield array + + async def tree(self, expand: bool | None = None, level: int | None = None) -> Any: + """ + Return a tree-like representation of a hierarchy. + + This requires the optional ``rich`` dependency. + + Parameters + ---------- + expand : bool, optional + This keyword is not yet supported. A NotImplementedError is raised if + it's used. + level : int, optional + The maximum depth below this Group to display in the tree. + + Returns + ------- + TreeRepr + A pretty-printable object displaying the hierarchy. + """ + from zarr.core._tree import group_tree_async + + if expand is not None: + raise NotImplementedError("'expand' is not yet implemented.") + return await group_tree_async(self, max_depth=level) + + async def empty( + self, *, name: str, shape: ChunkCoords, **kwargs: Any + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an empty array with the specified shape in this Group. The contents will + be filled with the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + return await async_api.empty(shape=shape, store=self.store_path, path=name, **kwargs) + + async def zeros( + self, *, name: str, shape: ChunkCoords, **kwargs: Any + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array, with zero being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.zeros(shape=shape, store=self.store_path, path=name, **kwargs) + + async def ones( + self, *, name: str, shape: ChunkCoords, **kwargs: Any + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array, with one being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.ones(shape=shape, store=self.store_path, path=name, **kwargs) + + async def full( + self, *, name: str, shape: ChunkCoords, fill_value: Any | None, **kwargs: Any + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + fill_value : scalar + Value to fill the array with. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.full( + shape=shape, + fill_value=fill_value, + store=self.store_path, + path=name, + **kwargs, + ) + + async def empty_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create an empty sub-array like `data`. The contents will be filled with + the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.empty_like(a=data, store=self.store_path, path=name, **kwargs) + + async def zeros_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create a sub-array of zeros like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.zeros_like(a=data, store=self.store_path, path=name, **kwargs) + + async def ones_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create a sub-array of ones like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.ones_like(a=data, store=self.store_path, path=name, **kwargs) + + async def full_like( + self, *, name: str, data: async_api.ArrayLike, **kwargs: Any + ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """Create a sub-array like `data` filled with the `fill_value` of `data` . + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + AsyncArray + The new array. + """ + return await async_api.full_like(a=data, store=self.store_path, path=name, **kwargs) + + async def move(self, source: str, dest: str) -> None: + """Move a sub-group or sub-array from one path to another. + + Notes + ----- + Not implemented + """ + raise NotImplementedError + + +@dataclass(frozen=True) +class Group(SyncMixin): + """ + A Zarr group. + """ + + _async_group: AsyncGroup + + @classmethod + def from_store( + cls, + store: StoreLike, + *, + attributes: dict[str, Any] | None = None, + zarr_format: ZarrFormat = 3, + overwrite: bool = False, + ) -> Group: + """Instantiate a group from an initialized store. + + Parameters + ---------- + store : StoreLike + StoreLike containing the Group. + attributes : dict, optional + A dictionary of JSON-serializable values with user-defined attributes. + zarr_format : {2, 3}, optional + Zarr storage format version. + overwrite : bool, optional + If True, do not raise an error if the group already exists. + + Returns + ------- + Group + Group instantiated from the store. + + Raises + ------ + ContainsArrayError, ContainsGroupError, ContainsArrayAndGroupError + """ + attributes = attributes or {} + obj = sync( + AsyncGroup.from_store( + store, + attributes=attributes, + overwrite=overwrite, + zarr_format=zarr_format, + ), + ) + + return cls(obj) + + @classmethod + def open( + cls, + store: StoreLike, + zarr_format: ZarrFormat | None = 3, + ) -> Group: + """Open a group from an initialized store. + + Parameters + ---------- + store : StoreLike + Store containing the Group. + zarr_format : {2, 3, None}, optional + Zarr storage format version. + + Returns + ------- + Group + Group instantiated from the store. + """ + obj = sync(AsyncGroup.open(store, zarr_format=zarr_format)) + return cls(obj) + + def __getitem__(self, path: str) -> Array | Group: + """Obtain a group member. + + Parameters + ---------- + path : str + Group member name. + + Returns + ------- + Array | Group + Group member (Array or Group) at the specified key + + Examples + -------- + >>> import zarr + >>> group = Group.from_store(zarr.storage.MemoryStore() + >>> group.create_array(name="subarray", shape=(10,), chunks=(10,)) + >>> group.create_group(name="subgroup").create_array(name="subarray", shape=(10,), chunks=(10,)) + >>> group["subarray"] + + >>> group["subgroup"] + + >>> group["subgroup"]["subarray"] + + + """ + obj = self._sync(self._async_group.getitem(path)) + if isinstance(obj, AsyncArray): + return Array(obj) + else: + return Group(obj) + + def get(self, path: str, default: DefaultT | None = None) -> Array | Group | DefaultT | None: + """Obtain a group member, returning default if not found. + + Parameters + ---------- + path : str + Group member name. + default : object + Default value to return if key is not found (default: None). + + Returns + ------- + object + Group member (Array or Group) or default if not found. + + Examples + -------- + >>> import zarr + >>> group = Group.from_store(zarr.storage.MemoryStore() + >>> group.create_array(name="subarray", shape=(10,), chunks=(10,)) + >>> group.create_group(name="subgroup") + >>> group.get("subarray") + + >>> group.get("subgroup") + + >>> group.get("nonexistent", None) + + """ + try: + return self[path] + except KeyError: + return default + + def __delitem__(self, key: str) -> None: + """Delete a group member. + + Parameters + ---------- + key : str + Group member name. + + Examples + -------- + >>> import zarr + >>> group = Group.from_store(zarr.storage.MemoryStore() + >>> group.create_array(name="subarray", shape=(10,), chunks=(10,)) + >>> del group["subarray"] + >>> "subarray" in group + False + """ + self._sync(self._async_group.delitem(key)) + + def __iter__(self) -> Iterator[str]: + """Return an iterator over group member names. + Examples + -------- + >>> import zarr + >>> g1 = zarr.group() + >>> g2 = g1.create_group('foo') + >>> g3 = g1.create_group('bar') + >>> d1 = g1.create_array('baz', shape=(10,), chunks=(10,)) + >>> d2 = g1.create_array('quux', shape=(10,), chunks=(10,)) + >>> for name in g1: + ... print(name) + baz + bar + foo + quux + """ + yield from self.keys() + + def __len__(self) -> int: + """Number of members.""" + return self.nmembers() + + def __setitem__(self, key: str, value: Any) -> None: + """Fastpath for creating a new array. + + New arrays will be created using default settings for the array type. + If you need to create an array with custom settings, use the `create_array` method. + + Parameters + ---------- + key : str + Array name. + value : Any + Array data. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> group["foo"] = zarr.zeros((10,)) + >>> group["foo"] + + """ + self._sync(self._async_group.setitem(key, value)) + + def __repr__(self) -> str: + return f"" + + async def update_attributes_async(self, new_attributes: dict[str, Any]) -> Group: + """Update the attributes of this group. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> await group.update_attributes_async({"foo": "bar"}) + >>> group.attrs.asdict() + {'foo': 'bar'} + """ + new_metadata = replace(self.metadata, attributes=new_attributes) + + # Write new metadata + to_save = new_metadata.to_buffer_dict(default_buffer_prototype()) + awaitables = [set_or_delete(self.store_path / key, value) for key, value in to_save.items()] + await asyncio.gather(*awaitables) + + async_group = replace(self._async_group, metadata=new_metadata) + return replace(self, _async_group=async_group) + + @property + def store_path(self) -> StorePath: + """Path-like interface for the Store.""" + return self._async_group.store_path + + @property + def metadata(self) -> GroupMetadata: + """Group metadata.""" + return self._async_group.metadata + + @property + def path(self) -> str: + """Storage path.""" + return self._async_group.path + + @property + def name(self) -> str: + """Group name following h5py convention.""" + return self._async_group.name + + @property + def basename(self) -> str: + """Final component of name.""" + return self._async_group.basename + + @property + def attrs(self) -> Attributes: + """Attributes of this Group""" + return Attributes(self) + + @property + def info(self) -> Any: + """ + Return the statically known information for a group. + + Returns + ------- + GroupInfo + + See Also + -------- + Group.info_complete + All information about a group, including dynamic information + like the children members. + """ + return self._async_group.info + + def info_complete(self) -> Any: + """ + Return information for a group. + + If this group doesn't contain consolidated metadata then + this will need to read from the backing Store. + + Returns + ------- + GroupInfo + + See Also + -------- + Group.info + """ + return self._sync(self._async_group.info_complete()) + + @property + def store(self) -> Store: + # Backwards compatibility for 2.x + return self._async_group.store + + @property + def read_only(self) -> bool: + # Backwards compatibility for 2.x + return self._async_group.read_only + + @property + def synchronizer(self) -> None: + # Backwards compatibility for 2.x + # Not implemented in 3.x yet. + return self._async_group.synchronizer + + def update_attributes(self, new_attributes: dict[str, Any]) -> Group: + """Update the attributes of this group. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> group.update_attributes({"foo": "bar"}) + >>> group.attrs.asdict() + {'foo': 'bar'} + """ + self._sync(self._async_group.update_attributes(new_attributes)) + return self + + def nmembers(self, max_depth: int | None = 0) -> int: + """Count the number of members in this group. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + + Returns + ------- + count : int + """ + + return self._sync(self._async_group.nmembers(max_depth=max_depth)) + + def members( + self, max_depth: int | None = 0, *, use_consolidated_for_children: bool = True + ) -> tuple[tuple[str, Array | Group], ...]: + """ + Returns an AsyncGenerator over the arrays and groups contained in this group. + This method requires that `store_path.store` supports directory listing. + + The results are not guaranteed to be ordered. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Returns + ------- + path: + A string giving the path to the target, relative to the Group ``self``. + value: AsyncArray or AsyncGroup + The AsyncArray or AsyncGroup that is a child of ``self``. + """ + _members = self._sync_iter(self._async_group.members(max_depth=max_depth)) + + return tuple((kv[0], _parse_async_node(kv[1])) for kv in _members) + + def create_hierarchy( + self, + nodes: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata], + *, + overwrite: bool = False, + ) -> Iterator[tuple[str, Group | Array]]: + """ + Create a hierarchy of arrays or groups rooted at this group. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}```. + + Explicitly specifying a root group, e.g. with ``nodes = {'': GroupMetadata()}`` is an error + because this group instance is the root group. + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the path of the group. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` as the parent group -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------- + tuple[str, Array | Group]. + + Examples + -------- + >>> import zarr + >>> from zarr.core.group import GroupMetadata + >>> root = zarr.create_group(store={}) + >>> for key, val in root.create_hierarchy({'a/b/c': GroupMetadata()}): + ... print(key, val) + ... + + + + """ + for key, node in self._sync_iter( + self._async_group.create_hierarchy(nodes, overwrite=overwrite) + ): + yield (key, _parse_async_node(node)) + + def keys(self) -> Generator[str, None]: + """Return an iterator over group member names. + + Examples + -------- + >>> import zarr + >>> g1 = zarr.group() + >>> g2 = g1.create_group('foo') + >>> g3 = g1.create_group('bar') + >>> d1 = g1.create_array('baz', shape=(10,), chunks=(10,)) + >>> d2 = g1.create_array('quux', shape=(10,), chunks=(10,)) + >>> for name in g1.keys(): + ... print(name) + baz + bar + foo + quux + """ + yield from self._sync_iter(self._async_group.keys()) + + def __contains__(self, member: str) -> bool: + """Test for group membership. + + Examples + -------- + >>> import zarr + >>> g1 = zarr.group() + >>> g2 = g1.create_group('foo') + >>> d1 = g1.create_array('bar', shape=(10,), chunks=(10,)) + >>> 'foo' in g1 + True + >>> 'bar' in g1 + True + >>> 'baz' in g1 + False + + """ + return self._sync(self._async_group.contains(member)) + + def groups(self) -> Generator[tuple[str, Group], None]: + """Return the sub-groups of this group as a generator of (name, group) pairs. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> group.create_group("subgroup") + >>> for name, subgroup in group.groups(): + ... print(name, subgroup) + subgroup + """ + for name, async_group in self._sync_iter(self._async_group.groups()): + yield name, Group(async_group) + + def group_keys(self) -> Generator[str, None]: + """Return an iterator over group member names. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> group.create_group("subgroup") + >>> for name in group.group_keys(): + ... print(name) + subgroup + """ + for name, _ in self.groups(): + yield name + + def group_values(self) -> Generator[Group, None]: + """Return an iterator over group members. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> group.create_group("subgroup") + >>> for subgroup in group.group_values(): + ... print(subgroup) + + """ + for _, group in self.groups(): + yield group + + def arrays(self) -> Generator[tuple[str, Array], None]: + """Return the sub-arrays of this group as a generator of (name, array) pairs + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> group.create_array("subarray", shape=(10,), chunks=(10,)) + >>> for name, subarray in group.arrays(): + ... print(name, subarray) + subarray + """ + for name, async_array in self._sync_iter(self._async_group.arrays()): + yield name, Array(async_array) + + def array_keys(self) -> Generator[str, None]: + """Return an iterator over group member names. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> group.create_array("subarray", shape=(10,), chunks=(10,)) + >>> for name in group.array_keys(): + ... print(name) + subarray + """ + + for name, _ in self.arrays(): + yield name + + def array_values(self) -> Generator[Array, None]: + """Return an iterator over group members. + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> group.create_array("subarray", shape=(10,), chunks=(10,)) + >>> for subarray in group.array_values(): + ... print(subarray) + + """ + for _, array in self.arrays(): + yield array + + def tree(self, expand: bool | None = None, level: int | None = None) -> Any: + """ + Return a tree-like representation of a hierarchy. + + This requires the optional ``rich`` dependency. + + Parameters + ---------- + expand : bool, optional + This keyword is not yet supported. A NotImplementedError is raised if + it's used. + level : int, optional + The maximum depth below this Group to display in the tree. + + Returns + ------- + TreeRepr + A pretty-printable object displaying the hierarchy. + """ + return self._sync(self._async_group.tree(expand=expand, level=level)) + + def create_group(self, name: str, **kwargs: Any) -> Group: + """Create a sub-group. + + Parameters + ---------- + name : str + Name of the new subgroup. + + Returns + ------- + Group + + Examples + -------- + >>> import zarr + >>> group = zarr.group() + >>> subgroup = group.create_group("subgroup") + >>> subgroup + + """ + return Group(self._sync(self._async_group.create_group(name, **kwargs))) + + def require_group(self, name: str, **kwargs: Any) -> Group: + """Obtain a sub-group, creating one if it doesn't exist. + + Parameters + ---------- + name : str + Group name. + + Returns + ------- + g : Group + """ + return Group(self._sync(self._async_group.require_group(name, **kwargs))) + + def require_groups(self, *names: str) -> tuple[Group, ...]: + """Convenience method to require multiple groups in a single call. + + Parameters + ---------- + *names : str + Group names. + + Returns + ------- + groups : tuple of Groups + """ + return tuple(map(Group, self._sync(self._async_group.require_groups(*names)))) + + def create(self, *args: Any, **kwargs: Any) -> Array: + # Backwards compatibility for 2.x + return self.create_array(*args, **kwargs) + + @_deprecate_positional_args + def create_array( + self, + name: str, + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + compressor: CompressorLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any | None = 0, + order: MemoryOrder | None = "C", + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfig | ArrayConfigLike | None = None, + ) -> Array: + """Create an array within this group. + + This method lightly wraps :func:`zarr.core.array.create_array`. + + Parameters + ---------- + name : str + The name of the array relative to the group. If ``path`` is ``None``, the array will be located + at the root of the store. + shape : ChunkCoords + Shape of the array. + dtype : npt.DTypeLike + Data type of the array. + chunks : ChunkCoords, optional + Chunk shape of the array. + If not specified, default are guessed based on the shape and dtype. + shards : ChunkCoords, optional + Shard shape of the array. The default value of ``None`` results in no sharding at all. + filters : Iterable[Codec], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v3_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v2_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no ``compressor`` is provided, a default compressor will be used. + in :mod:`zarr.core.config`. + Use ``None`` to omit the default compressor. + compressor : Codec, optional + Deprecated in favor of ``compressors``. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` + in :mod:`zarr.core.config`. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If no ``order`` is provided, a default order will be used. + This default can be changed by modifying the value of ``array.order`` in :mod:`zarr.core.config`. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncoding, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfig or ArrayConfigLike, optional + Runtime configuration for the array. + + Returns + ------- + AsyncArray + """ + compressors = _parse_deprecated_compressor( + compressor, compressors, zarr_format=self.metadata.zarr_format + ) + return Array( + self._sync( + self._async_group.create_array( + name=name, + shape=shape, + dtype=dtype, + chunks=chunks, + shards=shards, + fill_value=fill_value, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + compressors=compressors, + serializer=serializer, + dimension_names=dimension_names, + order=order, + filters=filters, + overwrite=overwrite, + storage_options=storage_options, + config=config, + ) + ) + ) + + @deprecated("Use Group.create_array instead.") + def create_dataset(self, name: str, **kwargs: Any) -> Array: + """Create an array. + + .. deprecated:: 3.0.0 + The h5py compatibility methods will be removed in 3.1.0. Use `Group.create_array` instead. + + + Arrays are known as "datasets" in HDF5 terminology. For compatibility + with h5py, Zarr groups also implement the :func:`zarr.Group.require_dataset` method. + + Parameters + ---------- + name : str + Array name. + **kwargs : dict + Additional arguments passed to :func:`zarr.Group.create_array` + + Returns + ------- + a : Array + """ + return Array(self._sync(self._async_group.create_dataset(name, **kwargs))) + + @deprecated("Use Group.require_array instead.") + def require_dataset(self, name: str, *, shape: ShapeLike, **kwargs: Any) -> Array: + """Obtain an array, creating if it doesn't exist. + + .. deprecated:: 3.0.0 + The h5py compatibility methods will be removed in 3.1.0. Use `Group.require_array` instead. + + Arrays are known as "datasets" in HDF5 terminology. For compatibility + with h5py, Zarr groups also implement the :func:`zarr.Group.create_dataset` method. + + Other `kwargs` are as per :func:`zarr.Group.create_dataset`. + + Parameters + ---------- + name : str + Array name. + **kwargs : + See :func:`zarr.Group.create_dataset`. + + Returns + ------- + a : Array + """ + return Array(self._sync(self._async_group.require_array(name, shape=shape, **kwargs))) + + def require_array(self, name: str, *, shape: ShapeLike, **kwargs: Any) -> Array: + """Obtain an array, creating if it doesn't exist. + + Other `kwargs` are as per :func:`zarr.Group.create_array`. + + Parameters + ---------- + name : str + Array name. + **kwargs : + See :func:`zarr.Group.create_array`. + + Returns + ------- + a : Array + """ + return Array(self._sync(self._async_group.require_array(name, shape=shape, **kwargs))) + + @_deprecate_positional_args + def empty(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array: + """Create an empty array with the specified shape in this Group. The contents will be filled with + the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + return Array(self._sync(self._async_group.empty(name=name, shape=shape, **kwargs))) + + @_deprecate_positional_args + def zeros(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array: + """Create an array, with zero being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.zeros(name=name, shape=shape, **kwargs))) + + @_deprecate_positional_args + def ones(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array: + """Create an array, with one being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.ones(name=name, shape=shape, **kwargs))) + + @_deprecate_positional_args + def full( + self, *, name: str, shape: ChunkCoords, fill_value: Any | None, **kwargs: Any + ) -> Array: + """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. + + Parameters + ---------- + name : str + Name of the array. + shape : int or tuple of int + Shape of the empty array. + fill_value : scalar + Value to fill the array with. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return Array( + self._sync( + self._async_group.full(name=name, shape=shape, fill_value=fill_value, **kwargs) + ) + ) + + @_deprecate_positional_args + def empty_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array: + """Create an empty sub-array like `data`. The contents will be filled + with the array's fill value or zeros if no fill value is provided. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create an empty array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + + Notes + ----- + The contents of an empty Zarr array are not defined. On attempting to + retrieve data from an empty Zarr array, any values may be returned, + and these are not guaranteed to be stable from one access to the next. + """ + return Array(self._sync(self._async_group.empty_like(name=name, data=data, **kwargs))) + + @_deprecate_positional_args + def zeros_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array: + """Create a sub-array of zeros like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + + return Array(self._sync(self._async_group.zeros_like(name=name, data=data, **kwargs))) + + @_deprecate_positional_args + def ones_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array: + """Create a sub-array of ones like `data`. + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.ones_like(name=name, data=data, **kwargs))) + + @_deprecate_positional_args + def full_like(self, *, name: str, data: async_api.ArrayLike, **kwargs: Any) -> Array: + """Create a sub-array like `data` filled with the `fill_value` of `data` . + + Parameters + ---------- + name : str + Name of the array. + data : array-like + The array to create the new array like. + **kwargs + Keyword arguments passed to :func:`zarr.api.asynchronous.create`. + + Returns + ------- + Array + The new array. + """ + return Array(self._sync(self._async_group.full_like(name=name, data=data, **kwargs))) + + def move(self, source: str, dest: str) -> None: + """Move a sub-group or sub-array from one path to another. + + Notes + ----- + Not implemented + """ + return self._sync(self._async_group.move(source, dest)) + + @deprecated("Use Group.create_array instead.") + @_deprecate_positional_args + def array( + self, + name: str, + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ChunkCoords | Literal["auto"] | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + compressor: CompressorLike = None, + serializer: SerializerLike = "auto", + fill_value: Any | None = 0, + order: MemoryOrder | None = "C", + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, + storage_options: dict[str, Any] | None = None, + overwrite: bool = False, + config: ArrayConfig | ArrayConfigLike | None = None, + data: npt.ArrayLike | None = None, + ) -> Array: + """Create an array within this group. + + .. deprecated:: 3.0.0 + Use `Group.create_array` instead. + + This method lightly wraps :func:`zarr.core.array.create_array`. + + Parameters + ---------- + name : str + The name of the array relative to the group. If ``path`` is ``None``, the array will be located + at the root of the store. + shape : ChunkCoords + Shape of the array. + dtype : npt.DTypeLike + Data type of the array. + chunks : ChunkCoords, optional + Chunk shape of the array. + If not specified, default are guessed based on the shape and dtype. + shards : ChunkCoords, optional + Shard shape of the array. The default value of ``None`` results in no sharding at all. + filters : Iterable[Codec], optional + Iterable of filters to apply to each chunk of the array, in order, before serializing that + chunk to bytes. + + For Zarr format 3, a "filter" is a codec that takes an array and returns an array, + and these values must be instances of ``ArrayArrayCodec``, or dict representations + of ``ArrayArrayCodec``. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v3_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + + For Zarr format 2, a "filter" can be any numcodecs codec; you should ensure that the + the order if your filters is consistent with the behavior of each filter. + If no ``filters`` are provided, a default set of filters will be used. + These defaults can be changed by modifying the value of ``array.v2_default_filters`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default filters. + compressors : Iterable[Codec], optional + List of compressors to apply to the array. Compressors are applied in order, and after any + filters are applied (if any are specified) and the data is serialized into bytes. + + For Zarr format 3, a "compressor" is a codec that takes a bytestream, and + returns another bytestream. Multiple compressors my be provided for Zarr format 3. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` + in :mod:`zarr.core.config`. + Use ``None`` to omit default compressors. + + For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may + be provided for Zarr format 2. + If no ``compressor`` is provided, a default compressor will be used. + in :mod:`zarr.core.config`. + Use ``None`` to omit the default compressor. + compressor : Codec, optional + Deprecated in favor of ``compressors``. + serializer : dict[str, JSON] | ArrayBytesCodec, optional + Array-to-bytes codec to use for encoding the array data. + Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` + in :mod:`zarr.core.config`. + fill_value : Any, optional + Fill value for the array. + order : {"C", "F"}, optional + The memory of the array (default is "C"). + For Zarr format 2, this parameter sets the memory order of the array. + For Zarr format 3, this parameter is deprecated, because memory order + is a runtime parameter for Zarr format 3 arrays. The recommended way to specify the memory + order for Zarr format 3 arrays is via the ``config`` parameter, e.g. ``{'config': 'C'}``. + If no ``order`` is provided, a default order will be used. + This default can be changed by modifying the value of ``array.order`` in :mod:`zarr.core.config`. + attributes : dict, optional + Attributes for the array. + chunk_key_encoding : ChunkKeyEncoding, optional + A specification of how the chunk keys are represented in storage. + For Zarr format 3, the default is ``{"name": "default", "separator": "/"}}``. + For Zarr format 2, the default is ``{"name": "v2", "separator": "."}}``. + dimension_names : Iterable[str], optional + The names of the dimensions (default is None). + Zarr format 3 only. Zarr format 2 arrays should not use this parameter. + storage_options : dict, optional + If using an fsspec URL to create the store, these will be passed to the backend implementation. + Ignored otherwise. + overwrite : bool, default False + Whether to overwrite an array with the same name in the store, if one exists. + config : ArrayConfig or ArrayConfigLike, optional + Runtime configuration for the array. + data : array_like + The data to fill the array with. + + Returns + ------- + AsyncArray + """ + compressors = _parse_deprecated_compressor(compressor, compressors) + return Array( + self._sync( + self._async_group.create_dataset( + name=name, + shape=shape, + dtype=dtype, + chunks=chunks, + shards=shards, + fill_value=fill_value, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + compressors=compressors, + serializer=serializer, + dimension_names=dimension_names, + order=order, + filters=filters, + overwrite=overwrite, + storage_options=storage_options, + config=config, + data=data, + ) + ) + ) + + +async def create_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> AsyncIterator[ + tuple[str, AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]] +]: + """ + Create a complete zarr hierarchy from a collection of metadata objects. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the root of the ``Store``. The root of the store can be specified with the empty + string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, AsyncGroup | AsyncArray] + This function yields (path, node) pairs, in the order the nodes were created. + + Examples + -------- + >>> from zarr.api.asynchronous import create_hierarchy + >>> from zarr.storage import MemoryStore + >>> from zarr.core.group import GroupMetadata + >>> import asyncio + >>> store = MemoryStore() + >>> nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} + >>> async def run(): + ... print(dict([x async for x in create_hierarchy(store=store, nodes=nodes)])) + >>> asyncio.run(run()) + # {'a': , '': } + """ + # normalize the keys to be valid paths + nodes_normed_keys = _normalize_path_keys(nodes) + + # ensure that all nodes have the same zarr_format, and add implicit groups as needed + nodes_parsed = _parse_hierarchy_dict(data=nodes_normed_keys) + redundant_implicit_groups = [] + + # empty hierarchies should be a no-op + if len(nodes_parsed) > 0: + # figure out which zarr format we are using + zarr_format = next(iter(nodes_parsed.values())).zarr_format + + # check which implicit groups will require materialization + implicit_group_keys = tuple( + filter(lambda k: isinstance(nodes_parsed[k], ImplicitGroupMarker), nodes_parsed) + ) + # read potential group metadata for each implicit group + maybe_extant_group_coros = ( + _read_group_metadata(store, k, zarr_format=zarr_format) for k in implicit_group_keys + ) + maybe_extant_groups = await asyncio.gather( + *maybe_extant_group_coros, return_exceptions=True + ) + + for key, value in zip(implicit_group_keys, maybe_extant_groups, strict=True): + if isinstance(value, BaseException): + if isinstance(value, FileNotFoundError): + # this is fine -- there was no group there, so we will create one + pass + else: + raise value + else: + # a loop exists already at ``key``, so we can avoid creating anything there + redundant_implicit_groups.append(key) + + if overwrite: + # we will remove any nodes that collide with arrays and non-implicit groups defined in + # nodes + + # track the keys of nodes we need to delete + to_delete_keys = [] + to_delete_keys.extend( + [k for k, v in nodes_parsed.items() if k not in implicit_group_keys] + ) + await asyncio.gather(*(store.delete_dir(key) for key in to_delete_keys)) + else: + # This type is long. + coros: ( + Generator[Coroutine[Any, Any, ArrayV2Metadata | GroupMetadata], None, None] + | Generator[Coroutine[Any, Any, ArrayV3Metadata | GroupMetadata], None, None] + ) + if zarr_format == 2: + coros = (_read_metadata_v2(store=store, path=key) for key in nodes_parsed) + elif zarr_format == 3: + coros = (_read_metadata_v3(store=store, path=key) for key in nodes_parsed) + else: # pragma: no cover + raise ValueError(f"Invalid zarr_format: {zarr_format}") # pragma: no cover + + extant_node_query = dict( + zip( + nodes_parsed.keys(), + await asyncio.gather(*coros, return_exceptions=True), + strict=False, + ) + ) + # iterate over the existing arrays / groups and figure out which of them conflict + # with the arrays / groups we want to create + for key, extant_node in extant_node_query.items(): + proposed_node = nodes_parsed[key] + if isinstance(extant_node, BaseException): + if isinstance(extant_node, FileNotFoundError): + # ignore FileNotFoundError, because they represent nodes we can safely create + pass + else: + # Any other exception is a real error + raise extant_node + else: + # this is a node that already exists, but a node with the same key was specified + # in nodes_parsed. + if isinstance(extant_node, GroupMetadata): + # a group already exists where we want to create a group + if isinstance(proposed_node, ImplicitGroupMarker): + # we have proposed an implicit group, which is OK -- we will just skip + # creating this particular metadata document + redundant_implicit_groups.append(key) + else: + # we have proposed an explicit group, which is an error, given that a + # group already exists. + raise ContainsGroupError(store, key) + elif isinstance(extant_node, ArrayV2Metadata | ArrayV3Metadata): + # we are trying to overwrite an existing array. this is an error. + raise ContainsArrayError(store, key) + + nodes_explicit: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {} + + for k, v in nodes_parsed.items(): + if k not in redundant_implicit_groups: + if isinstance(v, ImplicitGroupMarker): + nodes_explicit[k] = GroupMetadata(zarr_format=v.zarr_format) + else: + nodes_explicit[k] = v + + async for key, node in create_nodes(store=store, nodes=nodes_explicit): + yield key, node + + +async def create_nodes( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> AsyncIterator[ + tuple[str, AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]] +]: + """Create a collection of arrays and / or groups concurrently. + + Note: no attempt is made to validate that these arrays and / or groups collectively form a + valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that + the ``nodes`` parameter satisfies any correctness constraints. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes + in the hierarchy, and the values are the metadata of the nodes. The + metadata must be either an instance of GroupMetadata, ArrayV3Metadata + or ArrayV2Metadata. + + Yields + ------ + AsyncGroup | AsyncArray + The created nodes in the order they are created. + """ + + # Note: the only way to alter this value is via the config. If that's undesirable for some reason, + # then we should consider adding a keyword argument this this function + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + create_tasks: list[Coroutine[None, None, str]] = [] + + for key, value in nodes.items(): + # make the key absolute + create_tasks.extend(_persist_metadata(store, key, value, semaphore=semaphore)) + + created_object_keys = [] + + for coro in asyncio.as_completed(create_tasks): + created_key = await coro + # we need this to track which metadata documents were written so that we can yield a + # complete v2 Array / Group class after both .zattrs and the metadata JSON was created. + created_object_keys.append(created_key) + + # get the node name from the object key + if len(created_key.split("/")) == 1: + # this is the root node + meta_out = nodes[""] + node_name = "" + else: + # turn "foo/" into "foo" + node_name = created_key[: created_key.rfind("/")] + meta_out = nodes[node_name] + if meta_out.zarr_format == 3: + yield node_name, _build_node(store=store, path=node_name, metadata=meta_out) + else: + # For zarr v2 + # we only want to yield when both the metadata and attributes are created + # so we track which keys have been created, and wait for both the meta key and + # the attrs key to be created before yielding back the AsyncArray / AsyncGroup + + attrs_done = _join_paths([node_name, ZATTRS_JSON]) in created_object_keys + + if isinstance(meta_out, GroupMetadata): + meta_done = _join_paths([node_name, ZGROUP_JSON]) in created_object_keys + else: + meta_done = _join_paths([node_name, ZARRAY_JSON]) in created_object_keys + + if meta_done and attrs_done: + yield node_name, _build_node(store=store, path=node_name, metadata=meta_out) + + continue + + +def _get_roots( + data: Iterable[str], +) -> tuple[str, ...]: + """ + Return the keys of the root(s) of the hierarchy. A root is a key with the fewest number of + path segments. + """ + if "" in data: + return ("",) + keys_split = sorted((key.split("/") for key in data), key=len) + groups: defaultdict[int, list[str]] = defaultdict(list) + for key_split in keys_split: + groups[len(key_split)].append("/".join(key_split)) + return tuple(groups[min(groups.keys())]) + + +def _parse_hierarchy_dict( + *, + data: Mapping[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> dict[str, ImplicitGroupMarker | GroupMetadata | ArrayV2Metadata | ArrayV3Metadata]: + """ + Take an input with type Mapping[str, ArrayMetadata | GroupMetadata] and parse it into + a dict of str: node pairs that models a valid, complete Zarr hierarchy. + + If the input represents a complete Zarr hierarchy, i.e. one with no implicit groups, + then return a dict with the exact same data as the input. + + Otherwise, return a dict derived from the input with GroupMetadata inserted as needed to make + the hierarchy complete. + + For example, an input of {'a/b': ArrayMetadata} is incomplete, because it references two + groups (the root group '' and a group at 'a') that are not specified in the input. Applying this function + to that input will result in a return value of + {'': GroupMetadata, 'a': GroupMetadata, 'a/b': ArrayMetadata}, i.e. the implied groups + were added. + + The input is also checked for the following conditions; an error is raised if any are violated: + + - No arrays can contain group or arrays (i.e., all arrays must be leaf nodes). + - All arrays and groups must have the same ``zarr_format`` value. + + This function ensures that the input is transformed into a specification of a complete and valid + Zarr hierarchy. + """ + + # ensure that all nodes have the same zarr format + data_purified = _ensure_consistent_zarr_format(data) + + # ensure that keys are normalized to zarr paths + data_normed_keys = _normalize_path_keys(data_purified) + + # insert an implicit root group if a root was not specified + # but not if an empty dict was provided, because any empty hierarchy has no nodes + if len(data_normed_keys) > 0 and "" not in data_normed_keys: + z_format = next(iter(data_normed_keys.values())).zarr_format + data_normed_keys = data_normed_keys | {"": ImplicitGroupMarker(zarr_format=z_format)} + + out: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] = {**data_normed_keys} + + for k, v in data_normed_keys.items(): + key_split = k.split("/") + + # get every parent path + *subpaths, _ = accumulate(key_split, lambda a, b: _join_paths([a, b])) + + for subpath in subpaths: + # If a component is not already in the output dict, add ImplicitGroupMetadata + if subpath not in out: + out[subpath] = ImplicitGroupMarker(zarr_format=v.zarr_format) + else: + if not isinstance(out[subpath], GroupMetadata | ImplicitGroupMarker): + msg = ( + f"The node at {subpath} contains other nodes, but it is not a Zarr group. " + "This is invalid. Only Zarr groups can contain other nodes." + ) + raise ValueError(msg) + return out + + +def _ensure_consistent_zarr_format( + data: Mapping[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], +) -> Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]: + """ + Ensure that all values of the input dict have the same zarr format. If any do not, + then a value error is raised. + """ + observed_zarr_formats: dict[ZarrFormat, list[str]] = {2: [], 3: []} + + for k, v in data.items(): + observed_zarr_formats[v.zarr_format].append(k) + + if len(observed_zarr_formats[2]) > 0 and len(observed_zarr_formats[3]) > 0: + msg = ( + "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " + f"The following keys map to Zarr v2 nodes: {observed_zarr_formats.get(2)}. " + f"The following keys map to Zarr v3 nodes: {observed_zarr_formats.get(3)}." + "Ensure that all nodes have the same Zarr format." + ) + raise ValueError(msg) + + return cast( + "Mapping[str, GroupMetadata | ArrayV2Metadata] | Mapping[str, GroupMetadata | ArrayV3Metadata]", + data, + ) + + +async def _getitem_semaphore( + node: AsyncGroup, key: str, semaphore: asyncio.Semaphore | None +) -> AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup: + """ + Wrap Group.getitem with an optional semaphore. + + If the semaphore parameter is an + asyncio.Semaphore instance, then the getitem operation is performed inside an async context + manager provided by that semaphore. If the semaphore parameter is None, then getitem is invoked + without a context manager. + """ + if semaphore is not None: + async with semaphore: + return await node.getitem(key) + else: + return await node.getitem(key) + + +async def _iter_members( + node: AsyncGroup, + skip_keys: tuple[str, ...], + semaphore: asyncio.Semaphore | None, +) -> AsyncGenerator[ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup], None +]: + """ + Iterate over the arrays and groups contained in a group. + + Parameters + ---------- + node : AsyncGroup + The group to traverse. + skip_keys : tuple[str, ...] + A tuple of keys to skip when iterating over the possible members of the group. + semaphore : asyncio.Semaphore | None + An optional semaphore to use for concurrency control. + + Yields + ------ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup] + """ + + # retrieve keys from storage + keys = [key async for key in node.store.list_dir(node.path)] + keys_filtered = tuple(filter(lambda v: v not in skip_keys, keys)) + + node_tasks = tuple( + asyncio.create_task(_getitem_semaphore(node, key, semaphore), name=key) + for key in keys_filtered + ) + + for fetched_node_coro in asyncio.as_completed(node_tasks): + try: + fetched_node = await fetched_node_coro + except KeyError as e: + # keyerror is raised when `key` names an object (in the object storage sense), + # as opposed to a prefix, in the store under the prefix associated with this group + # in which case `key` cannot be the name of a sub-array or sub-group. + warnings.warn( + f"Object at {e.args[0]} is not recognized as a component of a Zarr hierarchy.", + UserWarning, + stacklevel=1, + ) + continue + match fetched_node: + case AsyncArray() | AsyncGroup(): + yield fetched_node.basename, fetched_node + case _: + raise ValueError(f"Unexpected type: {type(fetched_node)}") + + +async def _iter_members_deep( + group: AsyncGroup, + *, + max_depth: int | None, + skip_keys: tuple[str, ...], + semaphore: asyncio.Semaphore | None = None, + use_consolidated_for_children: bool = True, +) -> AsyncGenerator[ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup], None +]: + """ + Iterate over the arrays and groups contained in a group, and optionally the + arrays and groups contained in those groups. + + Parameters + ---------- + group : AsyncGroup + The group to traverse. + max_depth : int | None + The maximum depth of recursion. + skip_keys : tuple[str, ...] + A tuple of keys to skip when iterating over the possible members of the group. + semaphore : asyncio.Semaphore | None + An optional semaphore to use for concurrency control. + use_consolidated_for_children : bool, default True + Whether to use the consolidated metadata of child groups loaded + from the store. Note that this only affects groups loaded from the + store. If the current Group already has consolidated metadata, it + will always be used. + + Yields + ------ + tuple[str, AsyncArray[ArrayV3Metadata] | AsyncArray[ArrayV2Metadata] | AsyncGroup] + """ + + to_recurse = {} + do_recursion = max_depth is None or max_depth > 0 + + if max_depth is None: + new_depth = None + else: + new_depth = max_depth - 1 + async for name, node in _iter_members(group, skip_keys=skip_keys, semaphore=semaphore): + is_group = isinstance(node, AsyncGroup) + if ( + is_group + and not use_consolidated_for_children + and node.metadata.consolidated_metadata is not None # type: ignore [union-attr] + ): + node = cast("AsyncGroup", node) + # We've decided not to trust consolidated metadata at this point, because we're + # reconsolidating the metadata, for example. + node = replace(node, metadata=replace(node.metadata, consolidated_metadata=None)) + yield name, node + if is_group and do_recursion: + node = cast("AsyncGroup", node) + to_recurse[name] = _iter_members_deep( + node, max_depth=new_depth, skip_keys=skip_keys, semaphore=semaphore + ) + + for prefix, subgroup_iter in to_recurse.items(): + async for name, node in subgroup_iter: + key = f"{prefix}/{name}".lstrip("/") + yield key, node + + +async def _read_metadata_v3(store: Store, path: str) -> ArrayV3Metadata | GroupMetadata: + """ + Given a store_path, return ArrayV3Metadata or GroupMetadata defined by the metadata + document stored at store_path.path / zarr.json. If no such document is found, raise a + FileNotFoundError. + """ + zarr_json_bytes = await store.get( + _join_paths([path, ZARR_JSON]), prototype=default_buffer_prototype() + ) + if zarr_json_bytes is None: + raise FileNotFoundError(path) + else: + zarr_json = json.loads(zarr_json_bytes.to_bytes()) + return _build_metadata_v3(zarr_json) + + +async def _read_metadata_v2(store: Store, path: str) -> ArrayV2Metadata | GroupMetadata: + """ + Given a store_path, return ArrayV2Metadata or GroupMetadata defined by the metadata + document stored at store_path.path / (.zgroup | .zarray). If no such document is found, + raise a FileNotFoundError. + """ + # TODO: consider first fetching array metadata, and only fetching group metadata when we don't + # find an array + zarray_bytes, zgroup_bytes, zattrs_bytes = await asyncio.gather( + store.get(_join_paths([path, ZARRAY_JSON]), prototype=default_buffer_prototype()), + store.get(_join_paths([path, ZGROUP_JSON]), prototype=default_buffer_prototype()), + store.get(_join_paths([path, ZATTRS_JSON]), prototype=default_buffer_prototype()), + ) + + if zattrs_bytes is None: + zattrs = {} + else: + zattrs = json.loads(zattrs_bytes.to_bytes()) + + # TODO: decide how to handle finding both array and group metadata. The spec does not seem to + # consider this situation. A practical approach would be to ignore that combination, and only + # return the array metadata. + if zarray_bytes is not None: + zmeta = json.loads(zarray_bytes.to_bytes()) + else: + if zgroup_bytes is None: + # neither .zarray or .zgroup were found results in KeyError + raise FileNotFoundError(path) + else: + zmeta = json.loads(zgroup_bytes.to_bytes()) + + return _build_metadata_v2(zmeta, zattrs) + + +async def _read_group_metadata_v2(store: Store, path: str) -> GroupMetadata: + """ + Read group metadata or error + """ + meta = await _read_metadata_v2(store=store, path=path) + if not isinstance(meta, GroupMetadata): + raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") + return meta + + +async def _read_group_metadata_v3(store: Store, path: str) -> GroupMetadata: + """ + Read group metadata or error + """ + meta = await _read_metadata_v3(store=store, path=path) + if not isinstance(meta, GroupMetadata): + raise FileNotFoundError(f"Group metadata was not found in {store} at {path}") + return meta + + +async def _read_group_metadata( + store: Store, path: str, *, zarr_format: ZarrFormat +) -> GroupMetadata: + if zarr_format == 2: + return await _read_group_metadata_v2(store=store, path=path) + return await _read_group_metadata_v3(store=store, path=path) + + +def _build_metadata_v3(zarr_json: dict[str, JSON]) -> ArrayV3Metadata | GroupMetadata: + """ + Convert a dict representation of Zarr V3 metadata into the corresponding metadata class. + """ + if "node_type" not in zarr_json: + raise MetadataValidationError("node_type", "array or group", "nothing (the key is missing)") + match zarr_json: + case {"node_type": "array"}: + return ArrayV3Metadata.from_dict(zarr_json) + case {"node_type": "group"}: + return GroupMetadata.from_dict(zarr_json) + case _: # pragma: no cover + raise ValueError( + "invalid value for `node_type` key in metadata document" + ) # pragma: no cover + + +def _build_metadata_v2( + zarr_json: dict[str, JSON], attrs_json: dict[str, JSON] +) -> ArrayV2Metadata | GroupMetadata: + """ + Convert a dict representation of Zarr V2 metadata into the corresponding metadata class. + """ + match zarr_json: + case {"shape": _}: + return ArrayV2Metadata.from_dict(zarr_json | {"attributes": attrs_json}) + case _: # pragma: no cover + return GroupMetadata.from_dict(zarr_json | {"attributes": attrs_json}) + + +@overload +def _build_node( + *, store: Store, path: str, metadata: ArrayV2Metadata +) -> AsyncArray[ArrayV2Metadata]: ... + + +@overload +def _build_node( + *, store: Store, path: str, metadata: ArrayV3Metadata +) -> AsyncArray[ArrayV3Metadata]: ... + + +@overload +def _build_node(*, store: Store, path: str, metadata: GroupMetadata) -> AsyncGroup: ... + + +def _build_node( + *, store: Store, path: str, metadata: ArrayV3Metadata | ArrayV2Metadata | GroupMetadata +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup: + """ + Take a metadata object and return a node (AsyncArray or AsyncGroup). + """ + store_path = StorePath(store=store, path=path) + match metadata: + case ArrayV2Metadata() | ArrayV3Metadata(): + return AsyncArray(metadata, store_path=store_path) + case GroupMetadata(): + return AsyncGroup(metadata, store_path=store_path) + case _: # pragma: no cover + raise ValueError(f"Unexpected metadata type: {type(metadata)}") # pragma: no cover + + +async def _get_node_v2(store: Store, path: str) -> AsyncArray[ArrayV2Metadata] | AsyncGroup: + """ + Read a Zarr v2 AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + metadata = await _read_metadata_v2(store=store, path=path) + return _build_node(store=store, path=path, metadata=metadata) + + +async def _get_node_v3(store: Store, path: str) -> AsyncArray[ArrayV3Metadata] | AsyncGroup: + """ + Read a Zarr v3 AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + metadata = await _read_metadata_v3(store=store, path=path) + return _build_node(store=store, path=path, metadata=metadata) + + +async def get_node( + store: Store, path: str, zarr_format: ZarrFormat +) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | AsyncGroup: + """ + Get an AsyncArray or AsyncGroup from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + zarr_format : {2, 3} + The zarr format of the node to read. + + Returns + ------- + AsyncArray | AsyncGroup + """ + + match zarr_format: + case 2: + return await _get_node_v2(store=store, path=path) + case 3: + return await _get_node_v3(store=store, path=path) + case _: # pragma: no cover + raise ValueError(f"Unexpected zarr format: {zarr_format}") # pragma: no cover + + +async def _set_return_key( + *, store: Store, key: str, value: Buffer, semaphore: asyncio.Semaphore | None = None +) -> str: + """ + Write a value to storage at the given key. The key is returned. + Useful when saving values via routines that return results in execution order, + like asyncio.as_completed, because in this case we need to know which key was saved in order + to yield the right object to the caller. + + Parameters + ---------- + store : Store + The store to save the value to. + key : str + The key to save the value to. + value : Buffer + The value to save. + semaphore : asyncio.Semaphore | None + An optional semaphore to use to limit the number of concurrent writes. + """ + + if semaphore is not None: + async with semaphore: + await store.set(key, value) + else: + await store.set(key, value) + return key + + +def _persist_metadata( + store: Store, + path: str, + metadata: ArrayV2Metadata | ArrayV3Metadata | GroupMetadata, + semaphore: asyncio.Semaphore | None = None, +) -> tuple[Coroutine[None, None, str], ...]: + """ + Prepare to save a metadata document to storage, returning a tuple of coroutines that must be awaited. + """ + + to_save = metadata.to_buffer_dict(default_buffer_prototype()) + return tuple( + _set_return_key(store=store, key=_join_paths([path, key]), value=value, semaphore=semaphore) + for key, value in to_save.items() + ) + + +async def create_rooted_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> AsyncGroup | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: + """ + Create an ``AsyncGroup`` or ``AsyncArray`` from a store and a dict of metadata documents. + This function ensures that its input contains a specification of a root node, + calls ``create_hierarchy`` to create nodes, and returns the root node of the hierarchy. + """ + roots = _get_roots(nodes.keys()) + if len(roots) != 1: + msg = ( + "The input does not specify a root node. " + "This function can only create hierarchies that contain a root node, which is " + "defined as a group that is ancestral to all the other arrays and " + "groups in the hierarchy, or a single array." + ) + raise ValueError(msg) + else: + root_key = roots[0] + + nodes_created = [ + x async for x in create_hierarchy(store=store, nodes=nodes, overwrite=overwrite) + ] + return dict(nodes_created)[root_key] diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py new file mode 100644 index 0000000000..c11889f7f4 --- /dev/null +++ b/src/zarr/core/indexing.py @@ -0,0 +1,1391 @@ +from __future__ import annotations + +import itertools +import math +import numbers +import operator +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from enum import Enum +from functools import reduce +from types import EllipsisType +from typing import ( + TYPE_CHECKING, + Any, + Literal, + NamedTuple, + Protocol, + TypeAlias, + TypeGuard, + TypeVar, + cast, + runtime_checkable, +) + +import numpy as np +import numpy.typing as npt + +from zarr.core.common import product + +if TYPE_CHECKING: + from zarr.core.array import Array + from zarr.core.buffer import NDArrayLikeOrScalar + from zarr.core.chunk_grids import ChunkGrid + from zarr.core.common import ChunkCoords + +IntSequence = list[int] | npt.NDArray[np.intp] +ArrayOfIntOrBool = npt.NDArray[np.intp] | npt.NDArray[np.bool_] +BasicSelector = int | slice | EllipsisType +Selector = BasicSelector | ArrayOfIntOrBool +BasicSelection = BasicSelector | tuple[BasicSelector, ...] # also used for BlockIndex +CoordinateSelection = IntSequence | tuple[IntSequence, ...] +MaskSelection = npt.NDArray[np.bool_] +OrthogonalSelection = Selector | tuple[Selector, ...] +Selection = BasicSelection | CoordinateSelection | MaskSelection | OrthogonalSelection +CoordinateSelectionNormalized = tuple[npt.NDArray[np.intp], ...] +SelectionNormalized = tuple[Selector, ...] | ArrayOfIntOrBool +SelectionWithFields = Selection | str | Sequence[str] +SelectorTuple = tuple[Selector, ...] | npt.NDArray[np.intp] | slice +Fields = str | list[str] | tuple[str, ...] + + +class ArrayIndexError(IndexError): + pass + + +class BoundsCheckError(IndexError): + _msg = "" + + def __init__(self, dim_len: int) -> None: + self._msg = f"index out of bounds for dimension with length {dim_len}" + + +class NegativeStepError(IndexError): + _msg = "only slices with step >= 1 are supported" + + +class VindexInvalidSelectionError(IndexError): + _msg = ( + "unsupported selection type for vectorized indexing; only " + "coordinate selection (tuple of integer arrays) and mask selection " + "(single Boolean array) are supported; got {!r}" + ) + + +def err_too_many_indices(selection: Any, shape: ChunkCoords) -> None: + raise IndexError(f"too many indices for array; expected {len(shape)}, got {len(selection)}") + + +def _zarr_array_to_int_or_bool_array(arr: Array) -> npt.NDArray[np.intp] | npt.NDArray[np.bool_]: + if arr.dtype.kind in ("i", "b"): + return np.asarray(arr) + else: + raise IndexError( + f"Invalid array dtype: {arr.dtype}. Arrays used as indices must be of integer or boolean type" + ) + + +@runtime_checkable +class Indexer(Protocol): + shape: ChunkCoords + drop_axes: ChunkCoords + + def __iter__(self) -> Iterator[ChunkProjection]: ... + + +def ceildiv(a: float, b: float) -> int: + if a == 0: + return 0 + return math.ceil(a / b) + + +_ArrayIndexingOrder: TypeAlias = Literal["lexicographic"] + + +def _iter_grid( + grid_shape: Sequence[int], + *, + origin: Sequence[int] | None = None, + selection_shape: Sequence[int] | None = None, + order: _ArrayIndexingOrder = "lexicographic", +) -> Iterator[ChunkCoords]: + """ + Iterate over the elements of grid of integers, with the option to restrict the domain of + iteration to a contiguous subregion of that grid. + + Parameters + ---------- + grid_shape : Sequence[int] + The size of the domain to iterate over. + origin : Sequence[int] | None, default=None + The first coordinate of the domain to return. + selection_shape : Sequence[int] | None, default=None + The shape of the selection. + order : Literal["lexicographic"], default="lexicographic" + The linear indexing order to use. + + Returns + ------- + + itertools.product object + An iterator over tuples of integers + + Examples + -------- + >>> tuple(iter_grid((1,))) + ((0,),) + + >>> tuple(iter_grid((2,3))) + ((0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2)) + + >>> tuple(iter_grid((2,3)), origin=(1,1)) + ((1, 1), (1, 2), (1, 3), (2, 1), (2, 2), (2, 3)) + + >>> tuple(iter_grid((2,3)), origin=(1,1), selection_shape=(2,2)) + ((1, 1), (1, 2), (1, 3), (2, 1)) + """ + if origin is None: + origin_parsed = (0,) * len(grid_shape) + else: + if len(origin) != len(grid_shape): + msg = ( + "Shape and origin parameters must have the same length." + f"Got {len(grid_shape)} elements in shape, but {len(origin)} elements in origin." + ) + raise ValueError(msg) + origin_parsed = tuple(origin) + if selection_shape is None: + selection_shape_parsed = tuple( + g - o for o, g in zip(origin_parsed, grid_shape, strict=True) + ) + else: + selection_shape_parsed = tuple(selection_shape) + if order == "lexicographic": + dimensions: tuple[range, ...] = () + for idx, (o, gs, ss) in enumerate( + zip(origin_parsed, grid_shape, selection_shape_parsed, strict=True) + ): + if o + ss > gs: + raise IndexError( + f"Invalid selection shape ({selection_shape}) for origin ({origin}) and grid shape ({grid_shape}) at axis {idx}." + ) + dimensions += (range(o, o + ss),) + yield from itertools.product(*(dimensions)) + + else: + msg = f"Indexing order {order} is not supported at this time." # type: ignore[unreachable] + raise NotImplementedError(msg) + + +def is_integer(x: Any) -> TypeGuard[int]: + """True if x is an integer (both pure Python or NumPy).""" + return isinstance(x, numbers.Integral) and not is_bool(x) + + +def is_bool(x: Any) -> TypeGuard[bool | np.bool_]: + """True if x is a boolean (both pure Python or NumPy).""" + return type(x) in [bool, np.bool_] + + +def is_integer_list(x: Any) -> TypeGuard[list[int]]: + """True if x is a list of integers.""" + return isinstance(x, list) and len(x) > 0 and all(is_integer(i) for i in x) + + +def is_bool_list(x: Any) -> TypeGuard[list[bool | np.bool_]]: + """True if x is a list of boolean.""" + return isinstance(x, list) and len(x) > 0 and all(is_bool(i) for i in x) + + +def is_integer_array(x: Any, ndim: int | None = None) -> TypeGuard[npt.NDArray[np.intp]]: + t = not np.isscalar(x) and hasattr(x, "shape") and hasattr(x, "dtype") and x.dtype.kind in "ui" + if ndim is not None: + t = t and hasattr(x, "shape") and len(x.shape) == ndim + return t + + +def is_bool_array(x: Any, ndim: int | None = None) -> TypeGuard[npt.NDArray[np.bool_]]: + t = hasattr(x, "shape") and hasattr(x, "dtype") and x.dtype == bool + if ndim is not None: + t = t and hasattr(x, "shape") and len(x.shape) == ndim + return t + + +def is_int_or_bool_iterable(x: Any) -> bool: + return is_integer_list(x) or is_integer_array(x) or is_bool_array(x) or is_bool_list(x) + + +def is_scalar(value: Any, dtype: np.dtype[Any]) -> bool: + if np.isscalar(value): + return True + if hasattr(value, "shape") and value.shape == (): + return True + return isinstance(value, tuple) and dtype.names is not None and len(value) == len(dtype.names) + + +def is_pure_fancy_indexing(selection: Any, ndim: int) -> bool: + """Check whether a selection contains only scalars or integer/bool array-likes. + + Parameters + ---------- + selection : tuple, slice, or scalar + A valid selection value for indexing into arrays. + + Returns + ------- + is_pure : bool + True if the selection is a pure fancy indexing expression (ie not mixed + with boolean or slices). + """ + if is_bool_array(selection): + # is mask selection + return True + + if ndim == 1 and ( + is_integer_list(selection) or is_integer_array(selection) or is_bool_list(selection) + ): + return True + + # if not, we go through the normal path below, because a 1-tuple + # of integers is also allowed. + no_slicing = ( + isinstance(selection, tuple) + and len(selection) == ndim + and not (any(isinstance(elem, slice) or elem is Ellipsis for elem in selection)) + ) + return ( + no_slicing + and all( + is_integer(elem) or is_integer_list(elem) or is_integer_array(elem) + for elem in selection + ) + and any(is_integer_list(elem) or is_integer_array(elem) for elem in selection) + ) + + +def is_pure_orthogonal_indexing(selection: Selection, ndim: int) -> TypeGuard[OrthogonalSelection]: + if not ndim: + return False + + selection_normalized = (selection,) if not isinstance(selection, tuple) else selection + + # Case 1: Selection contains of iterable of integers or boolean + if len(selection_normalized) == ndim and all( + is_int_or_bool_iterable(s) for s in selection_normalized + ): + return True + + # Case 2: selection contains either zero or one integer iterables. + # All other selection elements are slices or integers + return ( + len(selection_normalized) <= ndim + and sum(is_int_or_bool_iterable(s) for s in selection_normalized) <= 1 + and all( + is_int_or_bool_iterable(s) or isinstance(s, int | slice) for s in selection_normalized + ) + ) + + +def get_chunk_shape(chunk_grid: ChunkGrid) -> ChunkCoords: + from zarr.core.chunk_grids import RegularChunkGrid + + assert isinstance(chunk_grid, RegularChunkGrid), ( + "Only regular chunk grid is supported, currently." + ) + return chunk_grid.chunk_shape + + +def normalize_integer_selection(dim_sel: int, dim_len: int) -> int: + # normalize type to int + dim_sel = int(dim_sel) + + # handle wraparound + if dim_sel < 0: + dim_sel = dim_len + dim_sel + + # handle out of bounds + if dim_sel >= dim_len or dim_sel < 0: + raise BoundsCheckError(dim_len) + + return dim_sel + + +class ChunkDimProjection(NamedTuple): + """A mapping from chunk to output array for a single dimension. + + Attributes + ---------- + dim_chunk_ix + Index of chunk. + dim_chunk_sel + Selection of items from chunk array. + dim_out_sel + Selection of items in target (output) array. + """ + + dim_chunk_ix: int + dim_chunk_sel: Selector + dim_out_sel: Selector | None + is_complete_chunk: bool + + +@dataclass(frozen=True) +class IntDimIndexer: + dim_sel: int + dim_len: int + dim_chunk_len: int + nitems: int = 1 + + def __init__(self, dim_sel: int, dim_len: int, dim_chunk_len: int) -> None: + object.__setattr__(self, "dim_sel", normalize_integer_selection(dim_sel, dim_len)) + object.__setattr__(self, "dim_len", dim_len) + object.__setattr__(self, "dim_chunk_len", dim_chunk_len) + + def __iter__(self) -> Iterator[ChunkDimProjection]: + dim_chunk_ix = self.dim_sel // self.dim_chunk_len + dim_offset = dim_chunk_ix * self.dim_chunk_len + dim_chunk_sel = self.dim_sel - dim_offset + dim_out_sel = None + is_complete_chunk = self.dim_chunk_len == 1 + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) + + +@dataclass(frozen=True) +class SliceDimIndexer: + dim_len: int + dim_chunk_len: int + nitems: int + nchunks: int + + start: int + stop: int + step: int + + def __init__(self, dim_sel: slice, dim_len: int, dim_chunk_len: int) -> None: + # normalize + start, stop, step = dim_sel.indices(dim_len) + if step < 1: + raise NegativeStepError + + object.__setattr__(self, "start", start) + object.__setattr__(self, "stop", stop) + object.__setattr__(self, "step", step) + + object.__setattr__(self, "dim_len", dim_len) + object.__setattr__(self, "dim_chunk_len", dim_chunk_len) + object.__setattr__(self, "nitems", max(0, ceildiv((stop - start), step))) + object.__setattr__(self, "nchunks", ceildiv(dim_len, dim_chunk_len)) + + def __iter__(self) -> Iterator[ChunkDimProjection]: + # figure out the range of chunks we need to visit + dim_chunk_ix_from = 0 if self.start == 0 else self.start // self.dim_chunk_len + dim_chunk_ix_to = ceildiv(self.stop, self.dim_chunk_len) + + # iterate over chunks in range + for dim_chunk_ix in range(dim_chunk_ix_from, dim_chunk_ix_to): + # compute offsets for chunk within overall array + dim_offset = dim_chunk_ix * self.dim_chunk_len + dim_limit = min(self.dim_len, (dim_chunk_ix + 1) * self.dim_chunk_len) + + # determine chunk length, accounting for trailing chunk + dim_chunk_len = dim_limit - dim_offset + + if self.start < dim_offset: + # selection starts before current chunk + dim_chunk_sel_start = 0 + remainder = (dim_offset - self.start) % self.step + if remainder: + dim_chunk_sel_start += self.step - remainder + # compute number of previous items, provides offset into output array + dim_out_offset = ceildiv((dim_offset - self.start), self.step) + + else: + # selection starts within current chunk + dim_chunk_sel_start = self.start - dim_offset + dim_out_offset = 0 + + if self.stop > dim_limit: + # selection ends after current chunk + dim_chunk_sel_stop = dim_chunk_len + + else: + # selection ends within current chunk + dim_chunk_sel_stop = self.stop - dim_offset + + dim_chunk_sel = slice(dim_chunk_sel_start, dim_chunk_sel_stop, self.step) + dim_chunk_nitems = ceildiv((dim_chunk_sel_stop - dim_chunk_sel_start), self.step) + + # If there are no elements on the selection within this chunk, then skip + if dim_chunk_nitems == 0: + continue + + dim_out_sel = slice(dim_out_offset, dim_out_offset + dim_chunk_nitems) + + is_complete_chunk = ( + dim_chunk_sel_start == 0 and (self.stop >= dim_limit) and self.step in [1, None] + ) + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) + + +def check_selection_length(selection: SelectionNormalized, shape: ChunkCoords) -> None: + if len(selection) > len(shape): + err_too_many_indices(selection, shape) + + +def replace_ellipsis(selection: Any, shape: ChunkCoords) -> SelectionNormalized: + selection = ensure_tuple(selection) + + # count number of ellipsis present + n_ellipsis = sum(1 for i in selection if i is Ellipsis) + + if n_ellipsis > 1: + # more than 1 is an error + raise IndexError("an index can only have a single ellipsis ('...')") + + elif n_ellipsis == 1: + # locate the ellipsis, count how many items to left and right + n_items_l = selection.index(Ellipsis) # items to left of ellipsis + n_items_r = len(selection) - (n_items_l + 1) # items to right of ellipsis + n_items = len(selection) - 1 # all non-ellipsis items + + if n_items >= len(shape): + # ellipsis does nothing, just remove it + selection = tuple(i for i in selection if i != Ellipsis) + + else: + # replace ellipsis with as many slices are needed for number of dims + new_item = selection[:n_items_l] + ((slice(None),) * (len(shape) - n_items)) + if n_items_r: + new_item += selection[-n_items_r:] + selection = new_item + + # fill out selection if not completely specified + if len(selection) < len(shape): + selection += (slice(None),) * (len(shape) - len(selection)) + + # check selection not too long + check_selection_length(selection, shape) + + return cast("SelectionNormalized", selection) + + +def replace_lists(selection: SelectionNormalized) -> SelectionNormalized: + return tuple( + np.asarray(dim_sel) if isinstance(dim_sel, list) else dim_sel for dim_sel in selection + ) + + +T = TypeVar("T") + + +def ensure_tuple(v: Any) -> SelectionNormalized: + if not isinstance(v, tuple): + v = (v,) + return cast("SelectionNormalized", v) + + +class ChunkProjection(NamedTuple): + """A mapping of items from chunk to output array. Can be used to extract items from the + chunk array for loading into an output array. Can also be used to extract items from a + value array for setting/updating in a chunk array. + + Attributes + ---------- + chunk_coords + Indices of chunk. + chunk_selection + Selection of items from chunk array. + out_selection + Selection of items in target (output) array. + is_complete_chunk: + True if a complete chunk is indexed + """ + + chunk_coords: ChunkCoords + chunk_selection: tuple[Selector, ...] | npt.NDArray[np.intp] + out_selection: tuple[Selector, ...] | npt.NDArray[np.intp] | slice + is_complete_chunk: bool + + +def is_slice(s: Any) -> TypeGuard[slice]: + return isinstance(s, slice) + + +def is_contiguous_slice(s: Any) -> TypeGuard[slice]: + return is_slice(s) and (s.step is None or s.step == 1) + + +def is_positive_slice(s: Any) -> TypeGuard[slice]: + return is_slice(s) and (s.step is None or s.step >= 1) + + +def is_contiguous_selection(selection: Any) -> TypeGuard[slice]: + selection = ensure_tuple(selection) + return all((is_integer_array(s) or is_contiguous_slice(s) or s == Ellipsis) for s in selection) + + +def is_basic_selection(selection: Any) -> TypeGuard[BasicSelection]: + selection = ensure_tuple(selection) + return all(is_integer(s) or is_positive_slice(s) for s in selection) + + +@dataclass(frozen=True) +class BasicIndexer(Indexer): + dim_indexers: list[IntDimIndexer | SliceDimIndexer] + shape: ChunkCoords + drop_axes: ChunkCoords + + def __init__( + self, + selection: BasicSelection, + shape: ChunkCoords, + chunk_grid: ChunkGrid, + ) -> None: + chunk_shape = get_chunk_shape(chunk_grid) + # handle ellipsis + selection_normalized = replace_ellipsis(selection, shape) + + # setup per-dimension indexers + dim_indexers: list[IntDimIndexer | SliceDimIndexer] = [] + for dim_sel, dim_len, dim_chunk_len in zip( + selection_normalized, shape, chunk_shape, strict=True + ): + dim_indexer: IntDimIndexer | SliceDimIndexer + if is_integer(dim_sel): + dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_chunk_len) + + elif is_slice(dim_sel): + dim_indexer = SliceDimIndexer(dim_sel, dim_len, dim_chunk_len) + + else: + raise IndexError( + "unsupported selection item for basic indexing; " + f"expected integer or slice, got {type(dim_sel)!r}" + ) + + dim_indexers.append(dim_indexer) + + object.__setattr__(self, "dim_indexers", dim_indexers) + object.__setattr__( + self, + "shape", + tuple(s.nitems for s in self.dim_indexers if not isinstance(s, IntDimIndexer)), + ) + object.__setattr__(self, "drop_axes", ()) + + def __iter__(self) -> Iterator[ChunkProjection]: + for dim_projections in itertools.product(*self.dim_indexers): + chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) + chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) + out_selection = tuple( + p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None + ) + is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) + + +@dataclass(frozen=True) +class BoolArrayDimIndexer: + dim_sel: npt.NDArray[np.bool_] + dim_len: int + dim_chunk_len: int + nchunks: int + + chunk_nitems: npt.NDArray[Any] + chunk_nitems_cumsum: npt.NDArray[Any] + nitems: int + dim_chunk_ixs: npt.NDArray[np.intp] + + def __init__(self, dim_sel: npt.NDArray[np.bool_], dim_len: int, dim_chunk_len: int) -> None: + # check number of dimensions + if not is_bool_array(dim_sel, 1): + raise IndexError("Boolean arrays in an orthogonal selection must be 1-dimensional only") + + # check shape + if dim_sel.shape[0] != dim_len: + raise IndexError( + f"Boolean array has the wrong length for dimension; expected {dim_len}, got {dim_sel.shape[0]}" + ) + + # precompute number of selected items for each chunk + nchunks = ceildiv(dim_len, dim_chunk_len) + chunk_nitems = np.zeros(nchunks, dtype="i8") + for dim_chunk_ix in range(nchunks): + dim_offset = dim_chunk_ix * dim_chunk_len + chunk_nitems[dim_chunk_ix] = np.count_nonzero( + dim_sel[dim_offset : dim_offset + dim_chunk_len] + ) + chunk_nitems_cumsum = np.cumsum(chunk_nitems) + nitems = chunk_nitems_cumsum[-1] + dim_chunk_ixs = np.nonzero(chunk_nitems)[0] + + # store attributes + object.__setattr__(self, "dim_sel", dim_sel) + object.__setattr__(self, "dim_len", dim_len) + object.__setattr__(self, "dim_chunk_len", dim_chunk_len) + object.__setattr__(self, "nchunks", nchunks) + object.__setattr__(self, "chunk_nitems", chunk_nitems) + object.__setattr__(self, "chunk_nitems_cumsum", chunk_nitems_cumsum) + object.__setattr__(self, "nitems", nitems) + object.__setattr__(self, "dim_chunk_ixs", dim_chunk_ixs) + + def __iter__(self) -> Iterator[ChunkDimProjection]: + # iterate over chunks with at least one item + for dim_chunk_ix in self.dim_chunk_ixs: + # find region in chunk + dim_offset = dim_chunk_ix * self.dim_chunk_len + dim_chunk_sel = self.dim_sel[dim_offset : dim_offset + self.dim_chunk_len] + + # pad out if final chunk + if dim_chunk_sel.shape[0] < self.dim_chunk_len: + tmp = np.zeros(self.dim_chunk_len, dtype=bool) + tmp[: dim_chunk_sel.shape[0]] = dim_chunk_sel + dim_chunk_sel = tmp + + # find region in output + if dim_chunk_ix == 0: + start = 0 + else: + start = self.chunk_nitems_cumsum[dim_chunk_ix - 1] + stop = self.chunk_nitems_cumsum[dim_chunk_ix] + dim_out_sel = slice(start, stop) + is_complete_chunk = False # TODO + + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) + + +class Order(Enum): + """ + Enum for indexing order. + """ + + UNKNOWN = 0 + INCREASING = 1 + DECREASING = 2 + UNORDERED = 3 + + @staticmethod + def check(a: npt.NDArray[Any]) -> Order: + diff = np.diff(a) + diff_positive = diff >= 0 + n_diff_positive = np.count_nonzero(diff_positive) + all_increasing = n_diff_positive == len(diff_positive) + any_increasing = n_diff_positive > 0 + if all_increasing: + order = Order.INCREASING + elif any_increasing: + order = Order.UNORDERED + else: + order = Order.DECREASING + return order + + +def wraparound_indices(x: npt.NDArray[Any], dim_len: int) -> None: + loc_neg = x < 0 + if np.any(loc_neg): + x[loc_neg] += dim_len + + +def boundscheck_indices(x: npt.NDArray[Any], dim_len: int) -> None: + if np.any(x < 0) or np.any(x >= dim_len): + raise BoundsCheckError(dim_len) + + +@dataclass(frozen=True) +class IntArrayDimIndexer: + """Integer array selection against a single dimension.""" + + dim_len: int + dim_chunk_len: int + nchunks: int + nitems: int + order: Order + dim_sel: npt.NDArray[np.intp] + dim_out_sel: npt.NDArray[np.intp] + chunk_nitems: int + dim_chunk_ixs: npt.NDArray[np.intp] + chunk_nitems_cumsum: npt.NDArray[np.intp] + + def __init__( + self, + dim_sel: npt.NDArray[np.intp], + dim_len: int, + dim_chunk_len: int, + wraparound: bool = True, + boundscheck: bool = True, + order: Order = Order.UNKNOWN, + ) -> None: + # ensure 1d array + dim_sel = np.asanyarray(dim_sel) + if not is_integer_array(dim_sel, 1): + raise IndexError("integer arrays in an orthogonal selection must be 1-dimensional only") + + nitems = len(dim_sel) + nchunks = ceildiv(dim_len, dim_chunk_len) + + # handle wraparound + if wraparound: + wraparound_indices(dim_sel, dim_len) + + # handle out of bounds + if boundscheck: + boundscheck_indices(dim_sel, dim_len) + + # determine which chunk is needed for each selection item + # note: for dense integer selections, the division operation here is the + # bottleneck + dim_sel_chunk = dim_sel // dim_chunk_len + + # determine order of indices + if order == Order.UNKNOWN: + order = Order.check(dim_sel) + order = Order(order) + + if order == Order.INCREASING: + dim_out_sel = None + elif order == Order.DECREASING: + dim_sel = dim_sel[::-1] + # TODO should be possible to do this without creating an arange + dim_out_sel = np.arange(nitems - 1, -1, -1) + else: + # sort indices to group by chunk + dim_out_sel = np.argsort(dim_sel_chunk) + dim_sel = np.take(dim_sel, dim_out_sel) + + # precompute number of selected items for each chunk + chunk_nitems = np.bincount(dim_sel_chunk, minlength=nchunks) + + # find chunks that we need to visit + dim_chunk_ixs = np.nonzero(chunk_nitems)[0] + + # compute offsets into the output array + chunk_nitems_cumsum = np.cumsum(chunk_nitems) + + # store attributes + object.__setattr__(self, "dim_len", dim_len) + object.__setattr__(self, "dim_chunk_len", dim_chunk_len) + object.__setattr__(self, "nchunks", nchunks) + object.__setattr__(self, "nitems", nitems) + object.__setattr__(self, "order", order) + object.__setattr__(self, "dim_sel", dim_sel) + object.__setattr__(self, "dim_out_sel", dim_out_sel) + object.__setattr__(self, "chunk_nitems", chunk_nitems) + object.__setattr__(self, "dim_chunk_ixs", dim_chunk_ixs) + object.__setattr__(self, "chunk_nitems_cumsum", chunk_nitems_cumsum) + + def __iter__(self) -> Iterator[ChunkDimProjection]: + for dim_chunk_ix in self.dim_chunk_ixs: + dim_out_sel: slice | npt.NDArray[np.intp] + # find region in output + if dim_chunk_ix == 0: + start = 0 + else: + start = self.chunk_nitems_cumsum[dim_chunk_ix - 1] + stop = self.chunk_nitems_cumsum[dim_chunk_ix] + if self.order == Order.INCREASING: + dim_out_sel = slice(start, stop) + else: + dim_out_sel = self.dim_out_sel[start:stop] + + # find region in chunk + dim_offset = dim_chunk_ix * self.dim_chunk_len + dim_chunk_sel = self.dim_sel[start:stop] - dim_offset + is_complete_chunk = False # TODO + yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel, is_complete_chunk) + + +def slice_to_range(s: slice, length: int) -> range: + return range(*s.indices(length)) + + +def ix_(selection: Any, shape: ChunkCoords) -> npt.NDArray[np.intp]: + """Convert an orthogonal selection to a numpy advanced (fancy) selection, like ``numpy.ix_`` + but with support for slices and single ints.""" + + # normalisation + selection = replace_ellipsis(selection, shape) + + # replace slice and int as these are not supported by numpy.ix_ + selection = [ + slice_to_range(dim_sel, dim_len) + if isinstance(dim_sel, slice) + else [dim_sel] + if is_integer(dim_sel) + else dim_sel + for dim_sel, dim_len in zip(selection, shape, strict=True) + ] + + # now get numpy to convert to a coordinate selection + selection = np.ix_(*selection) + + return cast("npt.NDArray[np.intp]", selection) + + +def oindex(a: npt.NDArray[Any], selection: Selection) -> npt.NDArray[Any]: + """Implementation of orthogonal indexing with slices and ints.""" + selection = replace_ellipsis(selection, a.shape) + drop_axes = tuple(i for i, s in enumerate(selection) if is_integer(s)) + selection = ix_(selection, a.shape) + result = a[selection] + if drop_axes: + result = result.squeeze(axis=drop_axes) + return result + + +def oindex_set(a: npt.NDArray[Any], selection: Selection, value: Any) -> None: + selection = replace_ellipsis(selection, a.shape) + drop_axes = tuple(i for i, s in enumerate(selection) if is_integer(s)) + selection = ix_(selection, a.shape) + if not np.isscalar(value) and drop_axes: + value = np.asanyarray(value) + value_selection: list[Selector | None] = [slice(None)] * len(a.shape) + for i in drop_axes: + value_selection[i] = np.newaxis + value = value[tuple(value_selection)] + a[selection] = value + + +@dataclass(frozen=True) +class OrthogonalIndexer(Indexer): + dim_indexers: list[IntDimIndexer | SliceDimIndexer | IntArrayDimIndexer | BoolArrayDimIndexer] + shape: ChunkCoords + chunk_shape: ChunkCoords + is_advanced: bool + drop_axes: tuple[int, ...] + + def __init__(self, selection: Selection, shape: ChunkCoords, chunk_grid: ChunkGrid) -> None: + chunk_shape = get_chunk_shape(chunk_grid) + + # handle ellipsis + selection = replace_ellipsis(selection, shape) + + # normalize list to array + selection = replace_lists(selection) + + # setup per-dimension indexers + dim_indexers: list[ + IntDimIndexer | SliceDimIndexer | IntArrayDimIndexer | BoolArrayDimIndexer + ] = [] + for dim_sel, dim_len, dim_chunk_len in zip(selection, shape, chunk_shape, strict=True): + dim_indexer: IntDimIndexer | SliceDimIndexer | IntArrayDimIndexer | BoolArrayDimIndexer + if is_integer(dim_sel): + dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_chunk_len) + + elif isinstance(dim_sel, slice): + dim_indexer = SliceDimIndexer(dim_sel, dim_len, dim_chunk_len) + + elif is_integer_array(dim_sel): + dim_indexer = IntArrayDimIndexer(dim_sel, dim_len, dim_chunk_len) + + elif is_bool_array(dim_sel): + dim_indexer = BoolArrayDimIndexer(dim_sel, dim_len, dim_chunk_len) + + else: + raise IndexError( + "unsupported selection item for orthogonal indexing; " + "expected integer, slice, integer array or Boolean " + f"array, got {type(dim_sel)!r}" + ) + + dim_indexers.append(dim_indexer) + + shape = tuple(s.nitems for s in dim_indexers if not isinstance(s, IntDimIndexer)) + is_advanced = not is_basic_selection(selection) + if is_advanced: + drop_axes = tuple( + i + for i, dim_indexer in enumerate(dim_indexers) + if isinstance(dim_indexer, IntDimIndexer) + ) + else: + drop_axes = () + + object.__setattr__(self, "dim_indexers", dim_indexers) + object.__setattr__(self, "shape", shape) + object.__setattr__(self, "chunk_shape", chunk_shape) + object.__setattr__(self, "is_advanced", is_advanced) + object.__setattr__(self, "drop_axes", drop_axes) + + def __iter__(self) -> Iterator[ChunkProjection]: + for dim_projections in itertools.product(*self.dim_indexers): + chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) + chunk_selection: tuple[Selector, ...] | npt.NDArray[Any] = tuple( + p.dim_chunk_sel for p in dim_projections + ) + out_selection: tuple[Selector, ...] | npt.NDArray[Any] = tuple( + p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None + ) + + # handle advanced indexing arrays orthogonally + if self.is_advanced: + # N.B., numpy doesn't support orthogonal indexing directly as yet, + # so need to work around via np.ix_. Also np.ix_ does not support a + # mixture of arrays and slices or integers, so need to convert slices + # and integers into ranges. + chunk_selection = ix_(chunk_selection, self.chunk_shape) + + # special case for non-monotonic indices + if not is_basic_selection(out_selection): + out_selection = ix_(out_selection, self.shape) + + is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) + + +@dataclass(frozen=True) +class OIndex: + array: Array + + # TODO: develop Array generic and move zarr.Array[np.intp] | zarr.Array[np.bool_] to ArrayOfIntOrBool + def __getitem__(self, selection: OrthogonalSelection | Array) -> NDArrayLikeOrScalar: + from zarr.core.array import Array + + # if input is a Zarr array, we materialize it now. + if isinstance(selection, Array): + selection = _zarr_array_to_int_or_bool_array(selection) + + fields, new_selection = pop_fields(selection) + new_selection = ensure_tuple(new_selection) + new_selection = replace_lists(new_selection) + return self.array.get_orthogonal_selection( + cast("OrthogonalSelection", new_selection), fields=fields + ) + + def __setitem__(self, selection: OrthogonalSelection, value: npt.ArrayLike) -> None: + fields, new_selection = pop_fields(selection) + new_selection = ensure_tuple(new_selection) + new_selection = replace_lists(new_selection) + return self.array.set_orthogonal_selection( + cast("OrthogonalSelection", new_selection), value, fields=fields + ) + + +@dataclass(frozen=True) +class BlockIndexer(Indexer): + dim_indexers: list[SliceDimIndexer] + shape: ChunkCoords + drop_axes: ChunkCoords + + def __init__( + self, selection: BasicSelection, shape: ChunkCoords, chunk_grid: ChunkGrid + ) -> None: + chunk_shape = get_chunk_shape(chunk_grid) + + # handle ellipsis + selection_normalized = replace_ellipsis(selection, shape) + + # normalize list to array + selection_normalized = replace_lists(selection_normalized) + + # setup per-dimension indexers + dim_indexers = [] + for dim_sel, dim_len, dim_chunk_size in zip( + selection_normalized, shape, chunk_shape, strict=True + ): + dim_numchunks = int(np.ceil(dim_len / dim_chunk_size)) + + if is_integer(dim_sel): + if dim_sel < 0: + dim_sel = dim_numchunks + dim_sel + + start = dim_sel * dim_chunk_size + stop = start + dim_chunk_size + slice_ = slice(start, stop) + + elif is_slice(dim_sel): + start = dim_sel.start if dim_sel.start is not None else 0 + stop = dim_sel.stop if dim_sel.stop is not None else dim_numchunks + + if dim_sel.step not in {1, None}: + raise IndexError( + "unsupported selection item for block indexing; " + f"expected integer or slice with step=1, got {type(dim_sel)!r}" + ) + + # Can't reuse wraparound_indices because it expects a numpy array + # We have integers here. + if start < 0: + start = dim_numchunks + start + if stop < 0: + stop = dim_numchunks + stop + + start *= dim_chunk_size + stop *= dim_chunk_size + slice_ = slice(start, stop) + + else: + raise IndexError( + "unsupported selection item for block indexing; " + f"expected integer or slice, got {type(dim_sel)!r}" + ) + + dim_indexer = SliceDimIndexer(slice_, dim_len, dim_chunk_size) + dim_indexers.append(dim_indexer) + + if start >= dim_len or start < 0: + raise BoundsCheckError(dim_len) + + shape = tuple(s.nitems for s in dim_indexers) + + object.__setattr__(self, "dim_indexers", dim_indexers) + object.__setattr__(self, "shape", shape) + object.__setattr__(self, "drop_axes", ()) + + def __iter__(self) -> Iterator[ChunkProjection]: + for dim_projections in itertools.product(*self.dim_indexers): + chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) + chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) + out_selection = tuple( + p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None + ) + is_complete_chunk = all(p.is_complete_chunk for p in dim_projections) + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) + + +@dataclass(frozen=True) +class BlockIndex: + array: Array + + def __getitem__(self, selection: BasicSelection) -> NDArrayLikeOrScalar: + fields, new_selection = pop_fields(selection) + new_selection = ensure_tuple(new_selection) + new_selection = replace_lists(new_selection) + return self.array.get_block_selection(cast("BasicSelection", new_selection), fields=fields) + + def __setitem__(self, selection: BasicSelection, value: npt.ArrayLike) -> None: + fields, new_selection = pop_fields(selection) + new_selection = ensure_tuple(new_selection) + new_selection = replace_lists(new_selection) + return self.array.set_block_selection( + cast("BasicSelection", new_selection), value, fields=fields + ) + + +def is_coordinate_selection( + selection: SelectionNormalized, shape: ChunkCoords +) -> TypeGuard[CoordinateSelectionNormalized]: + return ( + isinstance(selection, tuple) + and len(selection) == len(shape) + and all(is_integer(dim_sel) or is_integer_array(dim_sel) for dim_sel in selection) + ) + + +def is_mask_selection(selection: Selection, shape: ChunkCoords) -> TypeGuard[MaskSelection]: + return ( + isinstance(selection, tuple) + and len(selection) == 1 + and is_bool_array(selection[0]) + and selection[0].shape == shape + ) + + +@dataclass(frozen=True) +class CoordinateIndexer(Indexer): + sel_shape: ChunkCoords + selection: CoordinateSelectionNormalized + sel_sort: npt.NDArray[np.intp] | None + chunk_nitems_cumsum: npt.NDArray[np.intp] + chunk_rixs: npt.NDArray[np.intp] + chunk_mixs: tuple[npt.NDArray[np.intp], ...] + shape: ChunkCoords + chunk_shape: ChunkCoords + drop_axes: ChunkCoords + + def __init__( + self, selection: CoordinateSelection, shape: ChunkCoords, chunk_grid: ChunkGrid + ) -> None: + chunk_shape = get_chunk_shape(chunk_grid) + + cdata_shape: ChunkCoords + if shape == (): + cdata_shape = (1,) + else: + cdata_shape = tuple(math.ceil(s / c) for s, c in zip(shape, chunk_shape, strict=True)) + nchunks = reduce(operator.mul, cdata_shape, 1) + + # some initial normalization + selection_normalized = cast("CoordinateSelectionNormalized", ensure_tuple(selection)) + selection_normalized = tuple( + np.asarray([i]) if is_integer(i) else i for i in selection_normalized + ) + selection_normalized = cast( + "CoordinateSelectionNormalized", replace_lists(selection_normalized) + ) + + # validation + if not is_coordinate_selection(selection_normalized, shape): + raise IndexError( + "invalid coordinate selection; expected one integer " + "(coordinate) array per dimension of the target array, " + f"got {selection!r}" + ) + + # handle wraparound, boundscheck + for dim_sel, dim_len in zip(selection_normalized, shape, strict=True): + # handle wraparound + wraparound_indices(dim_sel, dim_len) + + # handle out of bounds + boundscheck_indices(dim_sel, dim_len) + + # compute chunk index for each point in the selection + chunks_multi_index = tuple( + dim_sel // dim_chunk_len + for (dim_sel, dim_chunk_len) in zip(selection_normalized, chunk_shape, strict=True) + ) + + # broadcast selection - this will raise error if array dimensions don't match + selection_broadcast = tuple(np.broadcast_arrays(*selection_normalized)) + chunks_multi_index_broadcast = np.broadcast_arrays(*chunks_multi_index) + + # remember shape of selection, because we will flatten indices for processing + sel_shape = selection_broadcast[0].shape or (1,) + + # flatten selection + selection_broadcast = tuple(dim_sel.reshape(-1) for dim_sel in selection_broadcast) + chunks_multi_index_broadcast = tuple( + dim_chunks.reshape(-1) for dim_chunks in chunks_multi_index_broadcast + ) + + # ravel chunk indices + chunks_raveled_indices = np.ravel_multi_index( + chunks_multi_index_broadcast, dims=cdata_shape + ) + + # group points by chunk + if np.any(np.diff(chunks_raveled_indices) < 0): + # optimisation, only sort if needed + sel_sort = np.argsort(chunks_raveled_indices) + selection_broadcast = tuple(dim_sel[sel_sort] for dim_sel in selection_broadcast) + else: + sel_sort = None + + shape = selection_broadcast[0].shape or (1,) + + # precompute number of selected items for each chunk + chunk_nitems = np.bincount(chunks_raveled_indices, minlength=nchunks) + chunk_nitems_cumsum = np.cumsum(chunk_nitems) + # locate the chunks we need to process + chunk_rixs = np.nonzero(chunk_nitems)[0] + + # unravel chunk indices + chunk_mixs = np.unravel_index(chunk_rixs, cdata_shape) + + object.__setattr__(self, "sel_shape", sel_shape) + object.__setattr__(self, "selection", selection_broadcast) + object.__setattr__(self, "sel_sort", sel_sort) + object.__setattr__(self, "chunk_nitems_cumsum", chunk_nitems_cumsum) + object.__setattr__(self, "chunk_rixs", chunk_rixs) + object.__setattr__(self, "chunk_mixs", chunk_mixs) + object.__setattr__(self, "chunk_shape", chunk_shape) + object.__setattr__(self, "shape", shape) + object.__setattr__(self, "drop_axes", ()) + + def __iter__(self) -> Iterator[ChunkProjection]: + # iterate over chunks + for i, chunk_rix in enumerate(self.chunk_rixs): + chunk_coords = tuple(m[i] for m in self.chunk_mixs) + if chunk_rix == 0: + start = 0 + else: + start = self.chunk_nitems_cumsum[chunk_rix - 1] + stop = self.chunk_nitems_cumsum[chunk_rix] + out_selection: slice | npt.NDArray[np.intp] + if self.sel_sort is None: + out_selection = slice(start, stop) + else: + out_selection = self.sel_sort[start:stop] + + chunk_offsets = tuple( + dim_chunk_ix * dim_chunk_len + for dim_chunk_ix, dim_chunk_len in zip(chunk_coords, self.chunk_shape, strict=True) + ) + chunk_selection = tuple( + dim_sel[start:stop] - dim_chunk_offset + for (dim_sel, dim_chunk_offset) in zip(self.selection, chunk_offsets, strict=True) + ) + + is_complete_chunk = False # TODO + yield ChunkProjection(chunk_coords, chunk_selection, out_selection, is_complete_chunk) + + +@dataclass(frozen=True) +class MaskIndexer(CoordinateIndexer): + def __init__(self, selection: MaskSelection, shape: ChunkCoords, chunk_grid: ChunkGrid) -> None: + # some initial normalization + selection_normalized = cast("tuple[MaskSelection]", ensure_tuple(selection)) + selection_normalized = cast("tuple[MaskSelection]", replace_lists(selection_normalized)) + + # validation + if not is_mask_selection(selection_normalized, shape): + raise IndexError( + "invalid mask selection; expected one Boolean (mask)" + f"array with the same shape as the target array, got {selection_normalized!r}" + ) + + # convert to indices + selection_indices = np.nonzero(selection_normalized[0]) + + # delegate the rest to superclass + super().__init__(selection_indices, shape, chunk_grid) + + +@dataclass(frozen=True) +class VIndex: + array: Array + + # TODO: develop Array generic and move zarr.Array[np.intp] | zarr.Array[np.bool_] to ArrayOfIntOrBool + def __getitem__( + self, selection: CoordinateSelection | MaskSelection | Array + ) -> NDArrayLikeOrScalar: + from zarr.core.array import Array + + # if input is a Zarr array, we materialize it now. + if isinstance(selection, Array): + selection = _zarr_array_to_int_or_bool_array(selection) + fields, new_selection = pop_fields(selection) + new_selection = ensure_tuple(new_selection) + new_selection = replace_lists(new_selection) + if is_coordinate_selection(new_selection, self.array.shape): + return self.array.get_coordinate_selection(new_selection, fields=fields) + elif is_mask_selection(new_selection, self.array.shape): + return self.array.get_mask_selection(new_selection, fields=fields) + else: + raise VindexInvalidSelectionError(new_selection) + + def __setitem__( + self, selection: CoordinateSelection | MaskSelection, value: npt.ArrayLike + ) -> None: + fields, new_selection = pop_fields(selection) + new_selection = ensure_tuple(new_selection) + new_selection = replace_lists(new_selection) + if is_coordinate_selection(new_selection, self.array.shape): + self.array.set_coordinate_selection(new_selection, value, fields=fields) + elif is_mask_selection(new_selection, self.array.shape): + self.array.set_mask_selection(new_selection, value, fields=fields) + else: + raise VindexInvalidSelectionError(new_selection) + + +def check_fields(fields: Fields | None, dtype: np.dtype[Any]) -> np.dtype[Any]: + # early out + if fields is None: + return dtype + # check type + if not isinstance(fields, str | list | tuple): + raise IndexError( + f"'fields' argument must be a string or list of strings; found {type(fields)!r}" + ) + if fields: + if dtype.names is None: + raise IndexError("invalid 'fields' argument, array does not have any fields") + try: + if isinstance(fields, str): + # single field selection + out_dtype = dtype[fields] + else: + # multiple field selection + out_dtype = np.dtype([(f, dtype[f]) for f in fields]) + except KeyError as e: + raise IndexError(f"invalid 'fields' argument, field not found: {e!r}") from e + else: + return out_dtype + else: + return dtype + + +def check_no_multi_fields(fields: Fields | None) -> Fields | None: + if isinstance(fields, list): + if len(fields) == 1: + return fields[0] + elif len(fields) > 1: + raise IndexError("multiple fields are not supported for this operation") + return fields + + +def pop_fields(selection: SelectionWithFields) -> tuple[Fields | None, Selection]: + if isinstance(selection, str): + # single field selection + return selection, () + elif not isinstance(selection, tuple): + # single selection item, no fields + # leave selection as-is + return None, cast("Selection", selection) + else: + # multiple items, split fields from selection items + fields: Fields = [f for f in selection if isinstance(f, str)] + fields = fields[0] if len(fields) == 1 else fields + selection_tuple = tuple(s for s in selection if not isinstance(s, str)) + selection = cast( + "Selection", selection_tuple[0] if len(selection_tuple) == 1 else selection_tuple + ) + return fields, selection + + +def make_slice_selection(selection: Any) -> list[slice]: + ls: list[slice] = [] + for dim_selection in selection: + if is_integer(dim_selection): + ls.append(slice(int(dim_selection), int(dim_selection) + 1, 1)) + elif isinstance(dim_selection, np.ndarray): + if len(dim_selection) == 1: + ls.append(slice(int(dim_selection[0]), int(dim_selection[0]) + 1, 1)) + else: + raise ArrayIndexError + else: + ls.append(dim_selection) + return ls + + +def decode_morton(z: int, chunk_shape: ChunkCoords) -> ChunkCoords: + # Inspired by compressed morton code as implemented in Neuroglancer + # https://github.com/google/neuroglancer/blob/master/src/neuroglancer/datasource/precomputed/volume.md#compressed-morton-code + bits = tuple(math.ceil(math.log2(c)) for c in chunk_shape) + max_coords_bits = max(bits) + input_bit = 0 + input_value = z + out = [0] * len(chunk_shape) + + for coord_bit in range(max_coords_bits): + for dim in range(len(chunk_shape)): + if coord_bit < bits[dim]: + bit = (input_value >> input_bit) & 1 + out[dim] |= bit << coord_bit + input_bit += 1 + return tuple(out) + + +def morton_order_iter(chunk_shape: ChunkCoords) -> Iterator[ChunkCoords]: + i = 0 + order: list[ChunkCoords] = [] + while len(order) < product(chunk_shape): + m = decode_morton(i, chunk_shape) + if m not in order and all(x < y for x, y in zip(m, chunk_shape, strict=False)): + order.append(m) + i += 1 + for j in range(product(chunk_shape)): + yield order[j] + + +def c_order_iter(chunks_per_shard: ChunkCoords) -> Iterator[ChunkCoords]: + return itertools.product(*(range(x) for x in chunks_per_shard)) + + +def get_indexer( + selection: SelectionWithFields, shape: ChunkCoords, chunk_grid: ChunkGrid +) -> Indexer: + _, pure_selection = pop_fields(selection) + if is_pure_fancy_indexing(pure_selection, len(shape)): + new_selection = ensure_tuple(selection) + new_selection = replace_lists(new_selection) + if is_coordinate_selection(new_selection, shape): + return CoordinateIndexer(cast("CoordinateSelection", selection), shape, chunk_grid) + elif is_mask_selection(new_selection, shape): + return MaskIndexer(cast("MaskSelection", selection), shape, chunk_grid) + else: + raise VindexInvalidSelectionError(new_selection) + elif is_pure_orthogonal_indexing(pure_selection, len(shape)): + return OrthogonalIndexer(cast("OrthogonalSelection", selection), shape, chunk_grid) + else: + return BasicIndexer(cast("BasicSelection", selection), shape, chunk_grid) diff --git a/src/zarr/core/metadata/__init__.py b/src/zarr/core/metadata/__init__.py new file mode 100644 index 0000000000..43b5ec98fe --- /dev/null +++ b/src/zarr/core/metadata/__init__.py @@ -0,0 +1,17 @@ +from typing import TypeAlias, TypeVar + +from .v2 import ArrayV2Metadata, ArrayV2MetadataDict +from .v3 import ArrayV3Metadata, ArrayV3MetadataDict + +ArrayMetadata: TypeAlias = ArrayV2Metadata | ArrayV3Metadata +ArrayMetadataDict: TypeAlias = ArrayV2MetadataDict | ArrayV3MetadataDict +T_ArrayMetadata = TypeVar("T_ArrayMetadata", ArrayV2Metadata, ArrayV3Metadata) + +__all__ = [ + "ArrayMetadata", + "ArrayMetadataDict", + "ArrayV2Metadata", + "ArrayV2MetadataDict", + "ArrayV3Metadata", + "ArrayV3MetadataDict", +] diff --git a/src/zarr/core/metadata/common.py b/src/zarr/core/metadata/common.py new file mode 100644 index 0000000000..44d3eb292b --- /dev/null +++ b/src/zarr/core/metadata/common.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from zarr.core.common import JSON + + +def parse_attributes(data: dict[str, JSON] | None) -> dict[str, JSON]: + if data is None: + return {} + + return data diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py new file mode 100644 index 0000000000..3ac75e0418 --- /dev/null +++ b/src/zarr/core/metadata/v2.py @@ -0,0 +1,331 @@ +from __future__ import annotations + +import warnings +from collections.abc import Iterable, Sequence +from functools import cached_property +from typing import TYPE_CHECKING, Any, TypeAlias, TypedDict, cast + +import numcodecs.abc + +from zarr.abc.metadata import Metadata +from zarr.core.chunk_grids import RegularChunkGrid +from zarr.core.dtype import get_data_type_from_json +from zarr.core.dtype.common import OBJECT_CODEC_IDS, DTypeSpec_V2 + +if TYPE_CHECKING: + from typing import Literal, Self + + import numpy.typing as npt + + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.common import ChunkCoords + from zarr.core.dtype.wrapper import ( + TBaseDType, + TBaseScalar, + TDType_co, + TScalar_co, + ZDType, + ) + +import json +from dataclasses import dataclass, field, fields, replace + +import numcodecs +import numpy as np + +from zarr.core.array_spec import ArrayConfig, ArraySpec +from zarr.core.chunk_key_encodings import parse_separator +from zarr.core.common import ( + JSON, + ZARRAY_JSON, + ZATTRS_JSON, + MemoryOrder, + parse_shapelike, +) +from zarr.core.config import config, parse_indexing_order +from zarr.core.metadata.common import parse_attributes + + +class ArrayV2MetadataDict(TypedDict): + """ + A typed dictionary model for Zarr format 2 metadata. + """ + + zarr_format: Literal[2] + attributes: dict[str, JSON] + + +# Union of acceptable types for v2 compressors +CompressorLikev2: TypeAlias = dict[str, JSON] | numcodecs.abc.Codec | None + + +@dataclass(frozen=True, kw_only=True) +class ArrayV2Metadata(Metadata): + shape: ChunkCoords + chunks: ChunkCoords + dtype: ZDType[TBaseDType, TBaseScalar] + fill_value: int | float | str | bytes | None = None + order: MemoryOrder = "C" + filters: tuple[numcodecs.abc.Codec, ...] | None = None + dimension_separator: Literal[".", "/"] = "." + compressor: CompressorLikev2 + attributes: dict[str, JSON] = field(default_factory=dict) + zarr_format: Literal[2] = field(init=False, default=2) + + def __init__( + self, + *, + shape: ChunkCoords, + dtype: ZDType[TDType_co, TScalar_co], + chunks: ChunkCoords, + fill_value: Any, + order: MemoryOrder, + dimension_separator: Literal[".", "/"] = ".", + compressor: CompressorLikev2 = None, + filters: Iterable[numcodecs.abc.Codec | dict[str, JSON]] | None = None, + attributes: dict[str, JSON] | None = None, + ) -> None: + """ + Metadata for a Zarr format 2 array. + """ + shape_parsed = parse_shapelike(shape) + chunks_parsed = parse_shapelike(chunks) + compressor_parsed = parse_compressor(compressor) + order_parsed = parse_indexing_order(order) + dimension_separator_parsed = parse_separator(dimension_separator) + filters_parsed = parse_filters(filters) + fill_value_parsed: TBaseScalar | None + if fill_value is not None: + fill_value_parsed = dtype.cast_scalar(fill_value) + else: + fill_value_parsed = fill_value + attributes_parsed = parse_attributes(attributes) + + object.__setattr__(self, "shape", shape_parsed) + object.__setattr__(self, "dtype", dtype) + object.__setattr__(self, "chunks", chunks_parsed) + object.__setattr__(self, "compressor", compressor_parsed) + object.__setattr__(self, "order", order_parsed) + object.__setattr__(self, "dimension_separator", dimension_separator_parsed) + object.__setattr__(self, "filters", filters_parsed) + object.__setattr__(self, "fill_value", fill_value_parsed) + object.__setattr__(self, "attributes", attributes_parsed) + + # ensure that the metadata document is consistent + _ = parse_metadata(self) + + @property + def ndim(self) -> int: + return len(self.shape) + + @cached_property + def chunk_grid(self) -> RegularChunkGrid: + return RegularChunkGrid(chunk_shape=self.chunks) + + @property + def shards(self) -> ChunkCoords | None: + return None + + def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: + zarray_dict = self.to_dict() + zattrs_dict = zarray_dict.pop("attributes", {}) + json_indent = config.get("json_indent") + return { + ZARRAY_JSON: prototype.buffer.from_bytes( + json.dumps(zarray_dict, indent=json_indent, allow_nan=False).encode() + ), + ZATTRS_JSON: prototype.buffer.from_bytes( + json.dumps(zattrs_dict, indent=json_indent, allow_nan=False).encode() + ), + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata: + # Make a copy to protect the original from modification. + _data = data.copy() + # Check that the zarr_format attribute is correct. + _ = parse_zarr_format(_data.pop("zarr_format")) + + # To resolve a numpy object dtype array, we need to search for an object codec, + # which could be in filters or as a compressor. + # we will reference a hard-coded collection of object codec ids for this search. + + _filters, _compressor = (data.get("filters"), data.get("compressor")) + if _filters is not None: + _filters = cast("tuple[dict[str, JSON], ...]", _filters) + object_codec_id = get_object_codec_id(tuple(_filters) + (_compressor,)) + else: + object_codec_id = get_object_codec_id((_compressor,)) + # we add a layer of indirection here around the dtype attribute of the array metadata + # because we also need to know the object codec id, if any, to resolve the data type + dtype_spec: DTypeSpec_V2 = { + "name": data["dtype"], + "object_codec_id": object_codec_id, + } + dtype = get_data_type_from_json(dtype_spec, zarr_format=2) + + _data["dtype"] = dtype + fill_value_encoded = _data.get("fill_value") + if fill_value_encoded is not None: + fill_value = dtype.from_json_scalar(fill_value_encoded, zarr_format=2) + _data["fill_value"] = fill_value + + # zarr v2 allowed arbitrary keys here. + # We don't want the ArrayV2Metadata constructor to fail just because someone put an + # extra key in the metadata. + expected = {x.name for x in fields(cls)} + expected |= {"dtype", "chunks"} + + # check if `filters` is an empty sequence; if so use None instead and raise a warning + filters = _data.get("filters") + if ( + isinstance(filters, Sequence) + and not isinstance(filters, (str, bytes)) + and len(filters) == 0 + ): + msg = ( + "Found an empty list of filters in the array metadata document. " + "This is contrary to the Zarr V2 specification, and will cause an error in the future. " + "Use None (or Null in a JSON document) instead of an empty list of filters." + ) + warnings.warn(msg, UserWarning, stacklevel=1) + _data["filters"] = None + + _data = {k: v for k, v in _data.items() if k in expected} + + return cls(**_data) + + def to_dict(self) -> dict[str, JSON]: + zarray_dict = super().to_dict() + if isinstance(zarray_dict["compressor"], numcodecs.abc.Codec): + codec_config = zarray_dict["compressor"].get_config() + # Hotfix for https://github.com/zarr-developers/zarr-python/issues/2647 + if codec_config["id"] == "zstd" and not codec_config.get("checksum", False): + codec_config.pop("checksum") + zarray_dict["compressor"] = codec_config + + if zarray_dict["filters"] is not None: + raw_filters = zarray_dict["filters"] + # TODO: remove this when we can stratically type the output JSON data structure + # entirely + if not isinstance(raw_filters, list | tuple): + raise TypeError("Invalid type for filters. Expected a list or tuple.") + new_filters = [] + for f in raw_filters: + if isinstance(f, numcodecs.abc.Codec): + new_filters.append(f.get_config()) + else: + new_filters.append(f) + zarray_dict["filters"] = new_filters + + # serialize the fill value after dtype-specific JSON encoding + if self.fill_value is not None: + fill_value = self.dtype.to_json_scalar(self.fill_value, zarr_format=2) + zarray_dict["fill_value"] = fill_value + + # pull the "name" attribute out of the dtype spec returned by self.dtype.to_json + zarray_dict["dtype"] = self.dtype.to_json(zarr_format=2)["name"] + + return zarray_dict + + def get_chunk_spec( + self, _chunk_coords: ChunkCoords, array_config: ArrayConfig, prototype: BufferPrototype + ) -> ArraySpec: + return ArraySpec( + shape=self.chunks, + dtype=self.dtype, + fill_value=self.fill_value, + config=array_config, + prototype=prototype, + ) + + def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: + chunk_identifier = self.dimension_separator.join(map(str, chunk_coords)) + return "0" if chunk_identifier == "" else chunk_identifier + + def update_shape(self, shape: ChunkCoords) -> Self: + return replace(self, shape=shape) + + def update_attributes(self, attributes: dict[str, JSON]) -> Self: + return replace(self, attributes=attributes) + + +def parse_dtype(data: npt.DTypeLike) -> np.dtype[Any]: + if isinstance(data, list): # this is a valid _VoidDTypeLike check + data = [tuple(d) for d in data] + return np.dtype(data) + + +def parse_zarr_format(data: object) -> Literal[2]: + if data == 2: + return 2 + raise ValueError(f"Invalid value. Expected 2. Got {data}.") + + +def parse_filters(data: object) -> tuple[numcodecs.abc.Codec, ...] | None: + """ + Parse a potential tuple of filters + """ + out: list[numcodecs.abc.Codec] = [] + + if data is None: + return data + if isinstance(data, Iterable): + for idx, val in enumerate(data): + if isinstance(val, numcodecs.abc.Codec): + out.append(val) + elif isinstance(val, dict): + out.append(numcodecs.get_codec(val)) + else: + msg = f"Invalid filter at index {idx}. Expected a numcodecs.abc.Codec or a dict representation of numcodecs.abc.Codec. Got {type(val)} instead." + raise TypeError(msg) + if len(out) == 0: + # Per the v2 spec, an empty tuple is not allowed -- use None to express "no filters" + return None + else: + return tuple(out) + # take a single codec instance and wrap it in a tuple + if isinstance(data, numcodecs.abc.Codec): + return (data,) + msg = f"Invalid filters. Expected None, an iterable of numcodecs.abc.Codec or dict representations of numcodecs.abc.Codec. Got {type(data)} instead." + raise TypeError(msg) + + +def parse_compressor(data: object) -> numcodecs.abc.Codec | None: + """ + Parse a potential compressor. + """ + if data is None or isinstance(data, numcodecs.abc.Codec): + return data + if isinstance(data, dict): + return numcodecs.get_codec(data) + msg = f"Invalid compressor. Expected None, a numcodecs.abc.Codec, or a dict representation of a numcodecs.abc.Codec. Got {type(data)} instead." + raise ValueError(msg) + + +def parse_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata: + if (l_chunks := len(data.chunks)) != (l_shape := len(data.shape)): + msg = ( + f"The `shape` and `chunks` attributes must have the same length. " + f"`chunks` has length {l_chunks}, but `shape` has length {l_shape}." + ) + raise ValueError(msg) + return data + + +def get_object_codec_id(maybe_object_codecs: Sequence[JSON]) -> str | None: + """ + Inspect a sequence of codecs / filters for an "object codec", i.e. a codec + that can serialize object arrays to contiguous bytes. Zarr python + maintains a hard-coded set of object codec ids. If any element from the input + has an id that matches one of the hard-coded object codec ids, that id + is returned immediately. + """ + object_codec_id = None + for maybe_object_codec in maybe_object_codecs: + if ( + isinstance(maybe_object_codec, dict) + and maybe_object_codec.get("id") in OBJECT_CODEC_IDS + ): + return cast("str", maybe_object_codec["id"]) + return object_codec_id diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py new file mode 100644 index 0000000000..84872d3dbd --- /dev/null +++ b/src/zarr/core/metadata/v3.py @@ -0,0 +1,352 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, TypedDict + +from zarr.abc.metadata import Metadata +from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.dtype import VariableLengthUTF8, ZDType, get_data_type_from_json +from zarr.core.dtype.common import check_dtype_spec_v3 + +if TYPE_CHECKING: + from typing import Self + + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.chunk_grids import ChunkGrid + from zarr.core.common import JSON, ChunkCoords + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar + + +import json +from collections.abc import Iterable +from dataclasses import dataclass, field, replace +from typing import Any, Literal + +from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec +from zarr.core.array_spec import ArrayConfig, ArraySpec +from zarr.core.chunk_grids import ChunkGrid, RegularChunkGrid +from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike +from zarr.core.common import ( + JSON, + ZARR_JSON, + ChunkCoords, + DimensionNames, + parse_named_configuration, + parse_shapelike, +) +from zarr.core.config import config +from zarr.core.metadata.common import parse_attributes +from zarr.errors import MetadataValidationError, NodeTypeValidationError +from zarr.registry import get_codec_class + + +def parse_zarr_format(data: object) -> Literal[3]: + if data == 3: + return 3 + raise MetadataValidationError("zarr_format", 3, data) + + +def parse_node_type_array(data: object) -> Literal["array"]: + if data == "array": + return "array" + raise NodeTypeValidationError("node_type", "array", data) + + +def parse_codecs(data: object) -> tuple[Codec, ...]: + out: tuple[Codec, ...] = () + + if not isinstance(data, Iterable): + raise TypeError(f"Expected iterable, got {type(data)}") + + for c in data: + if isinstance( + c, ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec + ): # Can't use Codec here because of mypy limitation + out += (c,) + else: + name_parsed, _ = parse_named_configuration(c, require_configuration=False) + out += (get_codec_class(name_parsed).from_dict(c),) + + return out + + +def validate_array_bytes_codec(codecs: tuple[Codec, ...]) -> ArrayBytesCodec: + # ensure that we have at least one ArrayBytesCodec + abcs: list[ArrayBytesCodec] = [codec for codec in codecs if isinstance(codec, ArrayBytesCodec)] + if len(abcs) == 0: + raise ValueError("At least one ArrayBytesCodec is required.") + elif len(abcs) > 1: + raise ValueError("Only one ArrayBytesCodec is allowed.") + + return abcs[0] + + +def validate_codecs(codecs: tuple[Codec, ...], dtype: ZDType[TBaseDType, TBaseScalar]) -> None: + """Check that the codecs are valid for the given dtype""" + from zarr.codecs.sharding import ShardingCodec + + abc = validate_array_bytes_codec(codecs) + + # Recursively resolve array-bytes codecs within sharding codecs + while isinstance(abc, ShardingCodec): + abc = validate_array_bytes_codec(abc.codecs) + + # we need to have special codecs if we are decoding vlen strings or bytestrings + # TODO: use codec ID instead of class name + codec_class_name = abc.__class__.__name__ + # TODO: Fix typing here + if isinstance(dtype, VariableLengthUTF8) and not codec_class_name == "VLenUTF8Codec": # type: ignore[unreachable] + raise ValueError( + f"For string dtype, ArrayBytesCodec must be `VLenUTF8Codec`, got `{codec_class_name}`." + ) + + +def parse_dimension_names(data: object) -> tuple[str | None, ...] | None: + if data is None: + return data + elif isinstance(data, Iterable) and all(isinstance(x, type(None) | str) for x in data): + return tuple(data) + else: + msg = f"Expected either None or a iterable of str, got {type(data)}" + raise TypeError(msg) + + +def parse_storage_transformers(data: object) -> tuple[dict[str, JSON], ...]: + """ + Parse storage_transformers. Zarr python cannot use storage transformers + at this time, so this function doesn't attempt to validate them. + """ + if data is None: + return () + if isinstance(data, Iterable): + if len(tuple(data)) >= 1: + return data # type: ignore[return-value] + else: + return () + raise TypeError( + f"Invalid storage_transformers. Expected an iterable of dicts. Got {type(data)} instead." + ) + + +class ArrayV3MetadataDict(TypedDict): + """ + A typed dictionary model for zarr v3 metadata. + """ + + zarr_format: Literal[3] + attributes: dict[str, JSON] + + +@dataclass(frozen=True, kw_only=True) +class ArrayV3Metadata(Metadata): + shape: ChunkCoords + data_type: ZDType[TBaseDType, TBaseScalar] + chunk_grid: ChunkGrid + chunk_key_encoding: ChunkKeyEncoding + fill_value: Any + codecs: tuple[Codec, ...] + attributes: dict[str, Any] = field(default_factory=dict) + dimension_names: tuple[str | None, ...] | None = None + zarr_format: Literal[3] = field(default=3, init=False) + node_type: Literal["array"] = field(default="array", init=False) + storage_transformers: tuple[dict[str, JSON], ...] + + def __init__( + self, + *, + shape: Iterable[int], + data_type: ZDType[TBaseDType, TBaseScalar], + chunk_grid: dict[str, JSON] | ChunkGrid, + chunk_key_encoding: ChunkKeyEncodingLike, + fill_value: object, + codecs: Iterable[Codec | dict[str, JSON]], + attributes: dict[str, JSON] | None, + dimension_names: DimensionNames, + storage_transformers: Iterable[dict[str, JSON]] | None = None, + ) -> None: + """ + Because the class is a frozen dataclass, we set attributes using object.__setattr__ + """ + + shape_parsed = parse_shapelike(shape) + chunk_grid_parsed = ChunkGrid.from_dict(chunk_grid) + chunk_key_encoding_parsed = ChunkKeyEncoding.from_dict(chunk_key_encoding) + dimension_names_parsed = parse_dimension_names(dimension_names) + # Note: relying on a type method is numpy-specific + fill_value_parsed = data_type.cast_scalar(fill_value) + attributes_parsed = parse_attributes(attributes) + codecs_parsed_partial = parse_codecs(codecs) + storage_transformers_parsed = parse_storage_transformers(storage_transformers) + + array_spec = ArraySpec( + shape=shape_parsed, + dtype=data_type, + fill_value=fill_value_parsed, + config=ArrayConfig.from_dict({}), # TODO: config is not needed here. + prototype=default_buffer_prototype(), # TODO: prototype is not needed here. + ) + codecs_parsed = tuple(c.evolve_from_array_spec(array_spec) for c in codecs_parsed_partial) + validate_codecs(codecs_parsed_partial, data_type) + + object.__setattr__(self, "shape", shape_parsed) + object.__setattr__(self, "data_type", data_type) + object.__setattr__(self, "chunk_grid", chunk_grid_parsed) + object.__setattr__(self, "chunk_key_encoding", chunk_key_encoding_parsed) + object.__setattr__(self, "codecs", codecs_parsed) + object.__setattr__(self, "dimension_names", dimension_names_parsed) + object.__setattr__(self, "fill_value", fill_value_parsed) + object.__setattr__(self, "attributes", attributes_parsed) + object.__setattr__(self, "storage_transformers", storage_transformers_parsed) + + self._validate_metadata() + + def _validate_metadata(self) -> None: + if isinstance(self.chunk_grid, RegularChunkGrid) and len(self.shape) != len( + self.chunk_grid.chunk_shape + ): + raise ValueError( + "`chunk_shape` and `shape` need to have the same number of dimensions." + ) + if self.dimension_names is not None and len(self.shape) != len(self.dimension_names): + raise ValueError( + "`dimension_names` and `shape` need to have the same number of dimensions." + ) + if self.fill_value is None: + raise ValueError("`fill_value` is required.") + for codec in self.codecs: + codec.validate(shape=self.shape, dtype=self.data_type, chunk_grid=self.chunk_grid) + + @property + def ndim(self) -> int: + return len(self.shape) + + @property + def dtype(self) -> ZDType[TBaseDType, TBaseScalar]: + return self.data_type + + @property + def chunks(self) -> ChunkCoords: + if isinstance(self.chunk_grid, RegularChunkGrid): + from zarr.codecs.sharding import ShardingCodec + + if len(self.codecs) == 1 and isinstance(self.codecs[0], ShardingCodec): + sharding_codec = self.codecs[0] + assert isinstance(sharding_codec, ShardingCodec) # for mypy + return sharding_codec.chunk_shape + else: + return self.chunk_grid.chunk_shape + + msg = ( + f"The `chunks` attribute is only defined for arrays using `RegularChunkGrid`." + f"This array has a {self.chunk_grid} instead." + ) + raise NotImplementedError(msg) + + @property + def shards(self) -> ChunkCoords | None: + if isinstance(self.chunk_grid, RegularChunkGrid): + from zarr.codecs.sharding import ShardingCodec + + if len(self.codecs) == 1 and isinstance(self.codecs[0], ShardingCodec): + return self.chunk_grid.chunk_shape + else: + return None + + msg = ( + f"The `shards` attribute is only defined for arrays using `RegularChunkGrid`." + f"This array has a {self.chunk_grid} instead." + ) + raise NotImplementedError(msg) + + @property + def inner_codecs(self) -> tuple[Codec, ...]: + if isinstance(self.chunk_grid, RegularChunkGrid): + from zarr.codecs.sharding import ShardingCodec + + if len(self.codecs) == 1 and isinstance(self.codecs[0], ShardingCodec): + return self.codecs[0].codecs + return self.codecs + + def get_chunk_spec( + self, _chunk_coords: ChunkCoords, array_config: ArrayConfig, prototype: BufferPrototype + ) -> ArraySpec: + assert isinstance(self.chunk_grid, RegularChunkGrid), ( + "Currently, only regular chunk grid is supported" + ) + return ArraySpec( + shape=self.chunk_grid.chunk_shape, + dtype=self.dtype, + fill_value=self.fill_value, + config=array_config, + prototype=prototype, + ) + + def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: + return self.chunk_key_encoding.encode_chunk_key(chunk_coords) + + def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: + json_indent = config.get("json_indent") + d = self.to_dict() + return { + ZARR_JSON: prototype.buffer.from_bytes( + json.dumps(d, allow_nan=False, indent=json_indent).encode() + ) + } + + @classmethod + def from_dict(cls, data: dict[str, JSON]) -> Self: + # make a copy because we are modifying the dict + _data = data.copy() + + # check that the zarr_format attribute is correct + _ = parse_zarr_format(_data.pop("zarr_format")) + # check that the node_type attribute is correct + _ = parse_node_type_array(_data.pop("node_type")) + + data_type_json = _data.pop("data_type") + if not check_dtype_spec_v3(data_type_json): + raise ValueError(f"Invalid data_type: {data_type_json!r}") + data_type = get_data_type_from_json(data_type_json, zarr_format=3) + + # check that the fill value is consistent with the data type + try: + fill = _data.pop("fill_value") + fill_value_parsed = data_type.from_json_scalar(fill, zarr_format=3) + except ValueError as e: + raise TypeError(f"Invalid fill_value: {fill!r}") from e + + # dimension_names key is optional, normalize missing to `None` + _data["dimension_names"] = _data.pop("dimension_names", None) + + # attributes key is optional, normalize missing to `None` + _data["attributes"] = _data.pop("attributes", None) + + return cls(**_data, fill_value=fill_value_parsed, data_type=data_type) # type: ignore[arg-type] + + def to_dict(self) -> dict[str, JSON]: + out_dict = super().to_dict() + out_dict["fill_value"] = self.data_type.to_json_scalar( + self.fill_value, zarr_format=self.zarr_format + ) + if not isinstance(out_dict, dict): + raise TypeError(f"Expected dict. Got {type(out_dict)}.") + + # if `dimension_names` is `None`, we do not include it in + # the metadata document + if out_dict["dimension_names"] is None: + out_dict.pop("dimension_names") + + # TODO: replace the `to_dict` / `from_dict` on the `Metadata`` class with + # to_json, from_json, and have ZDType inherit from `Metadata` + # until then, we have this hack here, which relies on the fact that to_dict will pass through + # any non-`Metadata` fields as-is. + dtype_meta = out_dict["data_type"] + if isinstance(dtype_meta, ZDType): + out_dict["data_type"] = dtype_meta.to_json(zarr_format=3) # type: ignore[unreachable] + + return out_dict + + def update_shape(self, shape: ChunkCoords) -> Self: + return replace(self, shape=shape) + + def update_attributes(self, attributes: dict[str, JSON]) -> Self: + return replace(self, attributes=attributes) diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py new file mode 100644 index 0000000000..ffb04e764d --- /dev/null +++ b/src/zarr/core/sync.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import asyncio +import atexit +import logging +import os +import threading +from concurrent.futures import ThreadPoolExecutor, wait +from typing import TYPE_CHECKING, TypeVar + +from typing_extensions import ParamSpec + +from zarr.core.config import config + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine + from typing import Any + +logger = logging.getLogger(__name__) + + +P = ParamSpec("P") +T = TypeVar("T") + +# From https://github.com/fsspec/filesystem_spec/blob/master/fsspec/asyn.py + +iothread: list[threading.Thread | None] = [None] # dedicated IO thread +loop: list[asyncio.AbstractEventLoop | None] = [ + None +] # global event loop for any non-async instance +_lock: threading.Lock | None = None # global lock placeholder +_executor: ThreadPoolExecutor | None = None # global executor placeholder + + +class SyncError(Exception): + pass + + +def _get_lock() -> threading.Lock: + """Allocate or return a threading lock. + + The lock is allocated on first use to allow setting one lock per forked process. + """ + global _lock + if not _lock: + _lock = threading.Lock() + return _lock + + +def _get_executor() -> ThreadPoolExecutor: + """Return Zarr Thread Pool Executor + + The executor is allocated on first use. + """ + global _executor + if not _executor: + max_workers = config.get("threading.max_workers", None) + logger.debug("Creating Zarr ThreadPoolExecutor with max_workers=%s", max_workers) + _executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="zarr_pool") + _get_loop().set_default_executor(_executor) + return _executor + + +def cleanup_resources() -> None: + global _executor + if _executor: + _executor.shutdown(wait=True, cancel_futures=True) + _executor = None + + if loop[0] is not None: + with _get_lock(): + # Stop the event loop safely + loop[0].call_soon_threadsafe(loop[0].stop) # Stop loop from another thread + if iothread[0] is not None: + iothread[0].join(timeout=0.2) # Add a timeout to avoid hanging + + if iothread[0].is_alive(): + logger.warning( + "Thread did not finish cleanly; forcefully closing the event loop." + ) + + # Forcefully close the event loop to release resources + loop[0].close() + + # dereference the loop and iothread + loop[0] = None + iothread[0] = None + + +atexit.register(cleanup_resources) + + +def reset_resources_after_fork() -> None: + """ + Ensure that global resources are reset after a fork. Without this function, + forked processes will retain invalid references to the parent process's resources. + """ + global loop, iothread, _executor + # These lines are excluded from coverage because this function only runs in a child process, + # which is not observed by the test coverage instrumentation. Despite the apparent lack of + # test coverage, this function should be adequately tested by any test that uses Zarr IO with + # multiprocessing. + loop[0] = None # pragma: no cover + iothread[0] = None # pragma: no cover + _executor = None # pragma: no cover + + +# this is only available on certain operating systems +if hasattr(os, "register_at_fork"): + os.register_at_fork(after_in_child=reset_resources_after_fork) + + +async def _runner(coro: Coroutine[Any, Any, T]) -> T | BaseException: + """ + Await a coroutine and return the result of running it. If awaiting the coroutine raises an + exception, the exception will be returned. + """ + try: + return await coro + except Exception as ex: + return ex + + +def sync( + coro: Coroutine[Any, Any, T], + loop: asyncio.AbstractEventLoop | None = None, + timeout: float | None = None, +) -> T: + """ + Make loop run coroutine until it returns. Runs in other thread + + Examples + -------- + >>> sync(async_function(), existing_loop) + """ + if loop is None: + # NB: if the loop is not running *yet*, it is OK to submit work + # and we will wait for it + loop = _get_loop() + if _executor is None and config.get("threading.max_workers", None) is not None: + # trigger executor creation and attach to loop + _ = _get_executor() + if not isinstance(loop, asyncio.AbstractEventLoop): + raise TypeError(f"loop cannot be of type {type(loop)}") + if loop.is_closed(): + raise RuntimeError("Loop is not running") + try: + loop0 = asyncio.events.get_running_loop() + if loop0 is loop: + raise SyncError("Calling sync() from within a running loop") + except RuntimeError: + pass + + future = asyncio.run_coroutine_threadsafe(_runner(coro), loop) + + finished, unfinished = wait([future], return_when=asyncio.ALL_COMPLETED, timeout=timeout) + if len(unfinished) > 0: + raise TimeoutError(f"Coroutine {coro} failed to finish within {timeout} s") + assert len(finished) == 1 + return_result = next(iter(finished)).result() + + if isinstance(return_result, BaseException): + raise return_result + else: + return return_result + + +def _get_loop() -> asyncio.AbstractEventLoop: + """Create or return the default fsspec IO loop + + The loop will be running on a separate thread. + """ + if loop[0] is None: + with _get_lock(): + # repeat the check just in case the loop got filled between the + # previous two calls from another thread + if loop[0] is None: + logger.debug("Creating Zarr event loop") + new_loop = asyncio.new_event_loop() + loop[0] = new_loop + iothread[0] = threading.Thread(target=new_loop.run_forever, name="zarr_io") + assert iothread[0] is not None + iothread[0].daemon = True + iothread[0].start() + assert loop[0] is not None + return loop[0] + + +async def _collect_aiterator(data: AsyncIterator[T]) -> tuple[T, ...]: + """ + Collect an entire async iterator into a tuple + """ + result = [x async for x in data] + return tuple(result) + + +def collect_aiterator(data: AsyncIterator[T]) -> tuple[T, ...]: + """ + Synchronously collect an entire async iterator into a tuple. + """ + return sync(_collect_aiterator(data)) + + +class SyncMixin: + def _sync(self, coroutine: Coroutine[Any, Any, T]) -> T: + # TODO: refactor this to to take *args and **kwargs and pass those to the method + # this should allow us to better type the sync wrapper + return sync( + coroutine, + timeout=config.get("async.timeout"), + ) + + def _sync_iter(self, async_iterator: AsyncIterator[T]) -> list[T]: + async def iter_to_list() -> list[T]: + return [item async for item in async_iterator] + + return self._sync(iter_to_list()) + + +async def _with_semaphore( + func: Callable[[], Awaitable[T]], semaphore: asyncio.Semaphore | None = None +) -> T: + """ + Await the result of invoking the no-argument-callable ``func`` within the context manager + provided by a Semaphore, if one is provided. Otherwise, just await the result of invoking + ``func``. + """ + if semaphore is None: + return await func() + async with semaphore: + return await func() diff --git a/src/zarr/core/sync_group.py b/src/zarr/core/sync_group.py new file mode 100644 index 0000000000..39d8a17992 --- /dev/null +++ b/src/zarr/core/sync_group.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from zarr.core.group import Group, GroupMetadata, _parse_async_node +from zarr.core.group import create_hierarchy as create_hierarchy_async +from zarr.core.group import create_nodes as create_nodes_async +from zarr.core.group import create_rooted_hierarchy as create_rooted_hierarchy_async +from zarr.core.group import get_node as get_node_async +from zarr.core.sync import _collect_aiterator, sync + +if TYPE_CHECKING: + from collections.abc import Iterator + + from zarr.abc.store import Store + from zarr.core.array import Array + from zarr.core.common import ZarrFormat + from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata + + +def create_nodes( + *, store: Store, nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata] +) -> Iterator[tuple[str, Group | Array]]: + """Create a collection of arrays and / or groups concurrently. + + Note: no attempt is made to validate that these arrays and / or groups collectively form a + valid Zarr hierarchy. It is the responsibility of the caller of this function to ensure that + the ``nodes`` parameter satisfies any correctness constraints. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes + in the hierarchy, and the values are the metadata of the nodes. The + metadata must be either an instance of GroupMetadata, ArrayV3Metadata + or ArrayV2Metadata. + + Yields + ------ + Group | Array + The created nodes. + """ + coro = create_nodes_async(store=store, nodes=nodes) + + for key, value in sync(_collect_aiterator(coro)): + yield key, _parse_async_node(value) + + +def create_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> Iterator[tuple[str, Group | Array]]: + """ + Create a complete zarr hierarchy from a collection of metadata objects. + + This function will parse its input to ensure that the hierarchy is complete. Any implicit groups + will be inserted as needed. For example, an input like + ```{'a/b': GroupMetadata}``` will be parsed to + ```{'': GroupMetadata, 'a': GroupMetadata, 'b': Groupmetadata}``` + + After input parsing, this function then creates all the nodes in the hierarchy concurrently. + + Arrays and Groups are yielded in the order they are created. This order is not stable and + should not be relied on. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes in the hierarchy, + relative to the root of the ``Store``. The root of the store can be specified with the empty + string ``''``. The values are instances of ``GroupMetadata`` or ``ArrayMetadata``. Note that + all values must have the same ``zarr_format`` -- it is an error to mix zarr versions in the + same hierarchy. + + Leading "/" characters from keys will be removed. + overwrite : bool + Whether to overwrite existing nodes. Defaults to ``False``, in which case an error is + raised instead of overwriting an existing array or group. + + This function will not erase an existing group unless that group is explicitly named in + ``nodes``. If ``nodes`` defines implicit groups, e.g. ``{`'a/b/c': GroupMetadata}``, and a + group already exists at path ``a``, then this function will leave the group at ``a`` as-is. + + Yields + ------ + tuple[str, Group | Array] + This function yields (path, node) pairs, in the order the nodes were created. + + Examples + -------- + >>> from zarr import create_hierarchy + >>> from zarr.storage import MemoryStore + >>> from zarr.core.group import GroupMetadata + + >>> store = MemoryStore() + >>> nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} + >>> nodes_created = dict(create_hierarchy(store=store, nodes=nodes)) + >>> print(nodes) + # {'a': GroupMetadata(attributes={'name': 'leaf'}, zarr_format=3, consolidated_metadata=None, node_type='group')} + """ + coro = create_hierarchy_async(store=store, nodes=nodes, overwrite=overwrite) + + for key, value in sync(_collect_aiterator(coro)): + yield key, _parse_async_node(value) + + +def create_rooted_hierarchy( + *, + store: Store, + nodes: dict[str, GroupMetadata | ArrayV2Metadata | ArrayV3Metadata], + overwrite: bool = False, +) -> Group | Array: + """ + Create a Zarr hierarchy with a root, and return the root node, which could be a ``Group`` + or ``Array`` instance. + + Parameters + ---------- + store : Store + The storage backend to use. + nodes : dict[str, GroupMetadata | ArrayV3Metadata | ArrayV2Metadata] + A dictionary defining the hierarchy. The keys are the paths of the nodes + in the hierarchy, and the values are the metadata of the nodes. The + metadata must be either an instance of GroupMetadata, ArrayV3Metadata + or ArrayV2Metadata. + overwrite : bool + Whether to overwrite existing nodes. Default is ``False``. + + Returns + ------- + Group | Array + """ + async_node = sync(create_rooted_hierarchy_async(store=store, nodes=nodes, overwrite=overwrite)) + return _parse_async_node(async_node) + + +def get_node(store: Store, path: str, zarr_format: ZarrFormat) -> Array | Group: + """ + Get an Array or Group from a path in a Store. + + Parameters + ---------- + store : Store + The store-like object to read from. + path : str + The path to the node to read. + zarr_format : {2, 3} + The zarr format of the node to read. + + Returns + ------- + Array | Group + """ + + return _parse_async_node(sync(get_node_async(store=store, path=path, zarr_format=zarr_format))) diff --git a/src/zarr/creation.py b/src/zarr/creation.py new file mode 100644 index 0000000000..8197c4950c --- /dev/null +++ b/src/zarr/creation.py @@ -0,0 +1,47 @@ +""" +Helpers for creating arrays. + +.. warning:: + + This sub-module is deprecated. All functions here are defined + in the top level zarr namespace instead. +""" + +import warnings + +from zarr.api.synchronous import ( + array, + create, + empty, + empty_like, + full, + full_like, + ones, + ones_like, + open_array, + open_like, + zeros, + zeros_like, +) + +__all__ = [ + "array", + "create", + "empty", + "empty_like", + "full", + "full_like", + "ones", + "ones_like", + "open_array", + "open_like", + "zeros", + "zeros_like", +] + +warnings.warn( + "zarr.creation is deprecated. " + "Import these functions from the top level zarr. namespace instead.", + DeprecationWarning, + stacklevel=2, +) diff --git a/src/zarr/dtype.py b/src/zarr/dtype.py new file mode 100644 index 0000000000..6e3789543b --- /dev/null +++ b/src/zarr/dtype.py @@ -0,0 +1,3 @@ +from zarr.core.dtype import ZDType, data_type_registry + +__all__ = ["ZDType", "data_type_registry"] diff --git a/src/zarr/errors.py b/src/zarr/errors.py new file mode 100644 index 0000000000..4f972a6703 --- /dev/null +++ b/src/zarr/errors.py @@ -0,0 +1,68 @@ +from typing import Any + +__all__ = [ + "BaseZarrError", + "ContainsArrayAndGroupError", + "ContainsArrayError", + "ContainsGroupError", + "GroupNotFoundError", + "MetadataValidationError", + "NodeTypeValidationError", +] + + +class BaseZarrError(ValueError): + """ + Base error which all zarr errors are sub-classed from. + """ + + _msg = "" + + def __init__(self, *args: Any) -> None: + super().__init__(self._msg.format(*args)) + + +class GroupNotFoundError(BaseZarrError, FileNotFoundError): + """ + Raised when a group isn't found at a certain path. + """ + + _msg = "No group found in store {!r} at path {!r}" + + +class ContainsGroupError(BaseZarrError): + """Raised when a group already exists at a certain path.""" + + _msg = "A group exists in store {!r} at path {!r}." + + +class ContainsArrayError(BaseZarrError): + """Raised when an array already exists at a certain path.""" + + _msg = "An array exists in store {!r} at path {!r}." + + +class ContainsArrayAndGroupError(BaseZarrError): + """Raised when both array and group metadata are found at the same path.""" + + _msg = ( + "Array and group metadata documents (.zarray and .zgroup) were both found in store " + "{!r} at path {!r}. " + "Only one of these files may be present in a given directory / prefix. " + "Remove the .zarray file, or the .zgroup file, or both." + ) + + +class MetadataValidationError(BaseZarrError): + """Raised when the Zarr metadata is invalid in some way""" + + _msg = "Invalid value for '{}'. Expected '{}'. Got '{}'." + + +class NodeTypeValidationError(MetadataValidationError): + """ + Specialized exception when the node_type of the metadata document is incorrect.. + + This can be raised when the value is invalid or unexpected given the context, + for example an 'array' node when we expected a 'group'. + """ diff --git a/MANIFEST.in b/src/zarr/py.typed similarity index 100% rename from MANIFEST.in rename to src/zarr/py.typed diff --git a/src/zarr/registry.py b/src/zarr/registry.py new file mode 100644 index 0000000000..eb345b24b1 --- /dev/null +++ b/src/zarr/registry.py @@ -0,0 +1,280 @@ +from __future__ import annotations + +import warnings +from collections import defaultdict +from importlib.metadata import entry_points as get_entry_points +from typing import TYPE_CHECKING, Any, Generic, TypeVar + +from zarr.core.config import BadConfigError, config +from zarr.core.dtype import data_type_registry + +if TYPE_CHECKING: + from importlib.metadata import EntryPoint + + from zarr.abc.codec import ( + ArrayArrayCodec, + ArrayBytesCodec, + BytesBytesCodec, + Codec, + CodecPipeline, + ) + from zarr.core.buffer import Buffer, NDBuffer + from zarr.core.common import JSON + +__all__ = [ + "Registry", + "get_buffer_class", + "get_codec_class", + "get_ndbuffer_class", + "get_pipeline_class", + "register_buffer", + "register_codec", + "register_ndbuffer", + "register_pipeline", +] + +T = TypeVar("T") + + +class Registry(dict[str, type[T]], Generic[T]): + def __init__(self) -> None: + super().__init__() + self.lazy_load_list: list[EntryPoint] = [] + + def lazy_load(self) -> None: + for e in self.lazy_load_list: + self.register(e.load()) + + self.lazy_load_list.clear() + + def register(self, cls: type[T], qualname: str | None = None) -> None: + if qualname is None: + qualname = fully_qualified_name(cls) + self[qualname] = cls + + +__codec_registries: dict[str, Registry[Codec]] = defaultdict(Registry) +__pipeline_registry: Registry[CodecPipeline] = Registry() +__buffer_registry: Registry[Buffer] = Registry() +__ndbuffer_registry: Registry[NDBuffer] = Registry() + +""" +The registry module is responsible for managing implementations of codecs, +pipelines, buffers and ndbuffers and collecting them from entrypoints. +The implementation used is determined by the config. + +The registry module is also responsible for managing dtypes. +""" + + +def _collect_entrypoints() -> list[Registry[Any]]: + """ + Collects codecs, pipelines, dtypes, buffers and ndbuffers from entrypoints. + Entry points can either be single items or groups of items. + Allowed syntax for entry_points.txt is e.g. + + [zarr.codecs] + gzip = package:EntrypointGzipCodec1 + [zarr.codecs.gzip] + some_name = package:EntrypointGzipCodec2 + another = package:EntrypointGzipCodec3 + + [zarr] + buffer = package:TestBuffer1 + [zarr.buffer] + xyz = package:TestBuffer2 + abc = package:TestBuffer3 + ... + """ + entry_points = get_entry_points() + + __buffer_registry.lazy_load_list.extend(entry_points.select(group="zarr.buffer")) + __buffer_registry.lazy_load_list.extend(entry_points.select(group="zarr", name="buffer")) + __ndbuffer_registry.lazy_load_list.extend(entry_points.select(group="zarr.ndbuffer")) + __ndbuffer_registry.lazy_load_list.extend(entry_points.select(group="zarr", name="ndbuffer")) + + data_type_registry.lazy_load_list.extend(entry_points.select(group="zarr.data_type")) + data_type_registry.lazy_load_list.extend(entry_points.select(group="zarr", name="data_type")) + + __pipeline_registry.lazy_load_list.extend(entry_points.select(group="zarr.codec_pipeline")) + __pipeline_registry.lazy_load_list.extend( + entry_points.select(group="zarr", name="codec_pipeline") + ) + for e in entry_points.select(group="zarr.codecs"): + __codec_registries[e.name].lazy_load_list.append(e) + for group in entry_points.groups: + if group.startswith("zarr.codecs."): + codec_name = group.split(".")[2] + __codec_registries[codec_name].lazy_load_list.extend(entry_points.select(group=group)) + return [ + *__codec_registries.values(), + __pipeline_registry, + __buffer_registry, + __ndbuffer_registry, + ] + + +def _reload_config() -> None: + config.refresh() + + +def fully_qualified_name(cls: type) -> str: + module = cls.__module__ + return module + "." + cls.__qualname__ + + +def register_codec(key: str, codec_cls: type[Codec]) -> None: + if key not in __codec_registries: + __codec_registries[key] = Registry() + __codec_registries[key].register(codec_cls) + + +def register_pipeline(pipe_cls: type[CodecPipeline]) -> None: + __pipeline_registry.register(pipe_cls) + + +def register_ndbuffer(cls: type[NDBuffer], qualname: str | None = None) -> None: + __ndbuffer_registry.register(cls, qualname) + + +def register_buffer(cls: type[Buffer], qualname: str | None = None) -> None: + __buffer_registry.register(cls, qualname) + + +def get_codec_class(key: str, reload_config: bool = False) -> type[Codec]: + if reload_config: + _reload_config() + + if key in __codec_registries: + # logger.debug("Auto loading codec '%s' from entrypoint", codec_id) + __codec_registries[key].lazy_load() + + codec_classes = __codec_registries[key] + if not codec_classes: + raise KeyError(key) + + config_entry = config.get("codecs", {}).get(key) + if config_entry is None: + if len(codec_classes) == 1: + return next(iter(codec_classes.values())) + warnings.warn( + f"Codec '{key}' not configured in config. Selecting any implementation.", + stacklevel=2, + ) + return list(codec_classes.values())[-1] + selected_codec_cls = codec_classes[config_entry] + + if selected_codec_cls: + return selected_codec_cls + raise KeyError(key) + + +def _resolve_codec(data: dict[str, JSON]) -> Codec: + """ + Get a codec instance from a dict representation of that codec. + """ + # TODO: narrow the type of the input to only those dicts that map on to codec class instances. + return get_codec_class(data["name"]).from_dict(data) # type: ignore[arg-type] + + +def _parse_bytes_bytes_codec(data: dict[str, JSON] | Codec) -> BytesBytesCodec: + """ + Normalize the input to a ``BytesBytesCodec`` instance. + If the input is already a ``BytesBytesCodec``, it is returned as is. If the input is a dict, it + is converted to a ``BytesBytesCodec`` instance via the ``_resolve_codec`` function. + """ + from zarr.abc.codec import BytesBytesCodec + + if isinstance(data, dict): + result = _resolve_codec(data) + if not isinstance(result, BytesBytesCodec): + msg = f"Expected a dict representation of a BytesBytesCodec; got a dict representation of a {type(result)} instead." + raise TypeError(msg) + else: + if not isinstance(data, BytesBytesCodec): + raise TypeError(f"Expected a BytesBytesCodec. Got {type(data)} instead.") + result = data + return result + + +def _parse_array_bytes_codec(data: dict[str, JSON] | Codec) -> ArrayBytesCodec: + """ + Normalize the input to a ``ArrayBytesCodec`` instance. + If the input is already a ``ArrayBytesCodec``, it is returned as is. If the input is a dict, it + is converted to a ``ArrayBytesCodec`` instance via the ``_resolve_codec`` function. + """ + from zarr.abc.codec import ArrayBytesCodec + + if isinstance(data, dict): + result = _resolve_codec(data) + if not isinstance(result, ArrayBytesCodec): + msg = f"Expected a dict representation of a ArrayBytesCodec; got a dict representation of a {type(result)} instead." + raise TypeError(msg) + else: + if not isinstance(data, ArrayBytesCodec): + raise TypeError(f"Expected a ArrayBytesCodec. Got {type(data)} instead.") + result = data + return result + + +def _parse_array_array_codec(data: dict[str, JSON] | Codec) -> ArrayArrayCodec: + """ + Normalize the input to a ``ArrayArrayCodec`` instance. + If the input is already a ``ArrayArrayCodec``, it is returned as is. If the input is a dict, it + is converted to a ``ArrayArrayCodec`` instance via the ``_resolve_codec`` function. + """ + from zarr.abc.codec import ArrayArrayCodec + + if isinstance(data, dict): + result = _resolve_codec(data) + if not isinstance(result, ArrayArrayCodec): + msg = f"Expected a dict representation of a ArrayArrayCodec; got a dict representation of a {type(result)} instead." + raise TypeError(msg) + else: + if not isinstance(data, ArrayArrayCodec): + raise TypeError(f"Expected a ArrayArrayCodec. Got {type(data)} instead.") + result = data + return result + + +def get_pipeline_class(reload_config: bool = False) -> type[CodecPipeline]: + if reload_config: + _reload_config() + __pipeline_registry.lazy_load() + path = config.get("codec_pipeline.path") + pipeline_class = __pipeline_registry.get(path) + if pipeline_class: + return pipeline_class + raise BadConfigError( + f"Pipeline class '{path}' not found in registered pipelines: {list(__pipeline_registry)}." + ) + + +def get_buffer_class(reload_config: bool = False) -> type[Buffer]: + if reload_config: + _reload_config() + __buffer_registry.lazy_load() + + path = config.get("buffer") + buffer_class = __buffer_registry.get(path) + if buffer_class: + return buffer_class + raise BadConfigError( + f"Buffer class '{path}' not found in registered buffers: {list(__buffer_registry)}." + ) + + +def get_ndbuffer_class(reload_config: bool = False) -> type[NDBuffer]: + if reload_config: + _reload_config() + __ndbuffer_registry.lazy_load() + path = config.get("ndbuffer") + ndbuffer_class = __ndbuffer_registry.get(path) + if ndbuffer_class: + return ndbuffer_class + raise BadConfigError( + f"NDBuffer class '{path}' not found in registered buffers: {list(__ndbuffer_registry)}." + ) + + +_collect_entrypoints() diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py new file mode 100644 index 0000000000..6721139375 --- /dev/null +++ b/src/zarr/storage/__init__.py @@ -0,0 +1,43 @@ +import sys +import warnings +from types import ModuleType +from typing import Any + +from zarr.storage._common import StoreLike, StorePath +from zarr.storage._fsspec import FsspecStore +from zarr.storage._local import LocalStore +from zarr.storage._logging import LoggingStore +from zarr.storage._memory import GpuMemoryStore, MemoryStore +from zarr.storage._obstore import ObjectStore +from zarr.storage._wrapper import WrapperStore +from zarr.storage._zip import ZipStore + +__all__ = [ + "FsspecStore", + "GpuMemoryStore", + "LocalStore", + "LoggingStore", + "MemoryStore", + "ObjectStore", + "StoreLike", + "StorePath", + "WrapperStore", + "ZipStore", +] + + +class VerboseModule(ModuleType): + def __setattr__(self, attr: str, value: Any) -> None: + if attr == "default_compressor": + warnings.warn( + "setting zarr.storage.default_compressor is deprecated, use " + "zarr.config to configure array.v2_default_compressor " + "e.g. config.set({'codecs.zstd':'numcodecs.Zstd', 'array.v2_default_compressor.numeric': 'zstd'})", + DeprecationWarning, + stacklevel=1, + ) + else: + super().__setattr__(attr, value) + + +sys.modules[__name__].__class__ = VerboseModule diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py new file mode 100644 index 0000000000..f264728cf2 --- /dev/null +++ b/src/zarr/storage/_common.py @@ -0,0 +1,526 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal, Self, TypeAlias + +from zarr.abc.store import ByteRequest, Store +from zarr.core.buffer import Buffer, default_buffer_prototype +from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZGROUP_JSON, AccessModeLiteral, ZarrFormat +from zarr.errors import ContainsArrayAndGroupError, ContainsArrayError, ContainsGroupError +from zarr.storage._local import LocalStore +from zarr.storage._memory import MemoryStore +from zarr.storage._utils import normalize_path + +_has_fsspec = importlib.util.find_spec("fsspec") +if _has_fsspec: + from fsspec.mapping import FSMap +else: + FSMap = None + +if TYPE_CHECKING: + from zarr.core.buffer import BufferPrototype + + +def _dereference_path(root: str, path: str) -> str: + assert isinstance(root, str) + assert isinstance(path, str) + root = root.rstrip("/") + path = f"{root}/{path}" if root else path + return path.rstrip("/") + + +class StorePath: + """ + Path-like interface for a Store. + + Parameters + ---------- + store : Store + The store to use. + path : str + The path within the store. + """ + + store: Store + path: str + + def __init__(self, store: Store, path: str = "") -> None: + self.store = store + self.path = normalize_path(path) + + @property + def read_only(self) -> bool: + return self.store.read_only + + @classmethod + async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = None) -> Self: + """ + Open StorePath based on the provided mode. + + * If the mode is 'w-' and the StorePath contains keys, raise a FileExistsError. + * If the mode is 'w', delete all keys nested within the StorePath + * If the mode is 'a', 'r', or 'r+', do nothing + + Parameters + ---------- + mode : AccessModeLiteral + The mode to use when initializing the store path. + + Raises + ------ + FileExistsError + If the mode is 'w-' and the store path already exists. + ValueError + If the mode is not "r" and the store is read-only, or + if the mode is "r" and the store is not read-only. + """ + + await store._ensure_open() + self = cls(store, path) + + # fastpath if mode is None + if mode is None: + return self + + if store.read_only and mode != "r": + raise ValueError(f"Store is read-only but mode is '{mode}'") + if not store.read_only and mode == "r": + raise ValueError(f"Store is not read-only but mode is '{mode}'") + + match mode: + case "w-": + if not await self.is_empty(): + msg = ( + f"{self} is not empty, but `mode` is set to 'w-'." + "Either remove the existing objects in storage," + "or set `mode` to a value that handles pre-existing objects" + "in storage, like `a` or `w`." + ) + raise FileExistsError(msg) + case "w": + await self.delete_dir() + case "a" | "r" | "r+": + # No init action + pass + case _: + raise ValueError(f"Invalid mode: {mode}") + + return self + + async def get( + self, + prototype: BufferPrototype | None = None, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + """ + Read bytes from the store. + + Parameters + ---------- + prototype : BufferPrototype, optional + The buffer prototype to use when reading the bytes. + byte_range : ByteRequest, optional + The range of bytes to read. + + Returns + ------- + buffer : Buffer or None + The read bytes, or None if the key does not exist. + """ + if prototype is None: + prototype = default_buffer_prototype() + return await self.store.get(self.path, prototype=prototype, byte_range=byte_range) + + async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None: + """ + Write bytes to the store. + + Parameters + ---------- + value : Buffer + The buffer to write. + byte_range : ByteRequest, optional + The range of bytes to write. If None, the entire buffer is written. + + Raises + ------ + NotImplementedError + If `byte_range` is not None, because Store.set does not support partial writes yet. + """ + if byte_range is not None: + raise NotImplementedError("Store.set does not have partial writes yet") + await self.store.set(self.path, value) + + async def delete(self) -> None: + """ + Delete the key from the store. + + Raises + ------ + NotImplementedError + If the store does not support deletion. + """ + await self.store.delete(self.path) + + async def delete_dir(self) -> None: + """ + Delete all keys with the given prefix from the store. + """ + await self.store.delete_dir(self.path) + + async def set_if_not_exists(self, default: Buffer) -> None: + """ + Store a key to ``value`` if the key is not already present. + + Parameters + ---------- + default : Buffer + The buffer to store if the key is not already present. + """ + await self.store.set_if_not_exists(self.path, default) + + async def exists(self) -> bool: + """ + Check if the key exists in the store. + + Returns + ------- + bool + True if the key exists in the store, False otherwise. + """ + return await self.store.exists(self.path) + + async def is_empty(self) -> bool: + """ + Check if any keys exist in the store with the given prefix. + + Returns + ------- + bool + True if no keys exist in the store with the given prefix, False otherwise. + """ + return await self.store.is_empty(self.path) + + def __truediv__(self, other: str) -> StorePath: + """Combine this store path with another path""" + return self.__class__(self.store, _dereference_path(self.path, other)) + + def __str__(self) -> str: + return _dereference_path(str(self.store), self.path) + + def __repr__(self) -> str: + return f"StorePath({self.store.__class__.__name__}, '{self}')" + + def __eq__(self, other: object) -> bool: + """ + Check if two StorePath objects are equal. + + Returns + ------- + bool + True if the two objects are equal, False otherwise. + + Notes + ----- + Two StorePath objects are considered equal if their stores are equal + and their paths are equal. + """ + try: + return self.store == other.store and self.path == other.path # type: ignore[attr-defined, no-any-return] + except Exception: + pass + return False + + +StoreLike: TypeAlias = Store | StorePath | FSMap | Path | str | dict[str, Buffer] + + +async def make_store_path( + store_like: StoreLike | None, + *, + path: str | None = "", + mode: AccessModeLiteral | None = None, + storage_options: dict[str, Any] | None = None, +) -> StorePath: + """ + Convert a `StoreLike` object into a StorePath object. + + This function takes a `StoreLike` object and returns a `StorePath` object. The + `StoreLike` object can be a `Store`, `StorePath`, `Path`, `str`, or `dict[str, Buffer]`. + If the `StoreLike` object is a Store or `StorePath`, it is converted to a + `StorePath` object. If the `StoreLike` object is a Path or str, it is converted + to a LocalStore object and then to a `StorePath` object. If the `StoreLike` + object is a dict[str, Buffer], it is converted to a `MemoryStore` object and + then to a `StorePath` object. + + If the `StoreLike` object is None, a `MemoryStore` object is created and + converted to a `StorePath` object. + + If the `StoreLike` object is a str and starts with a protocol, it is + converted to a RemoteStore object and then to a `StorePath` object. + + If the `StoreLike` object is a dict[str, Buffer] and the mode is not None, + the `MemoryStore` object is created with the given mode. + + If the `StoreLike` object is a str and starts with a protocol, the + RemoteStore object is created with the given mode and storage options. + + Parameters + ---------- + store_like : StoreLike | None + The object to convert to a `StorePath` object. + path : str | None, optional + The path to use when creating the `StorePath` object. If None, the + default path is the empty string. + mode : StoreAccessMode | None, optional + The mode to use when creating the `StorePath` object. If None, the + default mode is 'r'. + storage_options : dict[str, Any] | None, optional + The storage options to use when creating the `RemoteStore` object. If + None, the default storage options are used. + + Returns + ------- + StorePath + The converted StorePath object. + + Raises + ------ + TypeError + If the StoreLike object is not one of the supported types. + """ + from zarr.storage._fsspec import FsspecStore # circular import + + used_storage_options = False + path_normalized = normalize_path(path) + if isinstance(store_like, StorePath): + result = store_like / path_normalized + else: + assert mode in (None, "r", "r+", "a", "w", "w-") + # if mode 'r' was provided, we'll open any new stores as read-only + _read_only = mode == "r" + if isinstance(store_like, Store): + store = store_like + elif store_like is None: + store = await MemoryStore.open(read_only=_read_only) + elif isinstance(store_like, Path): + store = await LocalStore.open(root=store_like, read_only=_read_only) + elif isinstance(store_like, str): + storage_options = storage_options or {} + + if _is_fsspec_uri(store_like): + used_storage_options = True + store = FsspecStore.from_url( + store_like, storage_options=storage_options, read_only=_read_only + ) + else: + store = await LocalStore.open(root=Path(store_like), read_only=_read_only) + elif isinstance(store_like, dict): + # We deliberate only consider dict[str, Buffer] here, and not arbitrary mutable mappings. + # By only allowing dictionaries, which are in-memory, we know that MemoryStore appropriate. + store = await MemoryStore.open(store_dict=store_like, read_only=_read_only) + elif _has_fsspec and isinstance(store_like, FSMap): + if path: + raise ValueError( + "'path' was provided but is not used for FSMap store_like objects. Specify the path when creating the FSMap instance instead." + ) + if storage_options: + raise ValueError( + "'storage_options was provided but is not used for FSMap store_like objects. Specify the storage options when creating the FSMap instance instead." + ) + store = FsspecStore.from_mapper(store_like, read_only=_read_only) + else: + raise TypeError(f"Unsupported type for store_like: '{type(store_like).__name__}'") + + result = await StorePath.open(store, path=path_normalized, mode=mode) + + if storage_options and not used_storage_options: + msg = "'storage_options' was provided but unused. 'storage_options' is only used for fsspec filesystem stores." + raise TypeError(msg) + + return result + + +def _is_fsspec_uri(uri: str) -> bool: + """ + Check if a URI looks like a non-local fsspec URI. + + Examples + -------- + >>> _is_fsspec_uri("s3://bucket") + True + >>> _is_fsspec_uri("my-directory") + False + >>> _is_fsspec_uri("local://my-directory") + False + """ + return "://" in uri or ("::" in uri and "local://" not in uri) + + +async def ensure_no_existing_node(store_path: StorePath, zarr_format: ZarrFormat) -> None: + """ + Check if a store_path is safe for array / group creation. + Returns `None` or raises an exception. + + Parameters + ---------- + store_path : StorePath + The storage location to check. + zarr_format : ZarrFormat + The Zarr format to check. + + Raises + ------ + ContainsArrayError, ContainsGroupError, ContainsArrayAndGroupError + """ + if zarr_format == 2: + extant_node = await _contains_node_v2(store_path) + elif zarr_format == 3: + extant_node = await _contains_node_v3(store_path) + + if extant_node == "array": + raise ContainsArrayError(store_path.store, store_path.path) + elif extant_node == "group": + raise ContainsGroupError(store_path.store, store_path.path) + elif extant_node == "nothing": + return + msg = f"Invalid value for extant_node: {extant_node}" # type: ignore[unreachable] + raise ValueError(msg) + + +async def _contains_node_v3(store_path: StorePath) -> Literal["array", "group", "nothing"]: + """ + Check if a store_path contains nothing, an array, or a group. This function + returns the string "array", "group", or "nothing" to denote containing an array, a group, or + nothing. + + Parameters + ---------- + store_path : StorePath + The location in storage to check. + + Returns + ------- + Literal["array", "group", "nothing"] + A string representing the zarr node found at store_path. + """ + result: Literal["array", "group", "nothing"] = "nothing" + extant_meta_bytes = await (store_path / ZARR_JSON).get() + # if no metadata document could be loaded, then we just return "nothing" + if extant_meta_bytes is not None: + try: + extant_meta_json = json.loads(extant_meta_bytes.to_bytes()) + # avoid constructing a full metadata document here in the name of speed. + if extant_meta_json["node_type"] == "array": + result = "array" + elif extant_meta_json["node_type"] == "group": + result = "group" + except (KeyError, json.JSONDecodeError): + # either of these errors is consistent with no array or group present. + pass + return result + + +async def _contains_node_v2(store_path: StorePath) -> Literal["array", "group", "nothing"]: + """ + Check if a store_path contains nothing, an array, a group, or both. If both an array and a + group are detected, a `ContainsArrayAndGroup` exception is raised. Otherwise, this function + returns the string "array", "group", or "nothing" to denote containing an array, a group, or + nothing. + + Parameters + ---------- + store_path : StorePath + The location in storage to check. + + Returns + ------- + Literal["array", "group", "nothing"] + A string representing the zarr node found at store_path. + """ + _array = await contains_array(store_path=store_path, zarr_format=2) + _group = await contains_group(store_path=store_path, zarr_format=2) + + if _array and _group: + raise ContainsArrayAndGroupError(store_path.store, store_path.path) + elif _array: + return "array" + elif _group: + return "group" + else: + return "nothing" + + +async def contains_array(store_path: StorePath, zarr_format: ZarrFormat) -> bool: + """ + Check if an array exists at a given StorePath. + + Parameters + ---------- + store_path : StorePath + The StorePath to check for an existing group. + zarr_format : + The zarr format to check for. + + Returns + ------- + bool + True if the StorePath contains a group, False otherwise. + + """ + if zarr_format == 3: + extant_meta_bytes = await (store_path / ZARR_JSON).get() + if extant_meta_bytes is None: + return False + else: + try: + extant_meta_json = json.loads(extant_meta_bytes.to_bytes()) + # we avoid constructing a full metadata document here in the name of speed. + if extant_meta_json["node_type"] == "array": + return True + except (ValueError, KeyError): + return False + elif zarr_format == 2: + return await (store_path / ZARRAY_JSON).exists() + msg = f"Invalid zarr_format provided. Got {zarr_format}, expected 2 or 3" + raise ValueError(msg) + + +async def contains_group(store_path: StorePath, zarr_format: ZarrFormat) -> bool: + """ + Check if a group exists at a given StorePath. + + Parameters + ---------- + + store_path : StorePath + The StorePath to check for an existing group. + zarr_format : + The zarr format to check for. + + Returns + ------- + + bool + True if the StorePath contains a group, False otherwise + + """ + if zarr_format == 3: + extant_meta_bytes = await (store_path / ZARR_JSON).get() + if extant_meta_bytes is None: + return False + else: + try: + extant_meta_json = json.loads(extant_meta_bytes.to_bytes()) + # we avoid constructing a full metadata document here in the name of speed. + result: bool = extant_meta_json["node_type"] == "group" + except (ValueError, KeyError): + return False + else: + return result + elif zarr_format == 2: + return await (store_path / ZGROUP_JSON).exists() + msg = f"Invalid zarr_format provided. Got {zarr_format}, expected 2 or 3" # type: ignore[unreachable] + raise ValueError(msg) diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py new file mode 100644 index 0000000000..4f6929456e --- /dev/null +++ b/src/zarr/storage/_fsspec.py @@ -0,0 +1,463 @@ +from __future__ import annotations + +import json +import warnings +from contextlib import suppress +from typing import TYPE_CHECKING, Any + +from packaging.version import parse as parse_version + +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.core.buffer import Buffer +from zarr.storage._common import _dereference_path + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterable + + from fsspec import AbstractFileSystem + from fsspec.asyn import AsyncFileSystem + from fsspec.mapping import FSMap + + from zarr.core.buffer import BufferPrototype + from zarr.core.common import BytesLike + + +ALLOWED_EXCEPTIONS: tuple[type[Exception], ...] = ( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, +) + + +def _make_async(fs: AbstractFileSystem) -> AsyncFileSystem: + """Convert a sync FSSpec filesystem to an async FFSpec filesystem + + If the filesystem class supports async operations, a new async instance is created + from the existing instance. + + If the filesystem class does not support async operations, the existing instance + is wrapped with AsyncFileSystemWrapper. + """ + import fsspec + + fsspec_version = parse_version(fsspec.__version__) + if fs.async_impl and fs.asynchronous: + # Already an async instance of an async filesystem, nothing to do + return fs + if fs.async_impl: + # Convert sync instance of an async fs to an async instance + fs_dict = json.loads(fs.to_json()) + fs_dict["asynchronous"] = True + return fsspec.AbstractFileSystem.from_json(json.dumps(fs_dict)) + + # Wrap sync filesystems with the async wrapper + if type(fs) is fsspec.implementations.local.LocalFileSystem and not fs.auto_mkdir: + raise ValueError( + f"LocalFilesystem {fs} was created with auto_mkdir=False but Zarr requires the filesystem to automatically create directories" + ) + if fsspec_version < parse_version("2024.12.0"): + raise ImportError( + f"The filesystem '{fs}' is synchronous, and the required " + "AsyncFileSystemWrapper is not available. Upgrade fsspec to version " + "2024.12.0 or later to enable this functionality." + ) + + return fsspec.implementations.asyn_wrapper.AsyncFileSystemWrapper(fs, asynchronous=True) + + +class FsspecStore(Store): + """ + Store for remote data based on FSSpec. + + Parameters + ---------- + fs : AsyncFileSystem + The Async FSSpec filesystem to use with this store. + read_only : bool + Whether the store is read-only + path : str + The root path of the store. This should be a relative path and must not include the + filesystem scheme. + allowed_exceptions : tuple[type[Exception], ...] + When fetching data, these cases will be deemed to correspond to missing keys. + + Attributes + ---------- + fs + allowed_exceptions + supports_writes + supports_deletes + supports_partial_writes + supports_listing + + Raises + ------ + TypeError + If the Filesystem does not support async operations. + ValueError + If the path argument includes a scheme. + + Warns + ----- + UserWarning + If the file system (fs) was not created with `asynchronous=True`. + + See Also + -------- + FsspecStore.from_upath + FsspecStore.from_url + """ + + # based on FSSpec + supports_writes: bool = True + supports_deletes: bool = True + supports_partial_writes: bool = False + supports_listing: bool = True + + fs: AsyncFileSystem + allowed_exceptions: tuple[type[Exception], ...] + path: str + + def __init__( + self, + fs: AsyncFileSystem, + read_only: bool = False, + path: str = "/", + allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, + ) -> None: + super().__init__(read_only=read_only) + self.fs = fs + self.path = path + self.allowed_exceptions = allowed_exceptions + + if not self.fs.async_impl: + raise TypeError("Filesystem needs to support async operations.") + if not self.fs.asynchronous: + warnings.warn( + f"fs ({fs}) was not created with `asynchronous=True`, this may lead to surprising behavior", + stacklevel=2, + ) + if "://" in path and not path.startswith("http"): + # `not path.startswith("http")` is a special case for the http filesystem (¯\_(ツ)_/¯) + scheme, _ = path.split("://", maxsplit=1) + raise ValueError(f"path argument to FsspecStore must not include scheme ({scheme}://)") + + @classmethod + def from_upath( + cls, + upath: Any, + read_only: bool = False, + allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, + ) -> FsspecStore: + """ + Create a FsspecStore from an upath object. + + Parameters + ---------- + upath : UPath + The upath to the root of the store. + read_only : bool + Whether the store is read-only, defaults to False. + allowed_exceptions : tuple, optional + The exceptions that are allowed to be raised when accessing the + store. Defaults to ALLOWED_EXCEPTIONS. + + Returns + ------- + FsspecStore + """ + return cls( + fs=upath.fs, + path=upath.path.rstrip("/"), + read_only=read_only, + allowed_exceptions=allowed_exceptions, + ) + + @classmethod + def from_mapper( + cls, + fs_map: FSMap, + read_only: bool = False, + allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, + ) -> FsspecStore: + """ + Create a FsspecStore from a FSMap object. + + Parameters + ---------- + fs_map : FSMap + Fsspec mutable mapping object. + read_only : bool + Whether the store is read-only, defaults to False. + allowed_exceptions : tuple, optional + The exceptions that are allowed to be raised when accessing the + store. Defaults to ALLOWED_EXCEPTIONS. + + Returns + ------- + FsspecStore + """ + fs = _make_async(fs_map.fs) + return cls( + fs=fs, + path=fs_map.root, + read_only=read_only, + allowed_exceptions=allowed_exceptions, + ) + + @classmethod + def from_url( + cls, + url: str, + storage_options: dict[str, Any] | None = None, + read_only: bool = False, + allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, + ) -> FsspecStore: + """ + Create a FsspecStore from a URL. + + Parameters + ---------- + url : str + The URL to the root of the store. + storage_options : dict, optional + The options to pass to fsspec when creating the filesystem. + read_only : bool + Whether the store is read-only, defaults to False. + allowed_exceptions : tuple, optional + The exceptions that are allowed to be raised when accessing the + store. Defaults to ALLOWED_EXCEPTIONS. + + Returns + ------- + FsspecStore + """ + try: + from fsspec import url_to_fs + except ImportError: + # before fsspec==2024.3.1 + from fsspec.core import url_to_fs + + opts = storage_options or {} + opts = {"asynchronous": True, **opts} + + fs, path = url_to_fs(url, **opts) + if not fs.async_impl: + fs = _make_async(fs) + + # fsspec is not consistent about removing the scheme from the path, so check and strip it here + # https://github.com/fsspec/filesystem_spec/issues/1722 + if "://" in path and not path.startswith("http"): + # `not path.startswith("http")` is a special case for the http filesystem (¯\_(ツ)_/¯) + path = fs._strip_protocol(path) + + return cls(fs=fs, path=path, read_only=read_only, allowed_exceptions=allowed_exceptions) + + def with_read_only(self, read_only: bool = False) -> FsspecStore: + # docstring inherited + return type(self)( + fs=self.fs, + path=self.path, + allowed_exceptions=self.allowed_exceptions, + read_only=read_only, + ) + + async def clear(self) -> None: + # docstring inherited + try: + for subpath in await self.fs._find(self.path, withdirs=True): + if subpath != self.path: + await self.fs._rm(subpath, recursive=True) + except FileNotFoundError: + pass + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, type(self)) + and self.path == other.path + and self.read_only == other.read_only + and self.fs == other.fs + ) + + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + # docstring inherited + if not self._is_open: + await self._open() + path = _dereference_path(self.path, key) + + try: + if byte_range is None: + value = prototype.buffer.from_bytes(await self.fs._cat_file(path)) + elif isinstance(byte_range, RangeByteRequest): + value = prototype.buffer.from_bytes( + await self.fs._cat_file( + path, + start=byte_range.start, + end=byte_range.end, + ) + ) + elif isinstance(byte_range, OffsetByteRequest): + value = prototype.buffer.from_bytes( + await self.fs._cat_file(path, start=byte_range.offset, end=None) + ) + elif isinstance(byte_range, SuffixByteRequest): + value = prototype.buffer.from_bytes( + await self.fs._cat_file(path, start=-byte_range.suffix, end=None) + ) + else: + raise ValueError(f"Unexpected byte_range, got {byte_range}.") + except self.allowed_exceptions: + return None + except OSError as e: + if "not satisfiable" in str(e): + # this is an s3-specific condition we probably don't want to leak + return prototype.buffer.from_bytes(b"") + raise + else: + return value + + async def set( + self, + key: str, + value: Buffer, + byte_range: tuple[int, int] | None = None, + ) -> None: + # docstring inherited + if not self._is_open: + await self._open() + self._check_writable() + if not isinstance(value, Buffer): + raise TypeError( + f"FsspecStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) + path = _dereference_path(self.path, key) + # write data + if byte_range: + raise NotImplementedError + await self.fs._pipe_file(path, value.to_bytes()) + + async def delete(self, key: str) -> None: + # docstring inherited + self._check_writable() + path = _dereference_path(self.path, key) + try: + await self.fs._rm(path) + except FileNotFoundError: + pass + except self.allowed_exceptions: + pass + + async def delete_dir(self, prefix: str) -> None: + # docstring inherited + if not self.supports_deletes: + raise NotImplementedError( + "This method is only available for stores that support deletes." + ) + self._check_writable() + + path_to_delete = _dereference_path(self.path, prefix) + + with suppress(*self.allowed_exceptions): + await self.fs._rm(path_to_delete, recursive=True) + + async def exists(self, key: str) -> bool: + # docstring inherited + path = _dereference_path(self.path, key) + exists: bool = await self.fs._exists(path) + return exists + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + # docstring inherited + if key_ranges: + # _cat_ranges expects a list of paths, start, and end ranges, so we need to reformat each ByteRequest. + key_ranges = list(key_ranges) + paths: list[str] = [] + starts: list[int | None] = [] + stops: list[int | None] = [] + for key, byte_range in key_ranges: + paths.append(_dereference_path(self.path, key)) + if byte_range is None: + starts.append(None) + stops.append(None) + elif isinstance(byte_range, RangeByteRequest): + starts.append(byte_range.start) + stops.append(byte_range.end) + elif isinstance(byte_range, OffsetByteRequest): + starts.append(byte_range.offset) + stops.append(None) + elif isinstance(byte_range, SuffixByteRequest): + starts.append(-byte_range.suffix) + stops.append(None) + else: + raise ValueError(f"Unexpected byte_range, got {byte_range}.") + else: + return [] + # TODO: expectations for exceptions or missing keys? + res = await self.fs._cat_ranges(paths, starts, stops, on_error="return") + # the following is an s3-specific condition we probably don't want to leak + res = [b"" if (isinstance(r, OSError) and "not satisfiable" in str(r)) else r for r in res] + for r in res: + if isinstance(r, Exception) and not isinstance(r, self.allowed_exceptions): + raise r + + return [None if isinstance(r, Exception) else prototype.buffer.from_bytes(r) for r in res] + + async def set_partial_values( + self, key_start_values: Iterable[tuple[str, int, BytesLike]] + ) -> None: + # docstring inherited + raise NotImplementedError + + async def list(self) -> AsyncIterator[str]: + # docstring inherited + allfiles = await self.fs._find(self.path, detail=False, withdirs=False) + for onefile in (a.removeprefix(self.path + "/") for a in allfiles): + yield onefile + + async def list_dir(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + prefix = f"{self.path}/{prefix.rstrip('/')}" + try: + allfiles = await self.fs._ls(prefix, detail=False) + except FileNotFoundError: + return + for onefile in (a.replace(prefix + "/", "") for a in allfiles): + yield onefile.removeprefix(self.path).removeprefix("/") + + async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + for onefile in await self.fs._find( + f"{self.path}/{prefix}", detail=False, maxdepth=None, withdirs=False + ): + yield onefile.removeprefix(f"{self.path}/") + + async def getsize(self, key: str) -> int: + path = _dereference_path(self.path, key) + info = await self.fs._info(path) + + size = info.get("size") + + if size is None: + # Not all filesystems support size. Fall back to reading the entire object + return await super().getsize(key) + else: + # fsspec doesn't have typing. We'll need to assume or verify this is true + return int(size) diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py new file mode 100644 index 0000000000..43e585415d --- /dev/null +++ b/src/zarr/storage/_local.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import asyncio +import io +import os +import shutil +from pathlib import Path +from typing import TYPE_CHECKING + +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.core.buffer import Buffer +from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.common import concurrent_map + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterable + + from zarr.core.buffer import BufferPrototype + + +def _get(path: Path, prototype: BufferPrototype, byte_range: ByteRequest | None) -> Buffer: + if byte_range is None: + return prototype.buffer.from_bytes(path.read_bytes()) + with path.open("rb") as f: + size = f.seek(0, io.SEEK_END) + if isinstance(byte_range, RangeByteRequest): + f.seek(byte_range.start) + return prototype.buffer.from_bytes(f.read(byte_range.end - f.tell())) + elif isinstance(byte_range, OffsetByteRequest): + f.seek(byte_range.offset) + elif isinstance(byte_range, SuffixByteRequest): + f.seek(max(0, size - byte_range.suffix)) + else: + raise TypeError(f"Unexpected byte_range, got {byte_range}.") + return prototype.buffer.from_bytes(f.read()) + + +def _put( + path: Path, + value: Buffer, + start: int | None = None, + exclusive: bool = False, +) -> int | None: + path.parent.mkdir(parents=True, exist_ok=True) + if start is not None: + with path.open("r+b") as f: + f.seek(start) + # write takes any object supporting the buffer protocol + f.write(value.as_buffer_like()) + return None + else: + view = value.as_buffer_like() + if exclusive: + mode = "xb" + else: + mode = "wb" + with path.open(mode=mode) as f: + # write takes any object supporting the buffer protocol + return f.write(view) + + +class LocalStore(Store): + """ + Store for the local file system. + + Parameters + ---------- + root : str or Path + Directory to use as root of store. + read_only : bool + Whether the store is read-only + + Attributes + ---------- + supports_writes + supports_deletes + supports_partial_writes + supports_listing + root + """ + + supports_writes: bool = True + supports_deletes: bool = True + supports_partial_writes: bool = True + supports_listing: bool = True + + root: Path + + def __init__(self, root: Path | str, *, read_only: bool = False) -> None: + super().__init__(read_only=read_only) + if isinstance(root, str): + root = Path(root) + if not isinstance(root, Path): + raise TypeError( + f"'root' must be a string or Path instance. Got an instance of {type(root)} instead." + ) + self.root = root + + def with_read_only(self, read_only: bool = False) -> LocalStore: + # docstring inherited + return type(self)( + root=self.root, + read_only=read_only, + ) + + async def _open(self) -> None: + if not self.read_only: + self.root.mkdir(parents=True, exist_ok=True) + return await super()._open() + + async def clear(self) -> None: + # docstring inherited + self._check_writable() + shutil.rmtree(self.root) + self.root.mkdir() + + def __str__(self) -> str: + return f"file://{self.root.as_posix()}" + + def __repr__(self) -> str: + return f"LocalStore('{self}')" + + def __eq__(self, other: object) -> bool: + return isinstance(other, type(self)) and self.root == other.root + + async def get( + self, + key: str, + prototype: BufferPrototype | None = None, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + # docstring inherited + if prototype is None: + prototype = default_buffer_prototype() + if not self._is_open: + await self._open() + assert isinstance(key, str) + path = self.root / key + + try: + return await asyncio.to_thread(_get, path, prototype, byte_range) + except (FileNotFoundError, IsADirectoryError, NotADirectoryError): + return None + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + # docstring inherited + args = [] + for key, byte_range in key_ranges: + assert isinstance(key, str) + path = self.root / key + args.append((_get, path, prototype, byte_range)) + return await concurrent_map(args, asyncio.to_thread, limit=None) # TODO: fix limit + + async def set(self, key: str, value: Buffer) -> None: + # docstring inherited + return await self._set(key, value) + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + # docstring inherited + try: + return await self._set(key, value, exclusive=True) + except FileExistsError: + pass + + async def _set(self, key: str, value: Buffer, exclusive: bool = False) -> None: + if not self._is_open: + await self._open() + self._check_writable() + assert isinstance(key, str) + if not isinstance(value, Buffer): + raise TypeError( + f"LocalStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) + path = self.root / key + await asyncio.to_thread(_put, path, value, start=None, exclusive=exclusive) + + async def set_partial_values( + self, key_start_values: Iterable[tuple[str, int, bytes | bytearray | memoryview]] + ) -> None: + # docstring inherited + self._check_writable() + args = [] + for key, start, value in key_start_values: + assert isinstance(key, str) + path = self.root / key + args.append((_put, path, value, start)) + await concurrent_map(args, asyncio.to_thread, limit=None) # TODO: fix limit + + async def delete(self, key: str) -> None: + """ + Remove a key from the store. + + Parameters + ---------- + key : str + + Notes + ----- + If ``key`` is a directory within this store, the entire directory + at ``store.root / key`` is deleted. + """ + # docstring inherited + self._check_writable() + path = self.root / key + if path.is_dir(): # TODO: support deleting directories? shutil.rmtree? + shutil.rmtree(path) + else: + await asyncio.to_thread(path.unlink, True) # Q: we may want to raise if path is missing + + async def delete_dir(self, prefix: str) -> None: + # docstring inherited + self._check_writable() + path = self.root / prefix + if path.is_dir(): + shutil.rmtree(path) + elif path.is_file(): + raise ValueError(f"delete_dir was passed a {prefix=!r} that is a file") + else: + # Non-existent directory + # This path is tested by test_group:test_create_creates_parents for one + pass + + async def exists(self, key: str) -> bool: + # docstring inherited + path = self.root / key + return await asyncio.to_thread(path.is_file) + + async def list(self) -> AsyncIterator[str]: + # docstring inherited + to_strip = self.root.as_posix() + "/" + for p in list(self.root.rglob("*")): + if p.is_file(): + yield p.as_posix().replace(to_strip, "") + + async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + to_strip = self.root.as_posix() + "/" + prefix = prefix.rstrip("/") + for p in (self.root / prefix).rglob("*"): + if p.is_file(): + yield p.as_posix().replace(to_strip, "") + + async def list_dir(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + base = self.root / prefix + try: + key_iter = base.iterdir() + for key in key_iter: + yield key.relative_to(base).as_posix() + except (FileNotFoundError, NotADirectoryError): + pass + + async def move(self, dest_root: Path | str) -> None: + """ + Move the store to another path. The old root directory is deleted. + """ + if isinstance(dest_root, str): + dest_root = Path(dest_root) + os.makedirs(dest_root.parent, exist_ok=True) + if os.path.exists(dest_root): + raise FileExistsError(f"Destination root {dest_root} already exists.") + shutil.move(self.root, dest_root) + self.root = dest_root + + async def getsize(self, key: str) -> int: + return os.path.getsize(self.root / key) diff --git a/src/zarr/storage/_logging.py b/src/zarr/storage/_logging.py new file mode 100644 index 0000000000..a2164a418f --- /dev/null +++ b/src/zarr/storage/_logging.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import inspect +import logging +import sys +import time +from collections import defaultdict +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, Self, TypeVar + +from zarr.abc.store import Store +from zarr.storage._wrapper import WrapperStore + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Generator, Iterable + + from zarr.abc.store import ByteRequest + from zarr.core.buffer import Buffer, BufferPrototype + + counter: defaultdict[str, int] + +T_Store = TypeVar("T_Store", bound=Store) + + +class LoggingStore(WrapperStore[T_Store]): + """ + Store that logs all calls to another wrapped store. + + Parameters + ---------- + store : Store + Store to wrap + log_level : str + Log level + log_handler : logging.Handler + Log handler + + Attributes + ---------- + counter : dict + Counter of number of times each method has been called + """ + + counter: defaultdict[str, int] + + def __init__( + self, + store: T_Store, + log_level: str = "DEBUG", + log_handler: logging.Handler | None = None, + ) -> None: + super().__init__(store) + self.counter = defaultdict(int) + self.log_level = log_level + self.log_handler = log_handler + self._configure_logger(log_level, log_handler) + + def _configure_logger( + self, log_level: str = "DEBUG", log_handler: logging.Handler | None = None + ) -> None: + self.log_level = log_level + self.logger = logging.getLogger(f"LoggingStore({self._store})") + self.logger.setLevel(log_level) + + if not self.logger.hasHandlers(): + if not log_handler: + log_handler = self._default_handler() + # Add handler to logger + self.logger.addHandler(log_handler) + + def _default_handler(self) -> logging.Handler: + """Define a default log handler""" + handler = logging.StreamHandler(stream=sys.stdout) + handler.setLevel(self.log_level) + handler.setFormatter( + logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ) + return handler + + @contextmanager + def log(self, hint: Any = "") -> Generator[None, None, None]: + """Context manager to log method calls + + Each call to the wrapped store is logged to the configured logger and added to + the counter dict. + """ + method = inspect.stack()[2].function + op = f"{type(self._store).__name__}.{method}" + if hint: + op = f"{op}({hint})" + self.logger.info(" Calling %s", op) + start_time = time.time() + try: + self.counter[method] += 1 + yield + finally: + end_time = time.time() + self.logger.info("Finished %s [%.2f s]", op, end_time - start_time) + + @classmethod + async def open(cls: type[Self], store_cls: type[T_Store], *args: Any, **kwargs: Any) -> Self: + log_level = kwargs.pop("log_level", "DEBUG") + log_handler = kwargs.pop("log_handler", None) + store = store_cls(*args, **kwargs) + await store._open() + return cls(store=store, log_level=log_level, log_handler=log_handler) + + @property + def supports_writes(self) -> bool: + with self.log(): + return self._store.supports_writes + + @property + def supports_deletes(self) -> bool: + with self.log(): + return self._store.supports_deletes + + @property + def supports_partial_writes(self) -> bool: + with self.log(): + return self._store.supports_partial_writes + + @property + def supports_listing(self) -> bool: + with self.log(): + return self._store.supports_listing + + @property + def read_only(self) -> bool: + with self.log(): + return self._store.read_only + + @property + def _is_open(self) -> bool: + with self.log(): + return self._store._is_open + + @_is_open.setter + def _is_open(self, value: bool) -> None: + raise NotImplementedError("LoggingStore must be opened via the `_open` method") + + async def _open(self) -> None: + with self.log(): + return await self._store._open() + + async def _ensure_open(self) -> None: + with self.log(): + return await self._store._ensure_open() + + async def is_empty(self, prefix: str = "") -> bool: + # docstring inherited + with self.log(): + return await self._store.is_empty(prefix=prefix) + + async def clear(self) -> None: + # docstring inherited + with self.log(): + return await self._store.clear() + + def __str__(self) -> str: + return f"logging-{self._store}" + + def __repr__(self) -> str: + return f"LoggingStore({self._store.__class__.__name__}, '{self._store}')" + + def __eq__(self, other: object) -> bool: + with self.log(other): + return type(self) is type(other) and self._store.__eq__(other._store) # type: ignore[attr-defined] + + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + # docstring inherited + with self.log(key): + return await self._store.get(key=key, prototype=prototype, byte_range=byte_range) + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + # docstring inherited + keys = ",".join([k[0] for k in key_ranges]) + with self.log(keys): + return await self._store.get_partial_values(prototype=prototype, key_ranges=key_ranges) + + async def exists(self, key: str) -> bool: + # docstring inherited + with self.log(key): + return await self._store.exists(key) + + async def set(self, key: str, value: Buffer) -> None: + # docstring inherited + with self.log(key): + return await self._store.set(key=key, value=value) + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + # docstring inherited + with self.log(key): + return await self._store.set_if_not_exists(key=key, value=value) + + async def delete(self, key: str) -> None: + # docstring inherited + with self.log(key): + return await self._store.delete(key=key) + + async def set_partial_values( + self, key_start_values: Iterable[tuple[str, int, bytes | bytearray | memoryview]] + ) -> None: + # docstring inherited + keys = ",".join([k[0] for k in key_start_values]) + with self.log(keys): + return await self._store.set_partial_values(key_start_values=key_start_values) + + async def list(self) -> AsyncGenerator[str, None]: + # docstring inherited + with self.log(): + async for key in self._store.list(): + yield key + + async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + # docstring inherited + with self.log(prefix): + async for key in self._store.list_prefix(prefix=prefix): + yield key + + async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + # docstring inherited + with self.log(prefix): + async for key in self._store.list_dir(prefix=prefix): + yield key + + async def delete_dir(self, prefix: str) -> None: + # docstring inherited + with self.log(prefix): + await self._store.delete_dir(prefix=prefix) + + async def getsize(self, key: str) -> int: + with self.log(key): + return await self._store.getsize(key) + + async def getsize_prefix(self, prefix: str) -> int: + with self.log(prefix): + return await self._store.getsize_prefix(prefix) diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py new file mode 100644 index 0000000000..0dc6f13236 --- /dev/null +++ b/src/zarr/storage/_memory.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +from logging import getLogger +from typing import TYPE_CHECKING, Self + +from zarr.abc.store import ByteRequest, Store +from zarr.core.buffer import Buffer, gpu +from zarr.core.common import concurrent_map +from zarr.storage._utils import _normalize_byte_range_index + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterable, MutableMapping + + from zarr.core.buffer import BufferPrototype + + +logger = getLogger(__name__) + + +class MemoryStore(Store): + """ + Store for local memory. + + Parameters + ---------- + store_dict : dict + Initial data + read_only : bool + Whether the store is read-only + + Attributes + ---------- + supports_writes + supports_deletes + supports_partial_writes + supports_listing + """ + + supports_writes: bool = True + supports_deletes: bool = True + supports_partial_writes: bool = True + supports_listing: bool = True + + _store_dict: MutableMapping[str, Buffer] + + def __init__( + self, + store_dict: MutableMapping[str, Buffer] | None = None, + *, + read_only: bool = False, + ) -> None: + super().__init__(read_only=read_only) + if store_dict is None: + store_dict = {} + self._store_dict = store_dict + + def with_read_only(self, read_only: bool = False) -> MemoryStore: + # docstring inherited + return type(self)( + store_dict=self._store_dict, + read_only=read_only, + ) + + async def clear(self) -> None: + # docstring inherited + self._store_dict.clear() + + def __str__(self) -> str: + return f"memory://{id(self._store_dict)}" + + def __repr__(self) -> str: + return f"MemoryStore('{self}')" + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, type(self)) + and self._store_dict == other._store_dict + and self.read_only == other.read_only + ) + + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + # docstring inherited + if not self._is_open: + await self._open() + assert isinstance(key, str) + try: + value = self._store_dict[key] + start, stop = _normalize_byte_range_index(value, byte_range) + return prototype.buffer.from_buffer(value[start:stop]) + except KeyError: + return None + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + # docstring inherited + + # All the key-ranges arguments goes with the same prototype + async def _get(key: str, byte_range: ByteRequest | None) -> Buffer | None: + return await self.get(key, prototype=prototype, byte_range=byte_range) + + return await concurrent_map(key_ranges, _get, limit=None) + + async def exists(self, key: str) -> bool: + # docstring inherited + return key in self._store_dict + + async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: + # docstring inherited + self._check_writable() + await self._ensure_open() + assert isinstance(key, str) + if not isinstance(value, Buffer): + raise TypeError( + f"MemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) + + if byte_range is not None: + buf = self._store_dict[key] + buf[byte_range[0] : byte_range[1]] = value + self._store_dict[key] = buf + else: + self._store_dict[key] = value + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + # docstring inherited + self._check_writable() + await self._ensure_open() + self._store_dict.setdefault(key, value) + + async def delete(self, key: str) -> None: + # docstring inherited + self._check_writable() + try: + del self._store_dict[key] + except KeyError: + logger.debug("Key %s does not exist.", key) + + async def set_partial_values(self, key_start_values: Iterable[tuple[str, int, bytes]]) -> None: + # docstring inherited + raise NotImplementedError + + async def list(self) -> AsyncIterator[str]: + # docstring inherited + for key in self._store_dict: + yield key + + async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + # note: we materialize all dict keys into a list here so we can mutate the dict in-place (e.g. in delete_prefix) + for key in list(self._store_dict): + if key.startswith(prefix): + yield key + + async def list_dir(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + prefix = prefix.rstrip("/") + + if prefix == "": + keys_unique = {k.split("/")[0] for k in self._store_dict} + else: + # Our dictionary doesn't contain directory markers, but we want to include + # a pseudo directory when there's a nested item and we're listing an + # intermediate level. + keys_unique = { + key.removeprefix(prefix + "/").split("/")[0] + for key in self._store_dict + if key.startswith(prefix + "/") and key != prefix + } + + for key in keys_unique: + yield key + + +class GpuMemoryStore(MemoryStore): + """ + Store for GPU memory. + + Stores every chunk in GPU memory irrespective of the original location. + + The dictionary of buffers to initialize this memory store with *must* be + GPU Buffers. + + Writing data to this store through ``.set`` will move the buffer to the GPU + if necessary. + + Parameters + ---------- + store_dict : MutableMapping, optional + A mutable mapping with string keys and :class:`zarr.core.buffer.gpu.Buffer` + values. + read_only : bool + Whether to open the store in read-only mode. + """ + + _store_dict: MutableMapping[str, gpu.Buffer] # type: ignore[assignment] + + def __init__( + self, + store_dict: MutableMapping[str, gpu.Buffer] | None = None, + *, + read_only: bool = False, + ) -> None: + super().__init__(store_dict=store_dict, read_only=read_only) # type: ignore[arg-type] + + def __str__(self) -> str: + return f"gpumemory://{id(self._store_dict)}" + + def __repr__(self) -> str: + return f"GpuMemoryStore('{self}')" + + @classmethod + def from_dict(cls, store_dict: MutableMapping[str, Buffer]) -> Self: + """ + Create a GpuMemoryStore from a dictionary of buffers at any location. + + The dictionary backing the newly created ``GpuMemoryStore`` will not be + the same as ``store_dict``. + + Parameters + ---------- + store_dict : mapping + A mapping of strings keys to arbitrary Buffers. The buffer data + will be moved into a :class:`gpu.Buffer`. + + Returns + ------- + GpuMemoryStore + """ + gpu_store_dict = {k: gpu.Buffer.from_buffer(v) for k, v in store_dict.items()} + return cls(gpu_store_dict) + + async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: + # docstring inherited + self._check_writable() + assert isinstance(key, str) + if not isinstance(value, Buffer): + raise TypeError( + f"GpuMemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) + # Convert to gpu.Buffer + gpu_value = value if isinstance(value, gpu.Buffer) else gpu.Buffer.from_buffer(value) + await super().set(key, gpu_value, byte_range=byte_range) diff --git a/src/zarr/storage/_obstore.py b/src/zarr/storage/_obstore.py new file mode 100644 index 0000000000..047ed07fbb --- /dev/null +++ b/src/zarr/storage/_obstore.py @@ -0,0 +1,495 @@ +from __future__ import annotations + +import asyncio +import contextlib +import pickle +from collections import defaultdict +from typing import TYPE_CHECKING, TypedDict + +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.core.config import config + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Coroutine, Iterable + from typing import Any + + from obstore import ListResult, ListStream, ObjectMeta, OffsetRange, SuffixRange + from obstore.store import ObjectStore as _UpstreamObjectStore + + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.common import BytesLike + +__all__ = ["ObjectStore"] + +_ALLOWED_EXCEPTIONS: tuple[type[Exception], ...] = ( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, +) + + +class ObjectStore(Store): + """ + Store that uses obstore for fast read/write from AWS, GCP, Azure. + + Parameters + ---------- + store : obstore.store.ObjectStore + An obstore store instance that is set up with the proper credentials. + read_only : bool + Whether to open the store in read-only mode. + + Warnings + -------- + ObjectStore is experimental and subject to API changes without notice. Please + raise an issue with any comments/concerns about the store. + """ + + store: _UpstreamObjectStore + """The underlying obstore instance.""" + + def __eq__(self, value: object) -> bool: + if not isinstance(value, ObjectStore): + return False + + if not self.read_only == value.read_only: + return False + + return self.store == value.store + + def __init__(self, store: _UpstreamObjectStore, *, read_only: bool = False) -> None: + if not store.__class__.__module__.startswith("obstore"): + raise TypeError(f"expected ObjectStore class, got {store!r}") + super().__init__(read_only=read_only) + self.store = store + + def with_read_only(self, read_only: bool = False) -> ObjectStore: + # docstring inherited + return type(self)( + store=self.store, + read_only=read_only, + ) + + def __str__(self) -> str: + return f"object_store://{self.store}" + + def __repr__(self) -> str: + return f"{type(self).__name__}({self})" + + def __getstate__(self) -> dict[Any, Any]: + state = self.__dict__.copy() + state["store"] = pickle.dumps(self.store) + return state + + def __setstate__(self, state: dict[Any, Any]) -> None: + state["store"] = pickle.loads(state["store"]) + self.__dict__.update(state) + + async def get( + self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None + ) -> Buffer | None: + # docstring inherited + import obstore as obs + + try: + if byte_range is None: + resp = await obs.get_async(self.store, key) + return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] + elif isinstance(byte_range, RangeByteRequest): + bytes = await obs.get_range_async( + self.store, key, start=byte_range.start, end=byte_range.end + ) + return prototype.buffer.from_bytes(bytes) # type: ignore[arg-type] + elif isinstance(byte_range, OffsetByteRequest): + resp = await obs.get_async( + self.store, key, options={"range": {"offset": byte_range.offset}} + ) + return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] + elif isinstance(byte_range, SuffixByteRequest): + # some object stores (Azure) don't support suffix requests. In this + # case, our workaround is to first get the length of the object and then + # manually request the byte range at the end. + try: + resp = await obs.get_async( + self.store, key, options={"range": {"suffix": byte_range.suffix}} + ) + return prototype.buffer.from_bytes(await resp.bytes_async()) # type: ignore[arg-type] + except obs.exceptions.NotSupportedError: + head_resp = await obs.head_async(self.store, key) + file_size = head_resp["size"] + suffix_len = byte_range.suffix + buffer = await obs.get_range_async( + self.store, + key, + start=file_size - suffix_len, + length=suffix_len, + ) + return prototype.buffer.from_bytes(buffer) # type: ignore[arg-type] + else: + raise ValueError(f"Unexpected byte_range, got {byte_range}") + except _ALLOWED_EXCEPTIONS: + return None + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + # docstring inherited + return await _get_partial_values(self.store, prototype=prototype, key_ranges=key_ranges) + + async def exists(self, key: str) -> bool: + # docstring inherited + import obstore as obs + + try: + await obs.head_async(self.store, key) + except FileNotFoundError: + return False + else: + return True + + @property + def supports_writes(self) -> bool: + # docstring inherited + return True + + async def set(self, key: str, value: Buffer) -> None: + # docstring inherited + import obstore as obs + + self._check_writable() + + buf = value.as_buffer_like() + await obs.put_async(self.store, key, buf) + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + # docstring inherited + import obstore as obs + + self._check_writable() + buf = value.as_buffer_like() + with contextlib.suppress(obs.exceptions.AlreadyExistsError): + await obs.put_async(self.store, key, buf, mode="create") + + @property + def supports_deletes(self) -> bool: + # docstring inherited + return True + + async def delete(self, key: str) -> None: + # docstring inherited + import obstore as obs + + self._check_writable() + await obs.delete_async(self.store, key) + + @property + def supports_partial_writes(self) -> bool: + # docstring inherited + return False + + async def set_partial_values( + self, key_start_values: Iterable[tuple[str, int, BytesLike]] + ) -> None: + # docstring inherited + raise NotImplementedError + + @property + def supports_listing(self) -> bool: + # docstring inherited + return True + + def list(self) -> AsyncGenerator[str, None]: + # docstring inherited + import obstore as obs + + objects: ListStream[list[ObjectMeta]] = obs.list(self.store) + return _transform_list(objects) + + def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + # docstring inherited + import obstore as obs + + objects: ListStream[list[ObjectMeta]] = obs.list(self.store, prefix=prefix) + return _transform_list(objects) + + def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + # docstring inherited + import obstore as obs + + coroutine = obs.list_with_delimiter_async(self.store, prefix=prefix) + return _transform_list_dir(coroutine, prefix) + + +async def _transform_list( + list_stream: ListStream[list[ObjectMeta]], +) -> AsyncGenerator[str, None]: + """ + Transform the result of list into an async generator of paths. + """ + async for batch in list_stream: + for item in batch: + yield item["path"] + + +async def _transform_list_dir( + list_result_coroutine: Coroutine[Any, Any, ListResult[list[ObjectMeta]]], prefix: str +) -> AsyncGenerator[str, None]: + """ + Transform the result of list_with_delimiter into an async generator of paths. + """ + list_result = await list_result_coroutine + + # We assume that the underlying object-store implementation correctly handles the + # prefix, so we don't double-check that the returned results actually start with the + # given prefix. + prefixes = [obj.lstrip(prefix).lstrip("/") for obj in list_result["common_prefixes"]] + objects = [obj["path"].removeprefix(prefix).lstrip("/") for obj in list_result["objects"]] + for item in prefixes + objects: + yield item + + +class _BoundedRequest(TypedDict): + """Range request with a known start and end byte. + + These requests can be multiplexed natively on the Rust side with + `obstore.get_ranges_async`. + """ + + original_request_index: int + """The positional index in the original key_ranges input""" + + start: int + """Start byte offset.""" + + end: int + """End byte offset.""" + + +class _OtherRequest(TypedDict): + """Offset or suffix range requests. + + These requests cannot be concurrent on the Rust side, and each need their own call + to `obstore.get_async`, passing in the `range` parameter. + """ + + original_request_index: int + """The positional index in the original key_ranges input""" + + path: str + """The path to request from.""" + + range: OffsetRange | None + # Note: suffix requests are handled separately because some object stores (Azure) + # don't support them + """The range request type.""" + + +class _SuffixRequest(TypedDict): + """Offset or suffix range requests. + + These requests cannot be concurrent on the Rust side, and each need their own call + to `obstore.get_async`, passing in the `range` parameter. + """ + + original_request_index: int + """The positional index in the original key_ranges input""" + + path: str + """The path to request from.""" + + range: SuffixRange + """The suffix range.""" + + +class _Response(TypedDict): + """A response buffer associated with the original index that it should be restored to.""" + + original_request_index: int + """The positional index in the original key_ranges input""" + + buffer: Buffer + """The buffer returned from obstore's range request.""" + + +async def _make_bounded_requests( + store: _UpstreamObjectStore, + path: str, + requests: list[_BoundedRequest], + prototype: BufferPrototype, + semaphore: asyncio.Semaphore, +) -> list[_Response]: + """Make all bounded requests for a specific file. + + `obstore.get_ranges_async` allows for making concurrent requests for multiple ranges + within a single file, and will e.g. merge concurrent requests. This only uses one + single Python coroutine. + """ + import obstore as obs + + starts = [r["start"] for r in requests] + ends = [r["end"] for r in requests] + async with semaphore: + responses = await obs.get_ranges_async(store, path=path, starts=starts, ends=ends) + + buffer_responses: list[_Response] = [] + for request, response in zip(requests, responses, strict=True): + buffer_responses.append( + { + "original_request_index": request["original_request_index"], + "buffer": prototype.buffer.from_bytes(response), # type: ignore[arg-type] + } + ) + + return buffer_responses + + +async def _make_other_request( + store: _UpstreamObjectStore, + request: _OtherRequest, + prototype: BufferPrototype, + semaphore: asyncio.Semaphore, +) -> list[_Response]: + """Make offset or full-file requests. + + We return a `list[_Response]` for symmetry with `_make_bounded_requests` so that all + futures can be gathered together. + """ + import obstore as obs + + async with semaphore: + if request["range"] is None: + resp = await obs.get_async(store, request["path"]) + else: + resp = await obs.get_async(store, request["path"], options={"range": request["range"]}) + buffer = await resp.bytes_async() + + return [ + { + "original_request_index": request["original_request_index"], + "buffer": prototype.buffer.from_bytes(buffer), # type: ignore[arg-type] + } + ] + + +async def _make_suffix_request( + store: _UpstreamObjectStore, + request: _SuffixRequest, + prototype: BufferPrototype, + semaphore: asyncio.Semaphore, +) -> list[_Response]: + """Make suffix requests. + + This is separated out from `_make_other_request` because some object stores (Azure) + don't support suffix requests. In this case, our workaround is to first get the + length of the object and then manually request the byte range at the end. + + We return a `list[_Response]` for symmetry with `_make_bounded_requests` so that all + futures can be gathered together. + """ + import obstore as obs + + async with semaphore: + try: + resp = await obs.get_async(store, request["path"], options={"range": request["range"]}) + buffer = await resp.bytes_async() + except obs.exceptions.NotSupportedError: + head_resp = await obs.head_async(store, request["path"]) + file_size = head_resp["size"] + suffix_len = request["range"]["suffix"] + buffer = await obs.get_range_async( + store, + request["path"], + start=file_size - suffix_len, + length=suffix_len, + ) + + return [ + { + "original_request_index": request["original_request_index"], + "buffer": prototype.buffer.from_bytes(buffer), # type: ignore[arg-type] + } + ] + + +async def _get_partial_values( + store: _UpstreamObjectStore, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], +) -> list[Buffer | None]: + """Make multiple range requests. + + ObjectStore has a `get_ranges` method that will additionally merge nearby ranges, + but it's _per_ file. So we need to split these key_ranges into **per-file** key + ranges, and then reassemble the results in the original order. + + We separate into different requests: + + - One call to `obstore.get_ranges_async` **per target file** + - One call to `obstore.get_async` for each other request. + """ + key_ranges = list(key_ranges) + per_file_bounded_requests: dict[str, list[_BoundedRequest]] = defaultdict(list) + other_requests: list[_OtherRequest] = [] + suffix_requests: list[_SuffixRequest] = [] + + for idx, (path, byte_range) in enumerate(key_ranges): + if byte_range is None: + other_requests.append( + { + "original_request_index": idx, + "path": path, + "range": None, + } + ) + elif isinstance(byte_range, RangeByteRequest): + per_file_bounded_requests[path].append( + {"original_request_index": idx, "start": byte_range.start, "end": byte_range.end} + ) + elif isinstance(byte_range, OffsetByteRequest): + other_requests.append( + { + "original_request_index": idx, + "path": path, + "range": {"offset": byte_range.offset}, + } + ) + elif isinstance(byte_range, SuffixByteRequest): + suffix_requests.append( + { + "original_request_index": idx, + "path": path, + "range": {"suffix": byte_range.suffix}, + } + ) + else: + raise ValueError(f"Unsupported range input: {byte_range}") + + semaphore = asyncio.Semaphore(config.get("async.concurrency")) + + futs: list[Coroutine[Any, Any, list[_Response]]] = [] + for path, bounded_ranges in per_file_bounded_requests.items(): + futs.append( + _make_bounded_requests(store, path, bounded_ranges, prototype, semaphore=semaphore) + ) + + for request in other_requests: + futs.append(_make_other_request(store, request, prototype, semaphore=semaphore)) # noqa: PERF401 + + for suffix_request in suffix_requests: + futs.append(_make_suffix_request(store, suffix_request, prototype, semaphore=semaphore)) # noqa: PERF401 + + buffers: list[Buffer | None] = [None] * len(key_ranges) + + for responses in await asyncio.gather(*futs): + for resp in responses: + buffers[resp["original_request_index"]] = resp["buffer"] + + return buffers diff --git a/src/zarr/storage/_utils.py b/src/zarr/storage/_utils.py new file mode 100644 index 0000000000..145790278c --- /dev/null +++ b/src/zarr/storage/_utils.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import TYPE_CHECKING, TypeVar + +from zarr.abc.store import OffsetByteRequest, RangeByteRequest, SuffixByteRequest + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + + from zarr.abc.store import ByteRequest + from zarr.core.buffer import Buffer + + +def normalize_path(path: str | bytes | Path | None) -> str: + if path is None: + result = "" + elif isinstance(path, bytes): + result = str(path, "ascii") + + # handle pathlib.Path + elif isinstance(path, Path): + result = str(path) + + elif isinstance(path, str): + result = path + + else: + raise TypeError(f'Object {path} has an invalid type for "path": {type(path).__name__}') + + # convert backslash to forward slash + result = result.replace("\\", "/") + + # remove leading and trailing slashes + result = result.strip("/") + + # collapse any repeated slashes + pat = re.compile(r"//+") + result = pat.sub("/", result) + + # disallow path segments with just '.' or '..' + segments = result.split("/") + if any(s in {".", ".."} for s in segments): + raise ValueError( + f"The path {path!r} is invalid because its string representation contains '.' or '..' segments." + ) + + return result + + +def _normalize_byte_range_index(data: Buffer, byte_range: ByteRequest | None) -> tuple[int, int]: + """ + Convert an ByteRequest into an explicit start and stop + """ + if byte_range is None: + start = 0 + stop = len(data) + 1 + elif isinstance(byte_range, RangeByteRequest): + start = byte_range.start + stop = byte_range.end + elif isinstance(byte_range, OffsetByteRequest): + start = byte_range.offset + stop = len(data) + 1 + elif isinstance(byte_range, SuffixByteRequest): + start = len(data) - byte_range.suffix + stop = len(data) + 1 + else: + raise ValueError(f"Unexpected byte_range, got {byte_range}.") + return (start, stop) + + +def _join_paths(paths: Iterable[str]) -> str: + """ + Filter out instances of '' and join the remaining strings with '/'. + + Parameters + ---------- + paths : Iterable[str] + + Returns + ------- + str + + Examples + -------- + >>> _join_paths(["", "a", "b"]) + 'a/b' + >>> _join_paths(["a", "b", "c"]) + 'a/b/c' + """ + return "/".join(filter(lambda v: v != "", paths)) + + +def _relativize_path(*, path: str, prefix: str) -> str: + """ + Make a "/"-delimited path relative to some prefix. If the prefix is '', then the path is + returned as-is. Otherwise, the prefix is removed from the path as well as the separator + string "/". + + If ``prefix`` is not the empty string and ``path`` does not start with ``prefix`` + followed by a "/" character, then an error is raised. + + This function assumes that the prefix does not end with "/". + + Parameters + ---------- + path : str + The path to make relative to the prefix. + prefix : str + The prefix to make the path relative to. + + Returns + ------- + str + + Examples + -------- + >>> _relativize_path(path="", prefix="a/b") + 'a/b' + >>> _relativize_path(path="a/b", prefix="a/b/c") + 'c' + """ + if prefix == "": + return path + else: + _prefix = prefix + "/" + if not path.startswith(_prefix): + raise ValueError(f"The first component of {path} does not start with {prefix}.") + return path.removeprefix(f"{prefix}/") + + +def _normalize_paths(paths: Iterable[str]) -> tuple[str, ...]: + """ + Normalize the input paths according to the normalization scheme used for zarr node paths. + If any two paths normalize to the same value, raise a ValueError. + """ + path_map: dict[str, str] = {} + for path in paths: + parsed = normalize_path(path) + if parsed in path_map: + msg = ( + f"After normalization, the value '{path}' collides with '{path_map[parsed]}'. " + f"Both '{path}' and '{path_map[parsed]}' normalize to the same value: '{parsed}'. " + f"You should use either '{path}' or '{path_map[parsed]}', but not both." + ) + raise ValueError(msg) + path_map[parsed] = path + return tuple(path_map.keys()) + + +T = TypeVar("T") + + +def _normalize_path_keys(data: Mapping[str, T]) -> dict[str, T]: + """ + Normalize the keys of the input dict according to the normalization scheme used for zarr node + paths. If any two keys in the input normalize to the same value, raise a ValueError. + Returns a dict where the keys are the elements of the input and the values are the + normalized form of each key. + """ + parsed_keys = _normalize_paths(data.keys()) + return dict(zip(parsed_keys, data.values(), strict=True)) diff --git a/src/zarr/storage/_wrapper.py b/src/zarr/storage/_wrapper.py new file mode 100644 index 0000000000..f21d378191 --- /dev/null +++ b/src/zarr/storage/_wrapper.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Generic, TypeVar + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, AsyncIterator, Iterable + from types import TracebackType + from typing import Any, Self + + from zarr.abc.store import ByteRequest + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.common import BytesLike + +from zarr.abc.store import Store + +T_Store = TypeVar("T_Store", bound=Store) + + +class WrapperStore(Store, Generic[T_Store]): + """ + Store that wraps an existing Store. + + By default all of the store methods are delegated to the wrapped store instance, which is + accessible via the ``._store`` attribute of this class. + + Use this class to modify or extend the behavior of the other store classes. + """ + + _store: T_Store + + def __init__(self, store: T_Store) -> None: + self._store = store + + @classmethod + async def open(cls: type[Self], store_cls: type[T_Store], *args: Any, **kwargs: Any) -> Self: + store = store_cls(*args, **kwargs) + await store._open() + return cls(store=store) + + def __enter__(self) -> Self: + return type(self)(self._store.__enter__()) + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + return self._store.__exit__(exc_type, exc_value, traceback) + + async def _open(self) -> None: + await self._store._open() + + async def _ensure_open(self) -> None: + await self._store._ensure_open() + + async def is_empty(self, prefix: str) -> bool: + return await self._store.is_empty(prefix) + + @property + def _is_open(self) -> bool: + return self._store._is_open + + @_is_open.setter + def _is_open(self, value: bool) -> None: + raise NotImplementedError("WrapperStore must be opened via the `_open` method") + + async def clear(self) -> None: + return await self._store.clear() + + @property + def read_only(self) -> bool: + return self._store.read_only + + def _check_writable(self) -> None: + return self._store._check_writable() + + def __eq__(self, value: object) -> bool: + return type(self) is type(value) and self._store.__eq__(value._store) # type: ignore[attr-defined] + + def __str__(self) -> str: + return f"wrapping-{self._store}" + + def __repr__(self) -> str: + return f"WrapperStore({self._store.__class__.__name__}, '{self._store}')" + + async def get( + self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None + ) -> Buffer | None: + return await self._store.get(key, prototype, byte_range) + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + return await self._store.get_partial_values(prototype, key_ranges) + + async def exists(self, key: str) -> bool: + return await self._store.exists(key) + + async def set(self, key: str, value: Buffer) -> None: + await self._store.set(key, value) + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + return await self._store.set_if_not_exists(key, value) + + async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None: + await self._store._set_many(values) + + @property + def supports_writes(self) -> bool: + return self._store.supports_writes + + @property + def supports_deletes(self) -> bool: + return self._store.supports_deletes + + async def delete(self, key: str) -> None: + await self._store.delete(key) + + @property + def supports_partial_writes(self) -> bool: + return self._store.supports_partial_writes + + async def set_partial_values( + self, key_start_values: Iterable[tuple[str, int, BytesLike]] + ) -> None: + return await self._store.set_partial_values(key_start_values) + + @property + def supports_listing(self) -> bool: + return self._store.supports_listing + + def list(self) -> AsyncIterator[str]: + return self._store.list() + + def list_prefix(self, prefix: str) -> AsyncIterator[str]: + return self._store.list_prefix(prefix) + + def list_dir(self, prefix: str) -> AsyncIterator[str]: + return self._store.list_dir(prefix) + + async def delete_dir(self, prefix: str) -> None: + return await self._store.delete_dir(prefix) + + def close(self) -> None: + self._store.close() + + async def _get_many( + self, requests: Iterable[tuple[str, BufferPrototype, ByteRequest | None]] + ) -> AsyncGenerator[tuple[str, Buffer | None], None]: + async for req in self._store._get_many(requests): + yield req diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py new file mode 100644 index 0000000000..5d147deded --- /dev/null +++ b/src/zarr/storage/_zip.py @@ -0,0 +1,303 @@ +from __future__ import annotations + +import os +import shutil +import threading +import time +import zipfile +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal + +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.core.buffer import Buffer, BufferPrototype + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterable + +ZipStoreAccessModeLiteral = Literal["r", "w", "a"] + + +class ZipStore(Store): + """ + Store using a ZIP file. + + Parameters + ---------- + path : str + Location of file. + mode : str, optional + One of 'r' to read an existing file, 'w' to truncate and write a new + file, 'a' to append to an existing file, or 'x' to exclusively create + and write a new file. + compression : int, optional + Compression method to use when writing to the archive. + allowZip64 : bool, optional + If True (the default) will create ZIP files that use the ZIP64 + extensions when the zipfile is larger than 2 GiB. If False + will raise an exception when the ZIP file would require ZIP64 + extensions. + + Attributes + ---------- + allowed_exceptions + supports_writes + supports_deletes + supports_partial_writes + supports_listing + path + compression + allowZip64 + """ + + supports_writes: bool = True + supports_deletes: bool = False + supports_partial_writes: bool = False + supports_listing: bool = True + + path: Path + compression: int + allowZip64: bool + + _zf: zipfile.ZipFile + _lock: threading.RLock + + def __init__( + self, + path: Path | str, + *, + mode: ZipStoreAccessModeLiteral = "r", + read_only: bool | None = None, + compression: int = zipfile.ZIP_STORED, + allowZip64: bool = True, + ) -> None: + if read_only is None: + read_only = mode == "r" + + super().__init__(read_only=read_only) + + if isinstance(path, str): + path = Path(path) + assert isinstance(path, Path) + self.path = path # root? + + self._zmode = mode + self.compression = compression + self.allowZip64 = allowZip64 + + def _sync_open(self) -> None: + if self._is_open: + raise ValueError("store is already open") + + self._lock = threading.RLock() + + self._zf = zipfile.ZipFile( + self.path, + mode=self._zmode, + compression=self.compression, + allowZip64=self.allowZip64, + ) + + self._is_open = True + + async def _open(self) -> None: + self._sync_open() + + def __getstate__(self) -> dict[str, Any]: + # We need a copy to not modify the state of the original store + state = self.__dict__.copy() + for attr in ["_zf", "_lock"]: + state.pop(attr, None) + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__ = state + self._is_open = False + self._sync_open() + + def close(self) -> None: + # docstring inherited + super().close() + with self._lock: + self._zf.close() + + async def clear(self) -> None: + # docstring inherited + with self._lock: + self._check_writable() + self._zf.close() + os.remove(self.path) + self._zf = zipfile.ZipFile( + self.path, mode="w", compression=self.compression, allowZip64=self.allowZip64 + ) + + def __str__(self) -> str: + return f"zip://{self.path}" + + def __repr__(self) -> str: + return f"ZipStore('{self}')" + + def __eq__(self, other: object) -> bool: + return isinstance(other, type(self)) and self.path == other.path + + def _get( + self, + key: str, + prototype: BufferPrototype, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + if not self._is_open: + self._sync_open() + # docstring inherited + try: + with self._zf.open(key) as f: # will raise KeyError + if byte_range is None: + return prototype.buffer.from_bytes(f.read()) + elif isinstance(byte_range, RangeByteRequest): + f.seek(byte_range.start) + return prototype.buffer.from_bytes(f.read(byte_range.end - f.tell())) + size = f.seek(0, os.SEEK_END) + if isinstance(byte_range, OffsetByteRequest): + f.seek(byte_range.offset) + elif isinstance(byte_range, SuffixByteRequest): + f.seek(max(0, size - byte_range.suffix)) + else: + raise TypeError(f"Unexpected byte_range, got {byte_range}.") + return prototype.buffer.from_bytes(f.read()) + except KeyError: + return None + + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + # docstring inherited + assert isinstance(key, str) + + with self._lock: + return self._get(key, prototype=prototype, byte_range=byte_range) + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + # docstring inherited + out = [] + with self._lock: + for key, byte_range in key_ranges: + out.append(self._get(key, prototype=prototype, byte_range=byte_range)) + return out + + def _set(self, key: str, value: Buffer) -> None: + if not self._is_open: + self._sync_open() + # generally, this should be called inside a lock + keyinfo = zipfile.ZipInfo(filename=key, date_time=time.localtime(time.time())[:6]) + keyinfo.compress_type = self.compression + if keyinfo.filename[-1] == os.sep: + keyinfo.external_attr = 0o40775 << 16 # drwxrwxr-x + keyinfo.external_attr |= 0x10 # MS-DOS directory flag + else: + keyinfo.external_attr = 0o644 << 16 # ?rw-r--r-- + self._zf.writestr(keyinfo, value.to_bytes()) + + async def set(self, key: str, value: Buffer) -> None: + # docstring inherited + self._check_writable() + if not self._is_open: + self._sync_open() + assert isinstance(key, str) + if not isinstance(value, Buffer): + raise TypeError( + f"ZipStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead." + ) + with self._lock: + self._set(key, value) + + async def set_partial_values(self, key_start_values: Iterable[tuple[str, int, bytes]]) -> None: + raise NotImplementedError + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + self._check_writable() + with self._lock: + members = self._zf.namelist() + if key not in members: + self._set(key, value) + + async def delete_dir(self, prefix: str) -> None: + # only raise NotImplementedError if any keys are found + self._check_writable() + if prefix != "" and not prefix.endswith("/"): + prefix += "/" + async for _ in self.list_prefix(prefix): + raise NotImplementedError + + async def delete(self, key: str) -> None: + # docstring inherited + # we choose to only raise NotImplementedError here if the key exists + # this allows the array/group APIs to avoid the overhead of existence checks + self._check_writable() + if await self.exists(key): + raise NotImplementedError + + async def exists(self, key: str) -> bool: + # docstring inherited + with self._lock: + try: + self._zf.getinfo(key) + except KeyError: + return False + else: + return True + + async def list(self) -> AsyncIterator[str]: + # docstring inherited + with self._lock: + for key in self._zf.namelist(): + yield key + + async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + async for key in self.list(): + if key.startswith(prefix): + yield key + + async def list_dir(self, prefix: str) -> AsyncIterator[str]: + # docstring inherited + prefix = prefix.rstrip("/") + + keys = self._zf.namelist() + seen = set() + if prefix == "": + keys_unique = {k.split("/")[0] for k in keys} + for key in keys_unique: + if key not in seen: + seen.add(key) + yield key + else: + for key in keys: + if key.startswith(prefix + "/") and key.strip("/") != prefix: + k = key.removeprefix(prefix + "/").split("/")[0] + if k not in seen: + seen.add(k) + yield k + + async def move(self, path: Path | str) -> None: + """ + Move the store to another path. + """ + if isinstance(path, str): + path = Path(path) + self.close() + os.makedirs(path.parent, exist_ok=True) + shutil.move(self.path, path) + self.path = path + await self._open() diff --git a/src/zarr/testing/__init__.py b/src/zarr/testing/__init__.py new file mode 100644 index 0000000000..0b4d8cf417 --- /dev/null +++ b/src/zarr/testing/__init__.py @@ -0,0 +1,13 @@ +import importlib.util +import warnings + +if importlib.util.find_spec("pytest") is not None: + from zarr.testing.store import StoreTests +else: + warnings.warn("pytest not installed, skipping test suite", stacklevel=2) + +from zarr.testing.utils import assert_bytes_equal + +# TODO: import public buffer tests? + +__all__ = ["StoreTests", "assert_bytes_equal"] diff --git a/src/zarr/testing/buffer.py b/src/zarr/testing/buffer.py new file mode 100644 index 0000000000..c3694e268b --- /dev/null +++ b/src/zarr/testing/buffer.py @@ -0,0 +1,77 @@ +# mypy: ignore-errors +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Literal + +import numpy as np +import numpy.typing as npt + +from zarr.core.buffer import Buffer, BufferPrototype, cpu +from zarr.storage import MemoryStore + +if TYPE_CHECKING: + from collections.abc import Iterable + from typing import Self + + +__all__ = [ + "NDBufferUsingTestNDArrayLike", + "StoreExpectingTestBuffer", + "TestBuffer", +] + + +class TestNDArrayLike(np.ndarray): + """An example of a ndarray-like class""" + + __test__ = False + + +class TestBuffer(cpu.Buffer): + """Example of a custom Buffer that handles ArrayLike""" + + __test__ = False + + +class NDBufferUsingTestNDArrayLike(cpu.NDBuffer): + """Example of a custom NDBuffer that handles MyNDArrayLike""" + + @classmethod + def create( + cls, + *, + shape: Iterable[int], + dtype: npt.DTypeLike, + order: Literal["C", "F"] = "C", + fill_value: Any | None = None, + ) -> Self: + """Overwrite `NDBuffer.create` to create an TestNDArrayLike instance""" + ret = cls(TestNDArrayLike(shape=shape, dtype=dtype, order=order)) + if fill_value is not None: + ret.fill(fill_value) + return ret + + +class StoreExpectingTestBuffer(MemoryStore): + """Example of a custom Store that expect MyBuffer for all its non-metadata + + We assume that keys containing "json" is metadata + """ + + async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: + if "json" not in key: + assert isinstance(value, TestBuffer) + await super().set(key, value, byte_range) + + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: tuple[int, int | None] | None = None, + ) -> Buffer | None: + if "json" not in key: + assert prototype.buffer is TestBuffer + ret = await super().get(key=key, prototype=prototype, byte_range=byte_range) + if ret is not None: + assert isinstance(ret, prototype.buffer) + return ret diff --git a/src/zarr/testing/stateful.py b/src/zarr/testing/stateful.py new file mode 100644 index 0000000000..f83d942549 --- /dev/null +++ b/src/zarr/testing/stateful.py @@ -0,0 +1,585 @@ +import builtins +from typing import Any + +import hypothesis.extra.numpy as npst +import hypothesis.strategies as st +import numpy as np +from hypothesis import assume, note +from hypothesis.stateful import ( + RuleBasedStateMachine, + initialize, + invariant, + precondition, + rule, +) +from hypothesis.strategies import DataObject + +import zarr +from zarr import Array +from zarr.abc.store import Store +from zarr.codecs.bytes import BytesCodec +from zarr.core.buffer import Buffer, BufferPrototype, cpu, default_buffer_prototype +from zarr.core.sync import SyncMixin +from zarr.storage import LocalStore, MemoryStore +from zarr.testing.strategies import ( + basic_indices, + chunk_paths, + key_ranges, + node_names, + np_array_and_chunks, + numpy_arrays, +) +from zarr.testing.strategies import keys as zarr_keys + +MAX_BINARY_SIZE = 100 + + +def split_prefix_name(path: str) -> tuple[str, str]: + split = path.rsplit("/", maxsplit=1) + if len(split) > 1: + prefix, name = split + else: + prefix = "" + (name,) = split + return prefix, name + + +class ZarrHierarchyStateMachine(SyncMixin, RuleBasedStateMachine): + """ + This state machine models operations that modify a zarr store's + hierarchy. That is, user actions that modify arrays/groups as well + as list operations. It is intended to be used by external stores, and + compares their results to a MemoryStore that is assumed to be perfect. + """ + + def __init__(self, store: Store) -> None: + super().__init__() + + self.store = store + + self.model = MemoryStore() + zarr.group(store=self.model) + + # Track state of the hierarchy, these should contain fully qualified paths + self.all_groups: set[str] = set() + self.all_arrays: set[str] = set() + + @initialize() + def init_store(self) -> None: + # This lets us reuse the fixture provided store. + self._sync(self.store.clear()) + zarr.group(store=self.store) + + def can_add(self, path: str) -> bool: + return path not in self.all_groups and path not in self.all_arrays + + # -------------------- store operations ----------------------- + @rule(name=node_names, data=st.data()) + def add_group(self, name: str, data: DataObject) -> None: + # Handle possible case-insensitive file systems (e.g. MacOS) + if isinstance(self.store, LocalStore): + name = name.lower() + if self.all_groups: + parent = data.draw(st.sampled_from(sorted(self.all_groups)), label="Group parent") + else: + parent = "" + path = f"{parent}/{name}".lstrip("/") + assume(self.can_add(path)) + note(f"Adding group: path='{path}'") + self.all_groups.add(path) + zarr.group(store=self.store, path=path) + zarr.group(store=self.model, path=path) + + @rule( + data=st.data(), + name=node_names, + array_and_chunks=np_array_and_chunks(arrays=numpy_arrays(zarr_formats=st.just(3))), + ) + def add_array( + self, + data: DataObject, + name: str, + array_and_chunks: tuple[np.ndarray[Any, Any], tuple[int, ...]], + ) -> None: + # Handle possible case-insensitive file systems (e.g. MacOS) + if isinstance(self.store, LocalStore): + name = name.lower() + array, chunks = array_and_chunks + fill_value = data.draw(npst.from_dtype(array.dtype)) + if self.all_groups: + parent = data.draw(st.sampled_from(sorted(self.all_groups)), label="Array parent") + else: + parent = "" + # TODO: support creating deeper paths + # TODO: support overwriting potentially by just skipping `self.can_add` + path = f"{parent}/{name}".lstrip("/") + assume(self.can_add(path)) + note(f"Adding array: path='{path}' shape={array.shape} chunks={chunks}") + for store in [self.store, self.model]: + zarr.array( + array, + chunks=chunks, + path=path, + store=store, + fill_value=fill_value, + # Chose bytes codec to avoid wasting time compressing the data being written + codecs=[BytesCodec()], + ) + self.all_arrays.add(path) + + @rule() + def clear(self) -> None: + note("clearing") + import zarr + + self._sync(self.store.clear()) + self._sync(self.model.clear()) + + assert self._sync(self.store.is_empty("/")) + assert self._sync(self.model.is_empty("/")) + + self.all_groups.clear() + self.all_arrays.clear() + + zarr.group(store=self.store) + zarr.group(store=self.model) + + # TODO: MemoryStore is broken? + # assert not self._sync(self.store.is_empty("/")) + # assert not self._sync(self.model.is_empty("/")) + + def draw_directory(self, data: DataObject) -> str: + group_st = st.sampled_from(sorted(self.all_groups)) if self.all_groups else st.nothing() + array_st = st.sampled_from(sorted(self.all_arrays)) if self.all_arrays else st.nothing() + array_or_group = data.draw(st.one_of(group_st, array_st)) + if data.draw(st.booleans()) and array_or_group in self.all_arrays: + arr = zarr.open_array(path=array_or_group, store=self.model) + path = data.draw( + st.one_of( + st.sampled_from([array_or_group]), + chunk_paths(ndim=arr.ndim, numblocks=arr.cdata_shape).map( + lambda x: f"{array_or_group}/c/" + ), + ) + ) + else: + path = array_or_group + return path + + @precondition(lambda self: bool(self.all_groups)) + @rule(data=st.data()) + def check_list_dir(self, data: DataObject) -> None: + path = self.draw_directory(data) + note(f"list_dir for {path=!r}") + # Consider .list_dir("path/to/array") for an array with a single chunk. + # The MemoryStore model will return `"c", "zarr.json"` only if the chunk exists + # If that chunk was deleted, then `"c"` is not returned. + # LocalStore will not have this behaviour :/ + # There are similar consistency issues with delete_dir("/path/to/array/c/0/0") + assume(not isinstance(self.store, LocalStore)) + model_ls = sorted(self._sync_iter(self.model.list_dir(path))) + store_ls = sorted(self._sync_iter(self.store.list_dir(path))) + assert model_ls == store_ls, (model_ls, store_ls) + + @precondition(lambda self: bool(self.all_arrays)) + @rule(data=st.data()) + def delete_chunk(self, data: DataObject) -> None: + array = data.draw(st.sampled_from(sorted(self.all_arrays))) + arr = zarr.open_array(path=array, store=self.model) + chunk_path = data.draw(chunk_paths(ndim=arr.ndim, numblocks=arr.cdata_shape, subset=False)) + path = f"{array}/c/{chunk_path}" + note(f"deleting chunk {path=!r}") + self._sync(self.model.delete(path)) + self._sync(self.store.delete(path)) + + @precondition(lambda self: bool(self.all_arrays)) + @rule(data=st.data()) + def overwrite_array_basic_indexing(self, data: DataObject) -> None: + array = data.draw(st.sampled_from(sorted(self.all_arrays))) + model_array = zarr.open_array(path=array, store=self.model) + store_array = zarr.open_array(path=array, store=self.store) + slicer = data.draw(basic_indices(shape=model_array.shape)) + note(f"overwriting array with basic indexer: {slicer=}") + new_data = data.draw( + npst.arrays(shape=np.shape(model_array[slicer]), dtype=model_array.dtype) + ) + model_array[slicer] = new_data + store_array[slicer] = new_data + + @precondition(lambda self: bool(self.all_arrays)) + @rule(data=st.data()) + def resize_array(self, data: DataObject) -> None: + array = data.draw(st.sampled_from(sorted(self.all_arrays))) + model_array = zarr.open_array(path=array, store=self.model) + store_array = zarr.open_array(path=array, store=self.store) + ndim = model_array.ndim + new_shape = tuple( + 0 if oldsize == 0 else newsize + for newsize, oldsize in zip( + data.draw(npst.array_shapes(max_dims=ndim, min_dims=ndim, min_side=0)), + model_array.shape, + strict=True, + ) + ) + + note(f"resizing array from {model_array.shape} to {new_shape}") + model_array.resize(new_shape) + store_array.resize(new_shape) + + @precondition(lambda self: bool(self.all_arrays) or bool(self.all_groups)) + @rule(data=st.data()) + def delete_dir(self, data: DataObject) -> None: + path = self.draw_directory(data) + note(f"delete_dir with {path=!r}") + self._sync(self.model.delete_dir(path)) + self._sync(self.store.delete_dir(path)) + + matches = set() + for node in self.all_groups | self.all_arrays: + if node.startswith(path): + matches.add(node) + self.all_groups = self.all_groups - matches + self.all_arrays = self.all_arrays - matches + + # @precondition(lambda self: bool(self.all_groups)) + # @precondition(lambda self: bool(self.all_arrays)) + # @rule(data=st.data()) + # def move_array(self, data): + # array_path = data.draw(st.sampled_from(self.all_arrays), label="Array move source") + # to_group = data.draw(st.sampled_from(self.all_groups), label="Array move destination") + + # # fixme renaiming to self? + # array_name = os.path.basename(array_path) + # assume(self.model.can_add(to_group, array_name)) + # new_path = f"{to_group}/{array_name}".lstrip("/") + # note(f"moving array '{array_path}' -> '{new_path}'") + # self.model.rename(array_path, new_path) + # self.repo.store.rename(array_path, new_path) + + # @precondition(lambda self: len(self.all_groups) >= 2) + # @rule(data=st.data()) + # def move_group(self, data): + # from_group = data.draw(st.sampled_from(self.all_groups), label="Group move source") + # to_group = data.draw(st.sampled_from(self.all_groups), label="Group move destination") + # assume(not to_group.startswith(from_group)) + + # from_group_name = os.path.basename(from_group) + # assume(self.model.can_add(to_group, from_group_name)) + # # fixme renaiming to self? + # new_path = f"{to_group}/{from_group_name}".lstrip("/") + # note(f"moving group '{from_group}' -> '{new_path}'") + # self.model.rename(from_group, new_path) + # self.repo.store.rename(from_group, new_path) + + @precondition(lambda self: self.store.supports_deletes) + @precondition(lambda self: len(self.all_arrays) >= 1) + @rule(data=st.data()) + def delete_array_using_del(self, data: DataObject) -> None: + array_path = data.draw( + st.sampled_from(sorted(self.all_arrays)), label="Array deletion target" + ) + prefix, array_name = split_prefix_name(array_path) + note(f"Deleting array '{array_path}' ({prefix=!r}, {array_name=!r}) using del") + for store in [self.model, self.store]: + group = zarr.open_group(path=prefix, store=store) + group[array_name] # check that it exists + del group[array_name] + self.all_arrays.remove(array_path) + + @precondition(lambda self: self.store.supports_deletes) + @precondition(lambda self: len(self.all_groups) >= 2) # fixme don't delete root + @rule(data=st.data()) + def delete_group_using_del(self, data: DataObject) -> None: + group_path = data.draw( + st.sampled_from(sorted(self.all_groups)), label="Group deletion target" + ) + prefix, group_name = split_prefix_name(group_path) + note(f"Deleting group '{group_path=!r}', {prefix=!r}, {group_name=!r} using delete") + members = zarr.open_group(store=self.model, path=group_path).members(max_depth=None) + for _, obj in members: + if isinstance(obj, Array): + self.all_arrays.remove(obj.path) + else: + self.all_groups.remove(obj.path) + for store in [self.store, self.model]: + group = zarr.open_group(store=store, path=prefix) + group[group_name] # check that it exists + del group[group_name] + if group_path != "/": + # The root group is always present + self.all_groups.remove(group_path) + + # # --------------- assertions ----------------- + # def check_group_arrays(self, group): + # # note(f"Checking arrays of '{group}'") + # g1 = self.model.get_group(group) + # g2 = zarr.open_group(path=group, mode="r", store=self.repo.store) + # model_arrays = sorted(g1.arrays(), key=itemgetter(0)) + # our_arrays = sorted(g2.arrays(), key=itemgetter(0)) + # for (n1, a1), (n2, a2) in zip_longest(model_arrays, our_arrays): + # assert n1 == n2 + # assert_array_equal(a1, a2) + + # def check_subgroups(self, group_path): + # g1 = self.model.get_group(group_path) + # g2 = zarr.open_group(path=group_path, mode="r", store=self.repo.store) + # g1_children = [name for (name, _) in g1.groups()] + # g2_children = [name for (name, _) in g2.groups()] + # # note(f"Checking {len(g1_children)} subgroups of group '{group_path}'") + # assert g1_children == g2_children + + # def check_list_prefix_from_group(self, group): + # prefix = f"meta/root/{group}" + # model_list = sorted(self.model.list_prefix(prefix)) + # al_list = sorted(self.repo.store.list_prefix(prefix)) + # # note(f"Checking {len(model_list)} keys under '{prefix}'") + # assert model_list == al_list + + # prefix = f"data/root/{group}" + # model_list = sorted(self.model.list_prefix(prefix)) + # al_list = sorted(self.repo.store.list_prefix(prefix)) + # # note(f"Checking {len(model_list)} keys under '{prefix}'") + # assert model_list == al_list + + # @precondition(lambda self: self.model.is_persistent_session()) + # @rule(data=st.data()) + # def check_group_path(self, data): + # t0 = time.time() + # group = data.draw(st.sampled_from(self.all_groups)) + # self.check_list_prefix_from_group(group) + # self.check_subgroups(group) + # self.check_group_arrays(group) + # t1 = time.time() + # note(f"Checks took {t1 - t0} sec.") + @invariant() + def check_list_prefix_from_root(self) -> None: + model_list = self._sync_iter(self.model.list_prefix("")) + store_list = self._sync_iter(self.store.list_prefix("")) + note(f"Checking {len(model_list)} expected keys vs {len(store_list)} actual keys") + assert sorted(model_list) == sorted(store_list), ( + sorted(model_list), + sorted(store_list), + ) + + # check that our internal state matches that of the store and model + assert all(f"{path}/zarr.json" in model_list for path in self.all_groups | self.all_arrays) + assert all(f"{path}/zarr.json" in store_list for path in self.all_groups | self.all_arrays) + + +class SyncStoreWrapper(zarr.core.sync.SyncMixin): + def __init__(self, store: Store) -> None: + """Synchronous Store wrapper + + This class holds synchronous methods that map to async methods of Store classes. + The synchronous wrapper is needed because hypothesis' stateful testing infra does + not support asyncio so we redefine sync versions of the Store API. + https://github.com/HypothesisWorks/hypothesis/issues/3712#issuecomment-1668999041 + """ + self.store = store + + @property + def read_only(self) -> bool: + return self.store.read_only + + def set(self, key: str, data_buffer: Buffer) -> None: + return self._sync(self.store.set(key, data_buffer)) + + def list(self) -> builtins.list[str]: + return self._sync_iter(self.store.list()) + + def get(self, key: str, prototype: BufferPrototype) -> Buffer | None: + return self._sync(self.store.get(key, prototype=prototype)) + + def get_partial_values( + self, key_ranges: builtins.list[Any], prototype: BufferPrototype + ) -> builtins.list[Buffer | None]: + return self._sync(self.store.get_partial_values(prototype=prototype, key_ranges=key_ranges)) + + def delete(self, path: str) -> None: + return self._sync(self.store.delete(path)) + + def is_empty(self, prefix: str) -> bool: + return self._sync(self.store.is_empty(prefix=prefix)) + + def clear(self) -> None: + return self._sync(self.store.clear()) + + def exists(self, key: str) -> bool: + return self._sync(self.store.exists(key)) + + def list_dir(self, prefix: str) -> None: + raise NotImplementedError + + def list_prefix(self, prefix: str) -> None: + raise NotImplementedError + + def set_partial_values(self, key_start_values: Any) -> None: + raise NotImplementedError + + @property + def supports_listing(self) -> bool: + return self.store.supports_listing + + @property + def supports_partial_writes(self) -> bool: + return self.supports_partial_writes + + @property + def supports_writes(self) -> bool: + return self.store.supports_writes + + @property + def supports_deletes(self) -> bool: + return self.store.supports_deletes + + +class ZarrStoreStateMachine(RuleBasedStateMachine): + """ " + Zarr store state machine + + This is a subclass of a Hypothesis RuleBasedStateMachine. + It is testing a framework to ensure that the state of a Zarr store matches + an expected state after a set of random operations. It contains a store + (currently, a Zarr MemoryStore) and a model, a simplified version of a + zarr store (in this case, a dict). It also contains rules which represent + actions that can be applied to a zarr store. Rules apply an action to both + the store and the model, and invariants assert that the state of the model + is equal to the state of the store. Hypothesis then generates sequences of + rules, running invariants after each rule. It raises an error if a sequence + produces discontinuity between state of the model and state of the store + (ie. an invariant is violated). + https://hypothesis.readthedocs.io/en/latest/stateful.html + """ + + def __init__(self, store: Store) -> None: + super().__init__() + self.model: dict[str, Buffer] = {} + self.store = SyncStoreWrapper(store) + self.prototype = default_buffer_prototype() + + @initialize() + def init_store(self) -> None: + self.store.clear() + + @rule(key=zarr_keys(), data=st.binary(min_size=0, max_size=MAX_BINARY_SIZE)) + def set(self, key: str, data: bytes) -> None: + note(f"(set) Setting {key!r} with {data!r}") + assert not self.store.read_only + data_buf = cpu.Buffer.from_bytes(data) + self.store.set(key, data_buf) + self.model[key] = data_buf + + @precondition(lambda self: len(self.model.keys()) > 0) + @rule(key=zarr_keys(), data=st.data()) + def get(self, key: str, data: DataObject) -> None: + key = data.draw( + st.sampled_from(sorted(self.model.keys())) + ) # hypothesis wants to sample from sorted list + note("(get)") + store_value = self.store.get(key, self.prototype) + # to bytes here necessary because data_buf set to model in set() + assert self.model[key] == store_value + + @rule(key=zarr_keys(), data=st.data()) + def get_invalid_zarr_keys(self, key: str, data: DataObject) -> None: + note("(get_invalid)") + assume(key not in self.model) + assert self.store.get(key, self.prototype) is None + + @precondition(lambda self: len(self.model.keys()) > 0) + @rule(data=st.data()) + def get_partial_values(self, data: DataObject) -> None: + key_range = data.draw( + key_ranges(keys=st.sampled_from(sorted(self.model.keys())), max_size=MAX_BINARY_SIZE) + ) + note(f"(get partial) {key_range=}") + obs_maybe = self.store.get_partial_values(key_range, self.prototype) + observed = [] + + for obs in obs_maybe: + assert obs is not None + observed.append(obs.to_bytes()) + + model_vals_ls = [] + + for key, byte_range in key_range: + start = byte_range.start + stop = byte_range.end + model_vals_ls.append(self.model[key][start:stop]) + + assert all( + obs == exp.to_bytes() for obs, exp in zip(observed, model_vals_ls, strict=True) + ), ( + observed, + model_vals_ls, + ) + + @precondition(lambda self: self.store.supports_deletes) + @precondition(lambda self: len(self.model.keys()) > 0) + @rule(data=st.data()) + def delete(self, data: DataObject) -> None: + key = data.draw(st.sampled_from(sorted(self.model.keys()))) + note(f"(delete) Deleting {key=}") + + self.store.delete(key) + del self.model[key] + + @rule() + def clear(self) -> None: + assert not self.store.read_only + note("(clear)") + self.store.clear() + self.model.clear() + + assert self.store.is_empty("") + + assert len(self.model.keys()) == len(list(self.store.list())) == 0 + + @rule() + # Local store can be non-empty when there are subdirectories but no files + @precondition(lambda self: not isinstance(self.store.store, LocalStore)) + def is_empty(self) -> None: + note("(is_empty)") + + # make sure they either both are or both aren't empty (same state) + assert self.store.is_empty("") == (not self.model) + + @rule(key=zarr_keys()) + def exists(self, key: str) -> None: + note("(exists)") + + assert self.store.exists(key) == (key in self.model) + + @invariant() + def check_paths_equal(self) -> None: + note("Checking that paths are equal") + paths = sorted(self.store.list()) + + assert sorted(self.model.keys()) == paths + + @invariant() + def check_vals_equal(self) -> None: + note("Checking values equal") + for key, val in self.model.items(): + store_item = self.store.get(key, self.prototype) + assert val == store_item + + @invariant() + def check_num_zarr_keys_equal(self) -> None: + note("check num zarr_keys equal") + + assert len(self.model) == len(list(self.store.list())) + + @invariant() + def check_zarr_keys(self) -> None: + keys = list(self.store.list()) + + if not keys: + assert self.store.is_empty("") is True + + else: + assert self.store.is_empty("") is False + + for key in keys: + assert self.store.exists(key) is True + note("checking keys / exists / empty") diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py new file mode 100644 index 0000000000..970329f393 --- /dev/null +++ b/src/zarr/testing/store.py @@ -0,0 +1,582 @@ +from __future__ import annotations + +import asyncio +import pickle +from abc import abstractmethod +from typing import TYPE_CHECKING, Generic, TypeVar + +from zarr.storage import WrapperStore + +if TYPE_CHECKING: + from typing import Any + + from zarr.abc.store import ByteRequest + from zarr.core.buffer.core import BufferPrototype + +import pytest + +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.core.buffer import Buffer, default_buffer_prototype +from zarr.core.sync import _collect_aiterator +from zarr.storage._utils import _normalize_byte_range_index +from zarr.testing.utils import assert_bytes_equal + +__all__ = ["StoreTests"] + + +S = TypeVar("S", bound=Store) +B = TypeVar("B", bound=Buffer) + + +class StoreTests(Generic[S, B]): + store_cls: type[S] + buffer_cls: type[B] + + @abstractmethod + async def set(self, store: S, key: str, value: Buffer) -> None: + """ + Insert a value into a storage backend, with a specific key. + This should not use any store methods. Bypassing the store methods allows them to be + tested. + """ + ... + + @abstractmethod + async def get(self, store: S, key: str) -> Buffer: + """ + Retrieve a value from a storage backend, by key. + This should not use any store methods. Bypassing the store methods allows them to be + tested. + """ + ... + + @abstractmethod + @pytest.fixture + def store_kwargs(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + """Kwargs for instantiating a store""" + ... + + @abstractmethod + def test_store_repr(self, store: S) -> None: ... + + @abstractmethod + def test_store_supports_writes(self, store: S) -> None: ... + + @abstractmethod + def test_store_supports_partial_writes(self, store: S) -> None: ... + + @abstractmethod + def test_store_supports_listing(self, store: S) -> None: ... + + @pytest.fixture + def open_kwargs(self, store_kwargs: dict[str, Any]) -> dict[str, Any]: + return store_kwargs + + @pytest.fixture + async def store(self, open_kwargs: dict[str, Any]) -> Store: + return await self.store_cls.open(**open_kwargs) + + @pytest.fixture + async def store_not_open(self, store_kwargs: dict[str, Any]) -> Store: + return self.store_cls(**store_kwargs) + + def test_store_type(self, store: S) -> None: + assert isinstance(store, Store) + assert isinstance(store, self.store_cls) + + def test_store_eq(self, store: S, store_kwargs: dict[str, Any]) -> None: + # check self equality + assert store == store + + # check store equality with same inputs + # asserting this is important for being able to compare (de)serialized stores + store2 = self.store_cls(**store_kwargs) + assert store == store2 + + async def test_serializable_store(self, store: S) -> None: + new_store: S = pickle.loads(pickle.dumps(store)) + assert new_store == store + assert new_store.read_only == store.read_only + # quickly roundtrip data to a key to test that new store works + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "foo" + await store.set(key, data_buf) + observed = await store.get(key, prototype=default_buffer_prototype()) + assert_bytes_equal(observed, data_buf) + + def test_store_read_only(self, store: S) -> None: + assert not store.read_only + + with pytest.raises(AttributeError): + store.read_only = False # type: ignore[misc] + + @pytest.mark.parametrize("read_only", [True, False]) + async def test_store_open_read_only(self, open_kwargs: dict[str, Any], read_only: bool) -> None: + open_kwargs["read_only"] = read_only + store = await self.store_cls.open(**open_kwargs) + assert store._is_open + assert store.read_only == read_only + + async def test_store_context_manager(self, open_kwargs: dict[str, Any]) -> None: + # Test that the context manager closes the store + with await self.store_cls.open(**open_kwargs) as store: + assert store._is_open + # Test trying to open an already open store + with pytest.raises(ValueError, match="store is already open"): + await store._open() + assert not store._is_open + + async def test_read_only_store_raises(self, open_kwargs: dict[str, Any]) -> None: + kwargs = {**open_kwargs, "read_only": True} + store = await self.store_cls.open(**kwargs) + assert store.read_only + + # set + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await store.set("foo", self.buffer_cls.from_bytes(b"bar")) + + # delete + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await store.delete("foo") + + async def test_with_read_only_store(self, open_kwargs: dict[str, Any]) -> None: + kwargs = {**open_kwargs, "read_only": True} + store = await self.store_cls.open(**kwargs) + assert store.read_only + + # Test that you cannot write to a read-only store + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await store.set("foo", self.buffer_cls.from_bytes(b"bar")) + + # Check if the store implements with_read_only + try: + writer = store.with_read_only(read_only=False) + except NotImplementedError: + # Test that stores that do not implement with_read_only raise NotImplementedError with the correct message + with pytest.raises( + NotImplementedError, + match=f"with_read_only is not implemented for the {type(store)} store type.", + ): + store.with_read_only(read_only=False) + return + + # Test that you can write to a new store copy + assert not writer._is_open + assert not writer.read_only + await writer.set("foo", self.buffer_cls.from_bytes(b"bar")) + await writer.delete("foo") + + # Test that you cannot write to the original store + assert store.read_only + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await store.set("foo", self.buffer_cls.from_bytes(b"bar")) + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await store.delete("foo") + + # Test that you cannot write to a read-only store copy + reader = store.with_read_only(read_only=True) + assert reader.read_only + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await reader.set("foo", self.buffer_cls.from_bytes(b"bar")) + with pytest.raises( + ValueError, match="store was opened in read-only mode and does not support writing" + ): + await reader.delete("foo") + + @pytest.mark.parametrize("key", ["c/0", "foo/c/0.0", "foo/0/0"]) + @pytest.mark.parametrize( + ("data", "byte_range"), + [ + (b"\x01\x02\x03\x04", None), + (b"\x01\x02\x03\x04", RangeByteRequest(1, 4)), + (b"\x01\x02\x03\x04", OffsetByteRequest(1)), + (b"\x01\x02\x03\x04", SuffixByteRequest(1)), + (b"", None), + ], + ) + async def test_get(self, store: S, key: str, data: bytes, byte_range: ByteRequest) -> None: + """ + Ensure that data can be read from the store using the store.get method. + """ + data_buf = self.buffer_cls.from_bytes(data) + await self.set(store, key, data_buf) + observed = await store.get(key, prototype=default_buffer_prototype(), byte_range=byte_range) + start, stop = _normalize_byte_range_index(data_buf, byte_range=byte_range) + expected = data_buf[start:stop] + assert_bytes_equal(observed, expected) + + async def test_get_not_open(self, store_not_open: S) -> None: + """ + Ensure that data can be read from the store that isn't yet open using the store.get method. + """ + assert not store_not_open._is_open + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "c/0" + await self.set(store_not_open, key, data_buf) + observed = await store_not_open.get(key, prototype=default_buffer_prototype()) + assert_bytes_equal(observed, data_buf) + + async def test_get_raises(self, store: S) -> None: + """ + Ensure that a ValueError is raise for invalid byte range syntax + """ + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + await self.set(store, "c/0", data_buf) + with pytest.raises((ValueError, TypeError), match=r"Unexpected byte_range, got.*"): + await store.get("c/0", prototype=default_buffer_prototype(), byte_range=(0, 2)) # type: ignore[arg-type] + + async def test_get_many(self, store: S) -> None: + """ + Ensure that multiple keys can be retrieved at once with the _get_many method. + """ + keys = tuple(map(str, range(10))) + values = tuple(f"{k}".encode() for k in keys) + for k, v in zip(keys, values, strict=False): + await self.set(store, k, self.buffer_cls.from_bytes(v)) + observed_buffers = await _collect_aiterator( + store._get_many( + zip( + keys, + (default_buffer_prototype(),) * len(keys), + (None,) * len(keys), + strict=False, + ) + ) + ) + observed_kvs = sorted(((k, b.to_bytes()) for k, b in observed_buffers)) # type: ignore[union-attr] + expected_kvs = sorted(((k, b) for k, b in zip(keys, values, strict=False))) + assert observed_kvs == expected_kvs + + @pytest.mark.parametrize("key", ["c/0", "foo/c/0.0", "foo/0/0"]) + @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) + async def test_getsize(self, store: S, key: str, data: bytes) -> None: + """ + Test the result of store.getsize(). + """ + data_buf = self.buffer_cls.from_bytes(data) + expected = len(data_buf) + await self.set(store, key, data_buf) + observed = await store.getsize(key) + assert observed == expected + + async def test_getsize_prefix(self, store: S) -> None: + """ + Test the result of store.getsize_prefix(). + """ + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + keys = ["c/0/0", "c/0/1", "c/1/0", "c/1/1"] + keys_values = [(k, data_buf) for k in keys] + await store._set_many(keys_values) + expected = len(data_buf) * len(keys) + observed = await store.getsize_prefix("c") + assert observed == expected + + async def test_getsize_raises(self, store: S) -> None: + """ + Test that getsize() raise a FileNotFoundError if the key doesn't exist. + """ + with pytest.raises(FileNotFoundError): + await store.getsize("c/1000") + + @pytest.mark.parametrize("key", ["zarr.json", "c/0", "foo/c/0.0", "foo/0/0"]) + @pytest.mark.parametrize("data", [b"\x01\x02\x03\x04", b""]) + async def test_set(self, store: S, key: str, data: bytes) -> None: + """ + Ensure that data can be written to the store using the store.set method. + """ + assert not store.read_only + data_buf = self.buffer_cls.from_bytes(data) + await store.set(key, data_buf) + observed = await self.get(store, key) + assert_bytes_equal(observed, data_buf) + + async def test_set_not_open(self, store_not_open: S) -> None: + """ + Ensure that data can be written to the store that's not yet open using the store.set method. + """ + assert not store_not_open._is_open + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "c/0" + await store_not_open.set(key, data_buf) + observed = await self.get(store_not_open, key) + assert_bytes_equal(observed, data_buf) + + async def test_set_many(self, store: S) -> None: + """ + Test that a dict of key : value pairs can be inserted into the store via the + `_set_many` method. + """ + keys = ["zarr.json", "c/0", "foo/c/0.0", "foo/0/0"] + data_buf = [self.buffer_cls.from_bytes(k.encode()) for k in keys] + store_dict = dict(zip(keys, data_buf, strict=True)) + await store._set_many(store_dict.items()) + for k, v in store_dict.items(): + assert (await self.get(store, k)).to_bytes() == v.to_bytes() + + @pytest.mark.parametrize( + "key_ranges", + [ + [], + [("zarr.json", RangeByteRequest(0, 2))], + [("c/0", RangeByteRequest(0, 2)), ("zarr.json", None)], + [ + ("c/0/0", RangeByteRequest(0, 2)), + ("c/0/1", SuffixByteRequest(2)), + ("c/0/2", OffsetByteRequest(2)), + ], + ], + ) + async def test_get_partial_values( + self, store: S, key_ranges: list[tuple[str, ByteRequest]] + ) -> None: + # put all of the data + for key, _ in key_ranges: + await self.set(store, key, self.buffer_cls.from_bytes(bytes(key, encoding="utf-8"))) + + # read back just part of it + observed_maybe = await store.get_partial_values( + prototype=default_buffer_prototype(), key_ranges=key_ranges + ) + + observed: list[Buffer] = [] + expected: list[Buffer] = [] + + for obs in observed_maybe: + assert obs is not None + observed.append(obs) + + for idx in range(len(observed)): + key, byte_range = key_ranges[idx] + result = await store.get( + key, prototype=default_buffer_prototype(), byte_range=byte_range + ) + assert result is not None + expected.append(result) + + assert all( + obs.to_bytes() == exp.to_bytes() for obs, exp in zip(observed, expected, strict=True) + ) + + async def test_exists(self, store: S) -> None: + assert not await store.exists("foo") + await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) + assert await store.exists("foo/zarr.json") + + async def test_delete(self, store: S) -> None: + if not store.supports_deletes: + pytest.skip("store does not support deletes") + await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) + assert await store.exists("foo/zarr.json") + await store.delete("foo/zarr.json") + assert not await store.exists("foo/zarr.json") + + async def test_delete_dir(self, store: S) -> None: + if not store.supports_deletes: + pytest.skip("store does not support deletes") + await store.set("zarr.json", self.buffer_cls.from_bytes(b"root")) + await store.set("foo-bar/zarr.json", self.buffer_cls.from_bytes(b"root")) + await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) + await store.set("foo/c/0", self.buffer_cls.from_bytes(b"chunk")) + await store.delete_dir("foo") + assert await store.exists("zarr.json") + assert await store.exists("foo-bar/zarr.json") + assert not await store.exists("foo/zarr.json") + assert not await store.exists("foo/c/0") + + async def test_is_empty(self, store: S) -> None: + assert await store.is_empty("") + await self.set( + store, "foo/bar", self.buffer_cls.from_bytes(bytes("something", encoding="utf-8")) + ) + assert not await store.is_empty("") + assert await store.is_empty("fo") + assert not await store.is_empty("foo/") + assert not await store.is_empty("foo") + assert await store.is_empty("spam/") + + async def test_clear(self, store: S) -> None: + await self.set( + store, "key", self.buffer_cls.from_bytes(bytes("something", encoding="utf-8")) + ) + await store.clear() + assert await store.is_empty("") + + async def test_list(self, store: S) -> None: + assert await _collect_aiterator(store.list()) == () + prefix = "foo" + data = self.buffer_cls.from_bytes(b"") + store_dict = { + prefix + "/zarr.json": data, + **{prefix + f"/c/{idx}": data for idx in range(10)}, + } + await store._set_many(store_dict.items()) + expected_sorted = sorted(store_dict.keys()) + observed = await _collect_aiterator(store.list()) + observed_sorted = sorted(observed) + assert observed_sorted == expected_sorted + + async def test_list_prefix(self, store: S) -> None: + """ + Test that the `list_prefix` method works as intended. Given a prefix, it should return + all the keys in storage that start with this prefix. + """ + prefixes = ("", "a/", "a/b/", "a/b/c/") + data = self.buffer_cls.from_bytes(b"") + fname = "zarr.json" + store_dict = {p + fname: data for p in prefixes} + + await store._set_many(store_dict.items()) + + for prefix in prefixes: + observed = tuple(sorted(await _collect_aiterator(store.list_prefix(prefix)))) + expected: tuple[str, ...] = () + for key in store_dict: + if key.startswith(prefix): + expected += (key,) + expected = tuple(sorted(expected)) + assert observed == expected + + async def test_list_empty_path(self, store: S) -> None: + """ + Verify that list and list_prefix work correctly when path is an empty string, + i.e. no unwanted replacement occurs. + """ + data = self.buffer_cls.from_bytes(b"") + store_dict = { + "foo/bar/zarr.json": data, + "foo/bar/c/1": data, + "foo/baz/c/0": data, + } + await store._set_many(store_dict.items()) + + # Test list() + observed_list = await _collect_aiterator(store.list()) + observed_list_sorted = sorted(observed_list) + expected_list_sorted = sorted(store_dict.keys()) + assert observed_list_sorted == expected_list_sorted + + # Test list_prefix() with an empty prefix + observed_prefix_empty = await _collect_aiterator(store.list_prefix("")) + observed_prefix_empty_sorted = sorted(observed_prefix_empty) + expected_prefix_empty_sorted = sorted(store_dict.keys()) + assert observed_prefix_empty_sorted == expected_prefix_empty_sorted + + # Test list_prefix() with a non-empty prefix + observed_prefix = await _collect_aiterator(store.list_prefix("foo/bar/")) + observed_prefix_sorted = sorted(observed_prefix) + expected_prefix_sorted = sorted(k for k in store_dict if k.startswith("foo/bar/")) + assert observed_prefix_sorted == expected_prefix_sorted + + async def test_list_dir(self, store: S) -> None: + root = "foo" + store_dict = { + root + "/zarr.json": self.buffer_cls.from_bytes(b"bar"), + root + "/c/1": self.buffer_cls.from_bytes(b"\x01"), + } + + assert await _collect_aiterator(store.list_dir("")) == () + assert await _collect_aiterator(store.list_dir(root)) == () + + await store._set_many(store_dict.items()) + + keys_observed = await _collect_aiterator(store.list_dir(root)) + keys_expected = {k.removeprefix(root + "/").split("/")[0] for k in store_dict} + + assert sorted(keys_observed) == sorted(keys_expected) + + keys_observed = await _collect_aiterator(store.list_dir(root + "/")) + assert sorted(keys_expected) == sorted(keys_observed) + + async def test_set_if_not_exists(self, store: S) -> None: + key = "k" + data_buf = self.buffer_cls.from_bytes(b"0000") + await self.set(store, key, data_buf) + + new = self.buffer_cls.from_bytes(b"1111") + await store.set_if_not_exists("k", new) # no error + + result = await store.get(key, default_buffer_prototype()) + assert result == data_buf + + await store.set_if_not_exists("k2", new) # no error + + result = await store.get("k2", default_buffer_prototype()) + assert result == new + + +class LatencyStore(WrapperStore[Store]): + """ + A wrapper class that takes any store class in its constructor and + adds latency to the `set` and `get` methods. This can be used for + performance testing. + """ + + get_latency: float + set_latency: float + + def __init__(self, cls: Store, *, get_latency: float = 0, set_latency: float = 0) -> None: + self.get_latency = float(get_latency) + self.set_latency = float(set_latency) + self._store = cls + + async def set(self, key: str, value: Buffer) -> None: + """ + Add latency to the ``set`` method. + + Calls ``asyncio.sleep(self.set_latency)`` before invoking the wrapped ``set`` method. + + Parameters + ---------- + key : str + The key to set + value : Buffer + The value to set + + Returns + ------- + None + """ + await asyncio.sleep(self.set_latency) + await self._store.set(key, value) + + async def get( + self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None + ) -> Buffer | None: + """ + Add latency to the ``get`` method. + + Calls ``asyncio.sleep(self.get_latency)`` before invoking the wrapped ``get`` method. + + Parameters + ---------- + key : str + The key to get + prototype : BufferPrototype + The BufferPrototype to use. + byte_range : ByteRequest, optional + An optional byte range. + + Returns + ------- + buffer : Buffer or None + """ + await asyncio.sleep(self.get_latency) + return await self._store.get(key, prototype=prototype, byte_range=byte_range) diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py new file mode 100644 index 0000000000..5e070b5387 --- /dev/null +++ b/src/zarr/testing/strategies.py @@ -0,0 +1,441 @@ +import math +import sys +from collections.abc import Callable, Mapping +from typing import Any, Literal + +import hypothesis.extra.numpy as npst +import hypothesis.strategies as st +import numpy as np +import numpy.typing as npt +from hypothesis import event +from hypothesis.strategies import SearchStrategy + +import zarr +from zarr.abc.store import RangeByteRequest, Store +from zarr.codecs.bytes import BytesCodec +from zarr.core.array import Array +from zarr.core.chunk_grids import RegularChunkGrid +from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding +from zarr.core.common import JSON, ZarrFormat +from zarr.core.dtype import get_data_type_from_native_dtype +from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata +from zarr.core.sync import sync +from zarr.storage import MemoryStore, StoreLike +from zarr.storage._common import _dereference_path +from zarr.storage._utils import normalize_path + +# Copied from Xarray +_attr_keys = st.text(st.characters(), min_size=1) +_attr_values = st.recursive( + st.none() | st.booleans() | st.text(st.characters(), max_size=5), + lambda children: st.lists(children) | st.dictionaries(_attr_keys, children), + max_leaves=3, +) + + +@st.composite +def keys(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> str: + return draw(st.lists(node_names, min_size=1, max_size=max_num_nodes).map("/".join)) + + +@st.composite +def paths(draw: st.DrawFn, *, max_num_nodes: int | None = None) -> str: + return draw(st.just("/") | keys(max_num_nodes=max_num_nodes)) + + +def v3_dtypes() -> st.SearchStrategy[np.dtype[Any]]: + return ( + npst.boolean_dtypes() + | npst.integer_dtypes(endianness="=") + | npst.unsigned_integer_dtypes(endianness="=") + | npst.floating_dtypes(endianness="=") + | npst.complex_number_dtypes(endianness="=") + | npst.byte_string_dtypes(endianness="=") + | npst.unicode_string_dtypes(endianness="=") + | npst.datetime64_dtypes(endianness="=") + | npst.timedelta64_dtypes(endianness="=") + ) + + +def v2_dtypes() -> st.SearchStrategy[np.dtype[Any]]: + return ( + npst.boolean_dtypes() + | npst.integer_dtypes(endianness="=") + | npst.unsigned_integer_dtypes(endianness="=") + | npst.floating_dtypes(endianness="=") + | npst.complex_number_dtypes(endianness="=") + | npst.byte_string_dtypes(endianness="=") + | npst.unicode_string_dtypes(endianness="=") + | npst.datetime64_dtypes(endianness="=") + | npst.timedelta64_dtypes(endianness="=") + ) + + +def safe_unicode_for_dtype(dtype: np.dtype[np.str_]) -> st.SearchStrategy[str]: + """Generate UTF-8-safe text constrained to max_len of dtype.""" + # account for utf-32 encoding (i.e. 4 bytes/character) + max_len = max(1, dtype.itemsize // 4) + + return st.text( + alphabet=st.characters( + exclude_categories=["Cs"], # Avoid *technically allowed* surrogates + min_codepoint=32, + ), + min_size=1, + max_size=max_len, + ) + + +def clear_store(x: Store) -> Store: + sync(x.clear()) + return x + + +# From https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#node-names +# 1. must not be the empty string ("") +# 2. must not include the character "/" +# 3. must not be a string composed only of period characters, e.g. "." or ".." +# 4. must not start with the reserved prefix "__" +zarr_key_chars = st.sampled_from( + ".-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" +) +node_names = ( + st.text(zarr_key_chars, min_size=1) + .filter(lambda t: t not in (".", "..") and not t.startswith("__")) + .filter(lambda name: name.lower() != "zarr.json") +) +short_node_names = ( + st.text(zarr_key_chars, max_size=3, min_size=1) + .filter(lambda t: t not in (".", "..") and not t.startswith("__")) + .filter(lambda name: name.lower() != "zarr.json") +) +array_names = node_names +attrs: st.SearchStrategy[Mapping[str, JSON] | None] = st.none() | st.dictionaries( + _attr_keys, _attr_values +) +# st.builds will only call a new store constructor for different keyword arguments +# i.e. stores.examples() will always return the same object per Store class. +# So we map a clear to reset the store. +stores = st.builds(MemoryStore, st.just({})).map(clear_store) +compressors = st.sampled_from([None, "default"]) +zarr_formats: st.SearchStrategy[ZarrFormat] = st.sampled_from([3, 2]) +# We de-prioritize arrays having dim sizes 0, 1, 2 +array_shapes = npst.array_shapes(max_dims=4, min_side=3, max_side=5) | npst.array_shapes( + max_dims=4, min_side=0 +) + + +@st.composite +def dimension_names(draw: st.DrawFn, *, ndim: int | None = None) -> list[None | str] | None: + simple_text = st.text(zarr_key_chars, min_size=0) + return draw(st.none() | st.lists(st.none() | simple_text, min_size=ndim, max_size=ndim)) # type: ignore[arg-type] + + +@st.composite +def array_metadata( + draw: st.DrawFn, + *, + array_shapes: Callable[..., st.SearchStrategy[tuple[int, ...]]] = npst.array_shapes, + zarr_formats: st.SearchStrategy[Literal[2, 3]] = zarr_formats, + attributes: SearchStrategy[Mapping[str, JSON] | None] = attrs, +) -> ArrayV2Metadata | ArrayV3Metadata: + zarr_format = draw(zarr_formats) + # separator = draw(st.sampled_from(['/', '\\'])) + shape = draw(array_shapes()) + ndim = len(shape) + chunk_shape = draw(array_shapes(min_dims=ndim, max_dims=ndim)) + np_dtype = draw(v3_dtypes()) + dtype = get_data_type_from_native_dtype(np_dtype) + fill_value = draw(npst.from_dtype(np_dtype)) + if zarr_format == 2: + return ArrayV2Metadata( + shape=shape, + chunks=chunk_shape, + dtype=dtype, + fill_value=fill_value, + order=draw(st.sampled_from(["C", "F"])), + attributes=draw(attributes), # type: ignore[arg-type] + dimension_separator=draw(st.sampled_from([".", "/"])), + filters=None, + compressor=None, + ) + else: + return ArrayV3Metadata( + shape=shape, + data_type=dtype, + chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape), + fill_value=fill_value, + attributes=draw(attributes), # type: ignore[arg-type] + dimension_names=draw(dimension_names(ndim=ndim)), + chunk_key_encoding=DefaultChunkKeyEncoding(separator="/"), # FIXME + codecs=[BytesCodec()], + storage_transformers=(), + ) + + +@st.composite +def numpy_arrays( + draw: st.DrawFn, + *, + shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, + dtype: np.dtype[Any] | None = None, + zarr_formats: st.SearchStrategy[ZarrFormat] = zarr_formats, +) -> npt.NDArray[Any]: + """ + Generate numpy arrays that can be saved in the provided Zarr format. + """ + zarr_format = draw(zarr_formats) + if dtype is None: + dtype = draw(v3_dtypes() if zarr_format == 3 else v2_dtypes()) + if np.issubdtype(dtype, np.str_): + safe_unicode_strings = safe_unicode_for_dtype(dtype) + return draw(npst.arrays(dtype=dtype, shape=shapes, elements=safe_unicode_strings)) + + return draw(npst.arrays(dtype=dtype, shape=shapes)) + + +@st.composite +def chunk_shapes(draw: st.DrawFn, *, shape: tuple[int, ...]) -> tuple[int, ...]: + # We want this strategy to shrink towards arrays with smaller number of chunks + # 1. st.integers() shrinks towards smaller values. So we use that to generate number of chunks + numchunks = draw( + st.tuples(*[st.integers(min_value=0 if size == 0 else 1, max_value=size) for size in shape]) + ) + # 2. and now generate the chunks tuple + chunks = tuple( + size // nchunks if nchunks > 0 else 0 + for size, nchunks in zip(shape, numchunks, strict=True) + ) + + for c in chunks: + event("chunk size", c) + + if any((c != 0 and s % c != 0) for s, c in zip(shape, chunks, strict=True)): + event("smaller last chunk") + + return chunks + + +@st.composite +def shard_shapes( + draw: st.DrawFn, *, shape: tuple[int, ...], chunk_shape: tuple[int, ...] +) -> tuple[int, ...]: + # We want this strategy to shrink towards arrays with smaller number of shards + # shards must be an integral number of chunks + assert all(c != 0 for c in chunk_shape) + numchunks = tuple(s // c for s, c in zip(shape, chunk_shape, strict=True)) + multiples = tuple(draw(st.integers(min_value=1, max_value=nc)) for nc in numchunks) + return tuple(m * c for m, c in zip(multiples, chunk_shape, strict=True)) + + +@st.composite +def np_array_and_chunks( + draw: st.DrawFn, + *, + arrays: st.SearchStrategy[npt.NDArray[Any]] = numpy_arrays(), # noqa: B008 +) -> tuple[np.ndarray, tuple[int, ...]]: # type: ignore[type-arg] + """A hypothesis strategy to generate small sized random arrays. + + Returns: a tuple of the array and a suitable random chunking for it. + """ + array = draw(arrays) + return (array, draw(chunk_shapes(shape=array.shape))) + + +@st.composite +def arrays( + draw: st.DrawFn, + *, + shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, + compressors: st.SearchStrategy = compressors, + stores: st.SearchStrategy[StoreLike] = stores, + paths: st.SearchStrategy[str] = paths(), # noqa: B008 + array_names: st.SearchStrategy = array_names, + arrays: st.SearchStrategy | None = None, + attrs: st.SearchStrategy = attrs, + zarr_formats: st.SearchStrategy = zarr_formats, +) -> Array: + store = draw(stores) + path = draw(paths) + name = draw(array_names) + attributes = draw(attrs) + zarr_format = draw(zarr_formats) + if arrays is None: + arrays = numpy_arrays(shapes=shapes, zarr_formats=st.just(zarr_format)) + nparray = draw(arrays) + chunk_shape = draw(chunk_shapes(shape=nparray.shape)) + if zarr_format == 3 and all(c > 0 for c in chunk_shape): + shard_shape = draw(st.none() | shard_shapes(shape=nparray.shape, chunk_shape=chunk_shape)) + else: + shard_shape = None + # test that None works too. + fill_value = draw(st.one_of([st.none(), npst.from_dtype(nparray.dtype)])) + # compressor = draw(compressors) + + expected_attrs = {} if attributes is None else attributes + + array_path = _dereference_path(path, name) + root = zarr.open_group(store, mode="w", zarr_format=zarr_format) + + a = root.create_array( + array_path, + shape=nparray.shape, + chunks=chunk_shape, + shards=shard_shape, + dtype=nparray.dtype, + attributes=attributes, + # compressor=compressor, # FIXME + fill_value=fill_value, + ) + + assert isinstance(a, Array) + if a.metadata.zarr_format == 3: + assert a.fill_value is not None + assert a.name is not None + assert a.path == normalize_path(array_path) + assert a.name == "/" + a.path + assert isinstance(root[array_path], Array) + assert nparray.shape == a.shape + assert chunk_shape == a.chunks + assert shard_shape == a.shards + assert a.basename == name, (a.basename, name) + assert dict(a.attrs) == expected_attrs + + a[:] = nparray + + return a + + +@st.composite +def simple_arrays( + draw: st.DrawFn, + *, + shapes: st.SearchStrategy[tuple[int, ...]] = array_shapes, +) -> Any: + return draw( + arrays( + shapes=shapes, + paths=paths(max_num_nodes=2), + array_names=short_node_names, + attrs=st.none(), + compressors=st.sampled_from([None, "default"]), + ) + ) + + +def is_negative_slice(idx: Any) -> bool: + return isinstance(idx, slice) and idx.step is not None and idx.step < 0 + + +@st.composite +def end_slices(draw: st.DrawFn, *, shape: tuple[int, ...]) -> Any: + """ + A strategy that slices ranges that include the last chunk. + This is intended to stress-test handling of a possibly smaller last chunk. + """ + slicers = [] + for size in shape: + start = draw(st.integers(min_value=size // 2, max_value=size - 1)) + length = draw(st.integers(min_value=0, max_value=size - start)) + slicers.append(slice(start, start + length)) + event("drawing end slice") + return tuple(slicers) + + +@st.composite +def basic_indices( + draw: st.DrawFn, + *, + shape: tuple[int, ...], + min_dims: int = 0, + max_dims: int | None = None, + allow_newaxis: bool = False, + allow_ellipsis: bool = True, +) -> Any: + """Basic indices without unsupported negative slices.""" + strategy = npst.basic_indices( + shape=shape, + min_dims=min_dims, + max_dims=max_dims, + allow_newaxis=allow_newaxis, + allow_ellipsis=allow_ellipsis, + ).filter( + lambda idxr: ( + not ( + is_negative_slice(idxr) + or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr)) # type: ignore[redundant-expr] + ) + ) + ) + if math.prod(shape) >= 3: + strategy = end_slices(shape=shape) | strategy + return draw(strategy) + + +@st.composite +def orthogonal_indices( + draw: st.DrawFn, *, shape: tuple[int, ...] +) -> tuple[tuple[np.ndarray[Any, Any], ...], tuple[np.ndarray[Any, Any], ...]]: + """ + Strategy that returns + (1) a tuple of integer arrays used for orthogonal indexing of Zarr arrays. + (2) an tuple of integer arrays that can be used for equivalent indexing of numpy arrays + """ + zindexer = [] + npindexer = [] + ndim = len(shape) + for axis, size in enumerate(shape): + val = draw( + npst.integer_array_indices( + shape=(size,), result_shape=npst.array_shapes(min_side=1, max_side=size, max_dims=1) + ) + | basic_indices(min_dims=1, shape=(size,), allow_ellipsis=False) + .map(lambda x: (x,) if not isinstance(x, tuple) else x) # bare ints, slices + .filter(bool) # skip empty tuple + ) + (idxr,) = val + if isinstance(idxr, int): + idxr = np.array([idxr]) + zindexer.append(idxr) + if isinstance(idxr, slice): + idxr = np.arange(*idxr.indices(size)) + elif isinstance(idxr, (tuple, int)): + idxr = np.array(idxr) + newshape = [1] * ndim + newshape[axis] = idxr.size + npindexer.append(idxr.reshape(newshape)) + + # casting the output of broadcast_arrays is needed for numpy 1.25 + return tuple(zindexer), tuple(np.broadcast_arrays(*npindexer)) + + +def key_ranges( + keys: SearchStrategy[str] = node_names, max_size: int = sys.maxsize +) -> SearchStrategy[list[tuple[str, RangeByteRequest]]]: + """ + Function to generate key_ranges strategy for get_partial_values() + returns list strategy w/ form:: + + [(key, (range_start, range_end)), + (key, (range_start, range_end)),...] + """ + + def make_request(start: int, length: int) -> RangeByteRequest: + return RangeByteRequest(start, end=min(start + length, max_size)) + + byte_ranges = st.builds( + make_request, + start=st.integers(min_value=0, max_value=max_size), + length=st.integers(min_value=0, max_value=max_size), + ) + key_tuple = st.tuples(keys, byte_ranges) + return st.lists(key_tuple, min_size=1, max_size=10) + + +@st.composite +def chunk_paths(draw: st.DrawFn, ndim: int, numblocks: tuple[int, ...], subset: bool = True) -> str: + blockidx = draw( + st.tuples(*tuple(st.integers(min_value=0, max_value=max(0, b - 1)) for b in numblocks)) + ) + subset_slicer = slice(draw(st.integers(min_value=0, max_value=ndim))) if subset else slice(None) + return "/".join(map(str, blockidx[subset_slicer])) diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py new file mode 100644 index 0000000000..afc15d742c --- /dev/null +++ b/src/zarr/testing/utils.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, TypeVar, cast + +import pytest + +from zarr.core.buffer import Buffer + +if TYPE_CHECKING: + from zarr.core.common import BytesLike + +__all__ = ["assert_bytes_equal"] + + +def assert_bytes_equal(b1: Buffer | BytesLike | None, b2: Buffer | BytesLike | None) -> None: + """Help function to assert if two bytes-like or Buffers are equal + + Warnings + -------- + Always copies data, only use for testing and debugging + """ + if isinstance(b1, Buffer): + b1 = b1.to_bytes() + if isinstance(b2, Buffer): + b2 = b2.to_bytes() + assert b1 == b2 + + +def has_cupy() -> bool: + try: + import cupy + + return cast("bool", cupy.cuda.runtime.getDeviceCount() > 0) + except ImportError: + return False + except cupy.cuda.runtime.CUDARuntimeError: + return False + + +T = TypeVar("T") + + +# Decorator for GPU tests +def gpu_test(func: T) -> T: + return cast( + "T", + pytest.mark.gpu( + pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available")( + func + ) + ), + ) diff --git a/docs/_static/donotdelete b/tests/__init__.py similarity index 100% rename from docs/_static/donotdelete rename to tests/__init__.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..4d300a1fd4 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,444 @@ +from __future__ import annotations + +import os +import pathlib +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import numpy as np +import numpy.typing as npt +import pytest +from hypothesis import HealthCheck, Verbosity, settings + +from zarr import AsyncGroup, config +from zarr.abc.store import Store +from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation +from zarr.core.array import ( + _parse_chunk_encoding_v2, + _parse_chunk_encoding_v3, + _parse_chunk_key_encoding, +) +from zarr.core.chunk_grids import RegularChunkGrid, _auto_partition +from zarr.core.common import JSON, DimensionNames, parse_shapelike +from zarr.core.config import config as zarr_config +from zarr.core.dtype import ( + get_data_type_from_native_dtype, +) +from zarr.core.dtype.common import HasItemSize +from zarr.core.metadata.v2 import ArrayV2Metadata +from zarr.core.metadata.v3 import ArrayV3Metadata +from zarr.core.sync import sync +from zarr.storage import FsspecStore, LocalStore, MemoryStore, StorePath, ZipStore + +if TYPE_CHECKING: + from collections.abc import Generator + from typing import Any, Literal + + from _pytest.compat import LEGACY_PATH + + from zarr.abc.codec import Codec + from zarr.core.array import CompressorsLike, FiltersLike, SerializerLike, ShardsLike + from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike + from zarr.core.common import ChunkCoords, MemoryOrder, ShapeLike, ZarrFormat + from zarr.core.dtype.wrapper import ZDType + + +async def parse_store( + store: Literal["local", "memory", "fsspec", "zip"], path: str +) -> LocalStore | MemoryStore | FsspecStore | ZipStore: + if store == "local": + return await LocalStore.open(path) + if store == "memory": + return await MemoryStore.open() + if store == "fsspec": + return await FsspecStore.open(url=path) + if store == "zip": + return await ZipStore.open(path + "/zarr.zip", mode="w") + raise AssertionError + + +@pytest.fixture(params=[str, pathlib.Path]) +def path_type(request: pytest.FixtureRequest) -> Any: + return request.param + + +# todo: harmonize this with local_store fixture +@pytest.fixture +async def store_path(tmpdir: LEGACY_PATH) -> StorePath: + store = await LocalStore.open(str(tmpdir)) + return StorePath(store) + + +@pytest.fixture +async def local_store(tmpdir: LEGACY_PATH) -> LocalStore: + return await LocalStore.open(str(tmpdir)) + + +@pytest.fixture +async def remote_store(url: str) -> FsspecStore: + return await FsspecStore.open(url) + + +@pytest.fixture +async def memory_store() -> MemoryStore: + return await MemoryStore.open() + + +@pytest.fixture +async def zip_store(tmpdir: LEGACY_PATH) -> ZipStore: + return await ZipStore.open(str(tmpdir / "zarr.zip"), mode="w") + + +@pytest.fixture +async def store(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> Store: + param = request.param + return await parse_store(param, str(tmpdir)) + + +@pytest.fixture +async def store2(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> Store: + """Fixture to create a second store for testing copy operations between stores""" + param = request.param + store2_path = tmpdir.mkdir("store2") + return await parse_store(param, str(store2_path)) + + +@pytest.fixture(params=["local", "memory", "zip"]) +def sync_store(request: pytest.FixtureRequest, tmp_path: LEGACY_PATH) -> Store: + result = sync(parse_store(request.param, str(tmp_path))) + if not isinstance(result, Store): + raise TypeError("Wrong store class returned by test fixture! got " + result + " instead") + return result + + +@dataclass +class AsyncGroupRequest: + zarr_format: ZarrFormat + store: Literal["local", "fsspec", "memory", "zip"] + attributes: dict[str, Any] = field(default_factory=dict) + + +@pytest.fixture +async def async_group(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> AsyncGroup: + param: AsyncGroupRequest = request.param + + store = await parse_store(param.store, str(tmpdir)) + return await AsyncGroup.from_store( + store, + attributes=param.attributes, + zarr_format=param.zarr_format, + overwrite=False, + ) + + +@pytest.fixture(params=["numpy", "cupy"]) +def xp(request: pytest.FixtureRequest) -> Any: + """Fixture to parametrize over numpy-like libraries""" + + if request.param == "cupy": + request.node.add_marker(pytest.mark.gpu) + + return pytest.importorskip(request.param) + + +@pytest.fixture(autouse=True) +def reset_config() -> Generator[None, None, None]: + config.reset() + yield + config.reset() + + +@dataclass +class ArrayRequest: + shape: ChunkCoords + dtype: str + order: MemoryOrder + + +@pytest.fixture +def array_fixture(request: pytest.FixtureRequest) -> npt.NDArray[Any]: + array_request: ArrayRequest = request.param + return ( + np.arange(np.prod(array_request.shape)) + .reshape(array_request.shape, order=array_request.order) + .astype(array_request.dtype) + ) + + +@pytest.fixture(params=(2, 3), ids=["zarr2", "zarr3"]) +def zarr_format(request: pytest.FixtureRequest) -> ZarrFormat: + if request.param == 2: + return 2 + elif request.param == 3: + return 3 + msg = f"Invalid zarr format requested. Got {request.param}, expected on of (2,3)." + raise ValueError(msg) + + +def pytest_addoption(parser: Any) -> None: + parser.addoption( + "--run-slow-hypothesis", + action="store_true", + default=False, + help="run slow hypothesis tests", + ) + + +def pytest_collection_modifyitems(config: Any, items: Any) -> None: + if config.getoption("--run-slow-hypothesis"): + return + skip_slow_hyp = pytest.mark.skip(reason="need --run-slow-hypothesis option to run") + for item in items: + if "slow_hypothesis" in item.keywords: + item.add_marker(skip_slow_hyp) + + +settings.register_profile( + "default", + parent=settings.get_profile("default"), + max_examples=300, + suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], + deadline=None, + verbosity=Verbosity.verbose, +) +settings.register_profile( + "ci", + parent=settings.get_profile("ci"), + max_examples=300, + derandomize=True, # more like regression testing + deadline=None, + suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], +) +settings.register_profile( + "nightly", + max_examples=500, + parent=settings.get_profile("ci"), + derandomize=False, + stateful_step_count=100, +) + +settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) + + +# TODO: uncomment these overrides when we can get mypy to accept them +""" +@overload +def create_array_metadata( + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"], + shards: None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: MemoryOrder | None, + zarr_format: Literal[2], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: None, +) -> ArrayV2Metadata: ... + + +@overload +def create_array_metadata( + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"], + shards: ShardsLike | None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: None, + zarr_format: Literal[3], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: Iterable[str] | None, +) -> ArrayV3Metadata: ... +""" + + +def create_array_metadata( + *, + shape: ShapeLike, + dtype: npt.DTypeLike, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any = 0, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, +) -> ArrayV2Metadata | ArrayV3Metadata: + """ + Create array metadata + """ + dtype_parsed = get_data_type_from_native_dtype(dtype) + shape_parsed = parse_shapelike(shape) + chunk_key_encoding_parsed = _parse_chunk_key_encoding( + chunk_key_encoding, zarr_format=zarr_format + ) + item_size = 1 + if isinstance(dtype_parsed, HasItemSize): + item_size = dtype_parsed.item_size + shard_shape_parsed, chunk_shape_parsed = _auto_partition( + array_shape=shape_parsed, + shard_shape=shards, + chunk_shape=chunks, + item_size=item_size, + ) + + if order is None: + order_parsed = zarr_config.get("array.order") + else: + order_parsed = order + chunks_out: tuple[int, ...] + + if zarr_format == 2: + filters_parsed, compressor_parsed = _parse_chunk_encoding_v2( + compressor=compressors, filters=filters, dtype=dtype_parsed + ) + return ArrayV2Metadata( + shape=shape_parsed, + dtype=dtype_parsed, + chunks=chunk_shape_parsed, + order=order_parsed, + dimension_separator=chunk_key_encoding_parsed.separator, + fill_value=fill_value, + compressor=compressor_parsed, + filters=filters_parsed, + attributes=attributes, + ) + elif zarr_format == 3: + array_array, array_bytes, bytes_bytes = _parse_chunk_encoding_v3( + compressors=compressors, + filters=filters, + serializer=serializer, + dtype=dtype_parsed, + ) + + sub_codecs: tuple[Codec, ...] = (*array_array, array_bytes, *bytes_bytes) + codecs_out: tuple[Codec, ...] + if shard_shape_parsed is not None: + index_location = None + if isinstance(shards, dict): + index_location = ShardingCodecIndexLocation(shards.get("index_location", None)) + if index_location is None: + index_location = ShardingCodecIndexLocation.end + sharding_codec = ShardingCodec( + chunk_shape=chunk_shape_parsed, + codecs=sub_codecs, + index_location=index_location, + ) + sharding_codec.validate( + shape=chunk_shape_parsed, + dtype=dtype_parsed, + chunk_grid=RegularChunkGrid(chunk_shape=shard_shape_parsed), + ) + codecs_out = (sharding_codec,) + chunks_out = shard_shape_parsed + else: + chunks_out = chunk_shape_parsed + codecs_out = sub_codecs + + return ArrayV3Metadata( + shape=shape_parsed, + data_type=dtype_parsed, + chunk_grid=RegularChunkGrid(chunk_shape=chunks_out), + chunk_key_encoding=chunk_key_encoding_parsed, + fill_value=fill_value, + codecs=codecs_out, + attributes=attributes, + dimension_names=dimension_names, + ) + + raise ValueError(f"Invalid Zarr format: {zarr_format}") + + +# TODO: uncomment these overrides when we can get mypy to accept them +""" +@overload +def meta_from_array( + array: np.ndarray[Any, Any], + chunks: ChunkCoords | Literal["auto"], + shards: None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: MemoryOrder | None, + zarr_format: Literal[2], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: Iterable[str] | None, +) -> ArrayV2Metadata: ... + + +@overload +def meta_from_array( + array: np.ndarray[Any, Any], + chunks: ChunkCoords | Literal["auto"], + shards: ShardsLike | None, + filters: FiltersLike, + compressors: CompressorsLike, + serializer: SerializerLike, + fill_value: Any | None, + order: None, + zarr_format: Literal[3], + attributes: dict[str, JSON] | None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None, + dimension_names: Iterable[str] | None, +) -> ArrayV3Metadata: ... + +""" + + +def meta_from_array( + array: np.ndarray[Any, Any], + *, + chunks: ChunkCoords | Literal["auto"] = "auto", + shards: ShardsLike | None = None, + filters: FiltersLike = "auto", + compressors: CompressorsLike = "auto", + serializer: SerializerLike = "auto", + fill_value: Any = 0, + order: MemoryOrder | None = None, + zarr_format: ZarrFormat = 3, + attributes: dict[str, JSON] | None = None, + chunk_key_encoding: ChunkKeyEncoding | ChunkKeyEncodingLike | None = None, + dimension_names: DimensionNames = None, +) -> ArrayV3Metadata | ArrayV2Metadata: + """ + Create array metadata from an array + """ + return create_array_metadata( + shape=array.shape, + dtype=array.dtype, + chunks=chunks, + shards=shards, + filters=filters, + compressors=compressors, + serializer=serializer, + fill_value=fill_value, + order=order, + zarr_format=zarr_format, + attributes=attributes, + chunk_key_encoding=chunk_key_encoding, + dimension_names=dimension_names, + ) + + +def skip_object_dtype(dtype: ZDType[Any, Any]) -> None: + if dtype.dtype_cls is type(np.dtype("O")): + msg = ( + f"{dtype} uses the numpy object data type, which is not a valid target for data " + "type resolution" + ) + pytest.skip(msg) diff --git a/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt new file mode 100644 index 0000000000..7eb0eb7c86 --- /dev/null +++ b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt @@ -0,0 +1,16 @@ +[zarr.codecs] +test = package_with_entrypoint:TestEntrypointCodec +[zarr.codecs.test] +another_codec = package_with_entrypoint:TestEntrypointGroup.Codec +[zarr] +codec_pipeline = package_with_entrypoint:TestEntrypointCodecPipeline +ndbuffer = package_with_entrypoint:TestEntrypointNDBuffer +buffer = package_with_entrypoint:TestEntrypointBuffer +[zarr.buffer] +another_buffer = package_with_entrypoint:TestEntrypointGroup.Buffer +[zarr.ndbuffer] +another_ndbuffer = package_with_entrypoint:TestEntrypointGroup.NDBuffer +[zarr.codec_pipeline] +another_pipeline = package_with_entrypoint:TestEntrypointGroup.Pipeline +[zarr.data_type] +new_data_type = package_with_entrypoint:TestDataType \ No newline at end of file diff --git a/tests/package_with_entrypoint/__init__.py b/tests/package_with_entrypoint/__init__.py new file mode 100644 index 0000000000..e0d8a52c4d --- /dev/null +++ b/tests/package_with_entrypoint/__init__.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Literal, Self + +import numpy as np +import numpy.typing as npt + +import zarr.core.buffer +from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecPipeline +from zarr.codecs import BytesCodec +from zarr.core.buffer import Buffer, NDBuffer +from zarr.core.dtype.common import DataTypeValidationError, DTypeJSON, DTypeSpec_V2 +from zarr.core.dtype.npy.bool import Bool + +if TYPE_CHECKING: + from collections.abc import Iterable + from typing import ClassVar, Literal + + from zarr.core.array_spec import ArraySpec + from zarr.core.common import ZarrFormat + + +class TestEntrypointCodec(ArrayBytesCodec): + is_fixed_size = True + + async def encode( + self, + chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]], + ) -> Iterable[Buffer | None]: + return [None] + + async def decode( + self, + chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]], + ) -> npt.NDArray[Any]: + return np.array(1) + + def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int: + return input_byte_length + + +class TestEntrypointCodecPipeline(CodecPipeline): + def __init__(self, batch_size: int = 1) -> None: + pass + + async def encode( + self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]] + ) -> Iterable[Buffer | None]: + return [None] + + async def decode( + self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]] + ) -> Iterable[NDBuffer | None]: + return np.array(1) + + +class TestEntrypointBuffer(Buffer): + pass + + +class TestEntrypointNDBuffer(NDBuffer): + pass + + +class TestEntrypointGroup: + class Codec(BytesCodec): + pass + + class Buffer(zarr.core.buffer.Buffer): + pass + + class NDBuffer(zarr.core.buffer.NDBuffer): + pass + + class Pipeline(CodecPipeline): + pass + + +class TestDataType(Bool): + """ + This is a "data type" that serializes to "test" + """ + + _zarr_v3_name: ClassVar[Literal["test"]] = "test" # type: ignore[assignment] + + @classmethod + def from_json(cls, data: DTypeJSON, *, zarr_format: Literal[2, 3]) -> Self: + if zarr_format == 2 and data == {"name": cls._zarr_v3_name, "object_codec_id": None}: + return cls() + if zarr_format == 3 and data == cls._zarr_v3_name: + return cls() + raise DataTypeValidationError( + f"Invalid JSON representation of {cls.__name__}. Got {data!r}" + ) + + def to_json(self, zarr_format: ZarrFormat) -> str | DTypeSpec_V2: # type: ignore[override] + if zarr_format == 2: + return {"name": self._zarr_v3_name, "object_codec_id": None} + if zarr_format == 3: + return self._zarr_v3_name + raise ValueError("zarr_format must be 2 or 3") diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000000..2a95d7b97c --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,1374 @@ +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +import zarr.codecs +import zarr.storage + +if TYPE_CHECKING: + import pathlib + + from zarr.abc.store import Store + from zarr.core.common import JSON, MemoryOrder, ZarrFormat + +import contextlib +import warnings +from typing import Literal + +import numpy as np +import pytest +from numpy.testing import assert_array_equal + +import zarr +import zarr.api.asynchronous +import zarr.api.synchronous +import zarr.core.group +from zarr import Array, Group +from zarr.api.synchronous import ( + create, + create_array, + create_group, + from_array, + group, + load, + open_group, + save, + save_array, + save_group, +) +from zarr.core.buffer import NDArrayLike +from zarr.errors import MetadataValidationError +from zarr.storage import LocalStore, MemoryStore, ZipStore +from zarr.storage._utils import normalize_path +from zarr.testing.utils import gpu_test + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + +def test_create(memory_store: Store) -> None: + store = memory_store + + # create array + z = create(shape=100, store=store) + assert isinstance(z, Array) + assert z.shape == (100,) + + # create array, overwrite, specify chunk shape + z = create(shape=200, chunk_shape=20, store=store, overwrite=True) + assert isinstance(z, Array) + assert z.shape == (200,) + assert z.chunks == (20,) + + # create array, overwrite, specify chunk shape via chunks param + z = create(shape=400, chunks=40, store=store, overwrite=True) + assert isinstance(z, Array) + assert z.shape == (400,) + assert z.chunks == (40,) + + # create array with float shape + with pytest.raises(TypeError): + z = create(shape=(400.5, 100), store=store, overwrite=True) # type: ignore [arg-type] + + # create array with float chunk shape + with pytest.raises(TypeError): + z = create(shape=(400, 100), chunks=(16, 16.5), store=store, overwrite=True) # type: ignore [arg-type] + + +# TODO: parametrize over everything this function takes +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_create_array(store: Store, zarr_format: ZarrFormat) -> None: + attrs: dict[str, JSON] = {"foo": 100} # explicit type annotation to avoid mypy error + shape = (10, 10) + path = "foo" + data_val = 1 + array_w = create_array( + store, + name=path, + shape=shape, + attributes=attrs, + chunks=shape, + dtype="uint8", + zarr_format=zarr_format, + ) + array_w[:] = data_val + assert array_w.shape == shape + assert array_w.attrs == attrs + assert np.array_equal(array_w[:], np.zeros(shape, dtype=array_w.dtype) + data_val) + + +@pytest.mark.parametrize("write_empty_chunks", [True, False]) +def test_write_empty_chunks_warns(write_empty_chunks: bool, zarr_format: ZarrFormat) -> None: + """ + Test that using the `write_empty_chunks` kwarg on array access will raise a warning. + """ + match = "The `write_empty_chunks` keyword argument .*" + with pytest.warns(RuntimeWarning, match=match): + _ = zarr.array( + data=np.arange(10), + shape=(10,), + dtype="uint8", + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + ) + + with pytest.warns(RuntimeWarning, match=match): + _ = zarr.create( + shape=(10,), + dtype="uint8", + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + ) + + +@pytest.mark.parametrize("path", ["foo", "/", "/foo", "///foo/bar"]) +@pytest.mark.parametrize("node_type", ["array", "group"]) +def test_open_normalized_path( + memory_store: MemoryStore, path: str, node_type: Literal["array", "group"] +) -> None: + node: Group | Array + if node_type == "group": + node = group(store=memory_store, path=path) + elif node_type == "array": + node = create(store=memory_store, path=path, shape=(2,)) + + assert node.path == normalize_path(path) + + +async def test_open_array(memory_store: MemoryStore, zarr_format: ZarrFormat) -> None: + store = memory_store + + # open array, create if doesn't exist + z = zarr.api.synchronous.open(store=store, shape=100, zarr_format=zarr_format) + assert isinstance(z, Array) + assert z.shape == (100,) + + # open array, overwrite + # store._store_dict = {} + store = MemoryStore() + z = zarr.api.synchronous.open(store=store, shape=200, zarr_format=zarr_format) + assert isinstance(z, Array) + assert z.shape == (200,) + + # open array, read-only + store_cls = type(store) + ro_store = await store_cls.open(store_dict=store._store_dict, read_only=True) + z = zarr.api.synchronous.open(store=ro_store, mode="r") + assert isinstance(z, Array) + assert z.shape == (200,) + assert z.read_only + + # path not found + with pytest.raises(FileNotFoundError): + zarr.api.synchronous.open(store="doesnotexist", mode="r", zarr_format=zarr_format) + + +@pytest.mark.parametrize("store", ["memory", "local", "zip"], indirect=True) +def test_v2_and_v3_exist_at_same_path(store: Store) -> None: + zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=3) + zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=2) + msg = f"Both zarr.json (Zarr format 3) and .zarray (Zarr format 2) metadata objects exist at {store}. Zarr v3 will be used." + with pytest.warns(UserWarning, match=re.escape(msg)): + zarr.open(store=store) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +async def test_create_group(store: Store, zarr_format: ZarrFormat) -> None: + attrs = {"foo": 100} + path = "node" + node = create_group(store, path=path, attributes=attrs, zarr_format=zarr_format) + assert isinstance(node, Group) + assert node.attrs == attrs + assert node.metadata.zarr_format == zarr_format + + +async def test_open_group(memory_store: MemoryStore) -> None: + store = memory_store + + # open group, create if doesn't exist + g = open_group(store=store) + g.create_group("foo") + assert isinstance(g, Group) + assert "foo" in g + + # open group, overwrite + g = open_group(store=store, mode="w") + assert isinstance(g, Group) + assert "foo" not in g + + # open group, read-only + store_cls = type(store) + ro_store = await store_cls.open(store_dict=store._store_dict, read_only=True) + g = open_group(store=ro_store, mode="r") + assert isinstance(g, Group) + assert g.read_only + + +@pytest.mark.parametrize("zarr_format", [None, 2, 3]) +async def test_open_group_unspecified_version( + tmpdir: pathlib.Path, zarr_format: ZarrFormat +) -> None: + """Regression test for https://github.com/zarr-developers/zarr-python/issues/2175""" + + # create a group with specified zarr format (could be 2, 3, or None) + _ = await zarr.api.asynchronous.open_group( + store=str(tmpdir), mode="w", zarr_format=zarr_format, attributes={"foo": "bar"} + ) + + # now open that group without specifying the format + g2 = await zarr.api.asynchronous.open_group(store=str(tmpdir), mode="r") + + assert g2.attrs == {"foo": "bar"} + + if zarr_format is not None: + assert g2.metadata.zarr_format == zarr_format + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize("n_args", [10, 1, 0]) +@pytest.mark.parametrize("n_kwargs", [10, 1, 0]) +@pytest.mark.parametrize("path", [None, "some_path"]) +def test_save(store: Store, n_args: int, n_kwargs: int, path: None | str) -> None: + data = np.arange(10) + args = [np.arange(10) for _ in range(n_args)] + kwargs = {f"arg_{i}": data for i in range(n_kwargs)} + + if n_kwargs == 0 and n_args == 0: + with pytest.raises(ValueError): + save(store, path=path) + elif n_args == 1 and n_kwargs == 0: + save(store, *args, path=path) + array = zarr.api.synchronous.open(store, path=path) + assert isinstance(array, Array) + assert_array_equal(array[:], data) + else: + save(store, *args, path=path, **kwargs) # type: ignore [arg-type] + group = zarr.api.synchronous.open(store, path=path) + assert isinstance(group, Group) + for array in group.array_values(): + assert_array_equal(array[:], data) + for k in kwargs: + assert k in group + assert group.nmembers() == n_args + n_kwargs + + +def test_save_errors() -> None: + with pytest.raises(ValueError): + # no arrays provided + save_group("data/group.zarr") + with pytest.raises(TypeError): + # no array provided + save_array("data/group.zarr") + with pytest.raises(ValueError): + # no arrays provided + save("data/group.zarr") + with pytest.raises(TypeError): + # mode is no valid argument and would get handled as an array + a = np.arange(10) + zarr.save("data/example.zarr", a, mode="w") + + +def test_open_with_mode_r(tmp_path: pathlib.Path) -> None: + # 'r' means read only (must exist) + with pytest.raises(FileNotFoundError): + zarr.open(store=tmp_path, mode="r") + z1 = zarr.ones(store=tmp_path, shape=(3, 3)) + assert z1.fill_value == 1 + z2 = zarr.open(store=tmp_path, mode="r") + assert isinstance(z2, Array) + assert z2.fill_value == 1 + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() + with pytest.raises(ValueError): + z2[:] = 3 + + +def test_open_with_mode_r_plus(tmp_path: pathlib.Path) -> None: + # 'r+' means read/write (must exist) + with pytest.raises(FileNotFoundError): + zarr.open(store=tmp_path, mode="r+") + zarr.ones(store=tmp_path, shape=(3, 3)) + z2 = zarr.open(store=tmp_path, mode="r+") + assert isinstance(z2, Array) + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() + z2[:] = 3 + + +async def test_open_with_mode_a(tmp_path: pathlib.Path) -> None: + # Open without shape argument should default to group + g = zarr.open(store=tmp_path, mode="a") + assert isinstance(g, Group) + await g.store_path.delete() + + # 'a' means read/write (create if doesn't exist) + arr = zarr.open(store=tmp_path, mode="a", shape=(3, 3)) + assert isinstance(arr, Array) + arr[...] = 1 + z2 = zarr.open(store=tmp_path, mode="a") + assert isinstance(z2, Array) + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() + z2[:] = 3 + + +def test_open_with_mode_w(tmp_path: pathlib.Path) -> None: + # 'w' means create (overwrite if exists); + arr = zarr.open(store=tmp_path, mode="w", shape=(3, 3)) + assert isinstance(arr, Array) + + arr[...] = 3 + z2 = zarr.open(store=tmp_path, mode="w", shape=(3, 3)) + assert isinstance(z2, Array) + result = z2[:] + assert isinstance(result, NDArrayLike) + assert not (result == 3).all() + z2[:] = 3 + + +def test_open_with_mode_w_minus(tmp_path: pathlib.Path) -> None: + # 'w-' means create (fail if exists) + arr = zarr.open(store=tmp_path, mode="w-", shape=(3, 3)) + assert isinstance(arr, Array) + arr[...] = 1 + with pytest.raises(FileExistsError): + zarr.open(store=tmp_path, mode="w-") + + +def test_array_order(zarr_format: ZarrFormat) -> None: + arr = zarr.ones(shape=(2, 2), order=None, zarr_format=zarr_format) + expected = zarr.config.get("array.order") + assert arr.order == expected + + vals = np.asarray(arr) + if expected == "C": + assert vals.flags.c_contiguous + elif expected == "F": + assert vals.flags.f_contiguous + else: + raise AssertionError + + +@pytest.mark.parametrize("order", ["C", "F"]) +def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) -> None: + with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"): + arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format) + assert arr.order == order + + vals = np.asarray(arr) + if order == "C": + assert vals.flags.c_contiguous + elif order == "F": + assert vals.flags.f_contiguous + else: + raise AssertionError + + +# def test_lazy_loader(): +# foo = np.arange(100) +# bar = np.arange(100, 0, -1) +# store = "data/group.zarr" +# save(store, foo=foo, bar=bar) +# loader = load(store) +# assert "foo" in loader +# assert "bar" in loader +# assert "baz" not in loader +# assert len(loader) == 2 +# assert sorted(loader) == ["bar", "foo"] +# assert_array_equal(foo, loader["foo"]) +# assert_array_equal(bar, loader["bar"]) +# assert "LazyLoader: " in repr(loader) + + +def test_load_array(sync_store: Store) -> None: + store = sync_store + foo = np.arange(100) + bar = np.arange(100, 0, -1) + save(store, foo=foo, bar=bar) + + # can also load arrays directly into a numpy array + for array_name in ["foo", "bar"]: + array = load(store, path=array_name) + assert isinstance(array, np.ndarray) + if array_name == "foo": + assert_array_equal(foo, array) + else: + assert_array_equal(bar, array) + + +@pytest.mark.parametrize("path", ["data", None]) +@pytest.mark.parametrize("load_read_only", [True, False, None]) +def test_load_zip(tmp_path: pathlib.Path, path: str | None, load_read_only: bool | None) -> None: + file = tmp_path / "test.zip" + data = np.arange(100).reshape(10, 10) + + with ZipStore(file, mode="w", read_only=False) as zs: + save(zs, data, path=path) + with ZipStore(file, mode="r", read_only=load_read_only) as zs: + result = zarr.load(store=zs, path=path) + assert isinstance(result, np.ndarray) + assert np.array_equal(result, data) + with ZipStore(file, read_only=load_read_only) as zs: + result = zarr.load(store=zs, path=path) + assert isinstance(result, np.ndarray) + assert np.array_equal(result, data) + + +@pytest.mark.parametrize("path", ["data", None]) +@pytest.mark.parametrize("load_read_only", [True, False]) +def test_load_local(tmp_path: pathlib.Path, path: str | None, load_read_only: bool) -> None: + file = tmp_path / "test.zip" + data = np.arange(100).reshape(10, 10) + + with LocalStore(file, read_only=False) as zs: + save(zs, data, path=path) + with LocalStore(file, read_only=load_read_only) as zs: + result = zarr.load(store=zs, path=path) + assert isinstance(result, np.ndarray) + assert np.array_equal(result, data) + + +def test_tree() -> None: + pytest.importorskip("rich") + g1 = zarr.group() + g1.create_group("foo") + g3 = g1.create_group("bar") + g3.create_group("baz") + g5 = g3.create_group("qux") + g5.create_array("baz", shape=(100,), chunks=(10,), dtype="float64") + with pytest.warns(DeprecationWarning): + assert repr(zarr.tree(g1)) == repr(g1.tree()) + assert str(zarr.tree(g1)) == str(g1.tree()) + + +# @pytest.mark.parametrize("stores_from_path", [False, True]) +# @pytest.mark.parametrize( +# "with_chunk_store,listable", +# [(False, True), (True, True), (False, False)], +# ids=["default-listable", "with_chunk_store-listable", "default-unlistable"], +# ) +# def test_consolidate_metadata(with_chunk_store, listable, monkeypatch, stores_from_path): +# # setup initial data +# if stores_from_path: +# store = tempfile.mkdtemp() +# atexit.register(atexit_rmtree, store) +# if with_chunk_store: +# chunk_store = tempfile.mkdtemp() +# atexit.register(atexit_rmtree, chunk_store) +# else: +# chunk_store = None +# else: +# store = MemoryStore() +# chunk_store = MemoryStore() if with_chunk_store else None +# path = None +# z = group(store, chunk_store=chunk_store, path=path) + +# # Reload the actual store implementation in case str +# store_to_copy = z.store + +# z.create_group("g1") +# g2 = z.create_group("g2") +# g2.attrs["hello"] = "world" +# arr = g2.create_array("arr", shape=(20, 20), chunks=(5, 5), dtype="f8") +# assert 16 == arr.nchunks +# assert 0 == arr.nchunks_initialized +# arr.attrs["data"] = 1 +# arr[:] = 1.0 +# assert 16 == arr.nchunks_initialized + +# if stores_from_path: +# # get the actual store class for use with consolidate_metadata +# store_class = z._store +# else: +# store_class = store + +# # perform consolidation +# out = consolidate_metadata(store_class, path=path) +# assert isinstance(out, Group) +# assert ["g1", "g2"] == list(out) +# if not stores_from_path: +# assert isinstance(out._store, ConsolidatedMetadataStore) +# assert ".zmetadata" in store +# meta_keys = [ +# ".zgroup", +# "g1/.zgroup", +# "g2/.zgroup", +# "g2/.zattrs", +# "g2/arr/.zarray", +# "g2/arr/.zattrs", +# ] + +# for key in meta_keys: +# del store[key] + +# # https://github.com/zarr-developers/zarr-python/issues/993 +# # Make sure we can still open consolidated on an unlistable store: +# if not listable: +# fs_memory = pytest.importorskip("fsspec.implementations.memory") +# monkeypatch.setattr(fs_memory.MemoryFileSystem, "isdir", lambda x, y: False) +# monkeypatch.delattr(fs_memory.MemoryFileSystem, "ls") +# fs = fs_memory.MemoryFileSystem() +# store_to_open = FSStore("", fs=fs) +# # copy original store to new unlistable store +# store_to_open.update(store_to_copy) + +# else: +# store_to_open = store + +# # open consolidated +# z2 = open_consolidated(store_to_open, chunk_store=chunk_store, path=path) +# assert ["g1", "g2"] == list(z2) +# assert "world" == z2.g2.attrs["hello"] +# assert 1 == z2.g2.arr.attrs["data"] +# assert (z2.g2.arr[:] == 1.0).all() +# assert 16 == z2.g2.arr.nchunks +# if listable: +# assert 16 == z2.g2.arr.nchunks_initialized +# else: +# with pytest.raises(NotImplementedError): +# _ = z2.g2.arr.nchunks_initialized + +# if stores_from_path: +# # path string is note a BaseStore subclass so cannot be used to +# # initialize a ConsolidatedMetadataStore. + +# with pytest.raises(ValueError): +# cmd = ConsolidatedMetadataStore(store) +# else: +# # tests del/write on the store + +# cmd = ConsolidatedMetadataStore(store) +# with pytest.raises(PermissionError): +# del cmd[".zgroup"] +# with pytest.raises(PermissionError): +# cmd[".zgroup"] = None + +# # test getsize on the store +# assert isinstance(getsize(cmd), Integral) + +# # test new metadata are not writeable +# with pytest.raises(PermissionError): +# z2.create_group("g3") +# with pytest.raises(PermissionError): +# z2.create_dataset("spam", shape=42, chunks=7, dtype="i4") +# with pytest.raises(PermissionError): +# del z2["g2"] + +# # test consolidated metadata are not writeable +# with pytest.raises(PermissionError): +# z2.g2.attrs["hello"] = "universe" +# with pytest.raises(PermissionError): +# z2.g2.arr.attrs["foo"] = "bar" + +# # test the data are writeable +# z2.g2.arr[:] = 2 +# assert (z2.g2.arr[:] == 2).all() + +# # test invalid modes +# with pytest.raises(ValueError): +# open_consolidated(store, chunk_store=chunk_store, mode="a", path=path) +# with pytest.raises(ValueError): +# open_consolidated(store, chunk_store=chunk_store, mode="w", path=path) +# with pytest.raises(ValueError): +# open_consolidated(store, chunk_store=chunk_store, mode="w-", path=path) + +# # make sure keyword arguments are passed through without error +# open_consolidated( +# store, +# chunk_store=chunk_store, +# path=path, +# cache_attrs=True, +# synchronizer=None, +# ) + + +# @pytest.mark.parametrize( +# "options", +# ( +# {"dimension_separator": "/"}, +# {"dimension_separator": "."}, +# {"dimension_separator": None}, +# ), +# ) +# def test_save_array_separator(tmpdir, options): +# data = np.arange(6).reshape((3, 2)) +# url = tmpdir.join("test.zarr") +# save_array(url, data, **options) + + +# class TestCopyStore(unittest.TestCase): +# _version = 2 + +# def setUp(self): +# source = dict() +# source["foo"] = b"xxx" +# source["bar/baz"] = b"yyy" +# source["bar/qux"] = b"zzz" +# self.source = source + +# def _get_dest_store(self): +# return dict() + +# def test_no_paths(self): +# source = self.source +# dest = self._get_dest_store() +# copy_store(source, dest) +# assert len(source) == len(dest) +# for key in source: +# assert source[key] == dest[key] + +# def test_source_path(self): +# source = self.source +# # paths should be normalized +# for source_path in "bar", "bar/", "/bar", "/bar/": +# dest = self._get_dest_store() +# copy_store(source, dest, source_path=source_path) +# assert 2 == len(dest) +# for key in source: +# if key.startswith("bar/"): +# dest_key = key.split("bar/")[1] +# assert source[key] == dest[dest_key] +# else: +# assert key not in dest + +# def test_dest_path(self): +# source = self.source +# # paths should be normalized +# for dest_path in "new", "new/", "/new", "/new/": +# dest = self._get_dest_store() +# copy_store(source, dest, dest_path=dest_path) +# assert len(source) == len(dest) +# for key in source: +# if self._version == 3: +# dest_key = key[:10] + "new/" + key[10:] +# else: +# dest_key = "new/" + key +# assert source[key] == dest[dest_key] + +# def test_source_dest_path(self): +# source = self.source +# # paths should be normalized +# for source_path in "bar", "bar/", "/bar", "/bar/": +# for dest_path in "new", "new/", "/new", "/new/": +# dest = self._get_dest_store() +# copy_store(source, dest, source_path=source_path, dest_path=dest_path) +# assert 2 == len(dest) +# for key in source: +# if key.startswith("bar/"): +# dest_key = "new/" + key.split("bar/")[1] +# assert source[key] == dest[dest_key] +# else: +# assert key not in dest +# assert ("new/" + key) not in dest + +# def test_excludes_includes(self): +# source = self.source + +# # single excludes +# dest = self._get_dest_store() +# excludes = "f.*" +# copy_store(source, dest, excludes=excludes) +# assert len(dest) == 2 + +# root = "" +# assert root + "foo" not in dest + +# # multiple excludes +# dest = self._get_dest_store() +# excludes = "b.z", ".*x" +# copy_store(source, dest, excludes=excludes) +# assert len(dest) == 1 +# assert root + "foo" in dest +# assert root + "bar/baz" not in dest +# assert root + "bar/qux" not in dest + +# # excludes and includes +# dest = self._get_dest_store() +# excludes = "b.*" +# includes = ".*x" +# copy_store(source, dest, excludes=excludes, includes=includes) +# assert len(dest) == 2 +# assert root + "foo" in dest +# assert root + "bar/baz" not in dest +# assert root + "bar/qux" in dest + +# def test_dry_run(self): +# source = self.source +# dest = self._get_dest_store() +# copy_store(source, dest, dry_run=True) +# assert 0 == len(dest) + +# def test_if_exists(self): +# source = self.source +# dest = self._get_dest_store() +# root = "" +# dest[root + "bar/baz"] = b"mmm" + +# # default ('raise') +# with pytest.raises(CopyError): +# copy_store(source, dest) + +# # explicit 'raise' +# with pytest.raises(CopyError): +# copy_store(source, dest, if_exists="raise") + +# # skip +# copy_store(source, dest, if_exists="skip") +# assert 3 == len(dest) +# assert dest[root + "foo"] == b"xxx" +# assert dest[root + "bar/baz"] == b"mmm" +# assert dest[root + "bar/qux"] == b"zzz" + +# # replace +# copy_store(source, dest, if_exists="replace") +# assert 3 == len(dest) +# assert dest[root + "foo"] == b"xxx" +# assert dest[root + "bar/baz"] == b"yyy" +# assert dest[root + "bar/qux"] == b"zzz" + +# # invalid option +# with pytest.raises(ValueError): +# copy_store(source, dest, if_exists="foobar") + + +# def check_copied_array(original, copied, without_attrs=False, expect_props=None): +# # setup +# source_h5py = original.__module__.startswith("h5py.") +# dest_h5py = copied.__module__.startswith("h5py.") +# zarr_to_zarr = not (source_h5py or dest_h5py) +# h5py_to_h5py = source_h5py and dest_h5py +# zarr_to_h5py = not source_h5py and dest_h5py +# h5py_to_zarr = source_h5py and not dest_h5py +# if expect_props is None: +# expect_props = dict() +# else: +# expect_props = expect_props.copy() + +# # common properties in zarr and h5py +# for p in "dtype", "shape", "chunks": +# expect_props.setdefault(p, getattr(original, p)) + +# # zarr-specific properties +# if zarr_to_zarr: +# for p in "compressor", "filters", "order", "fill_value": +# expect_props.setdefault(p, getattr(original, p)) + +# # h5py-specific properties +# if h5py_to_h5py: +# for p in ( +# "maxshape", +# "compression", +# "compression_opts", +# "shuffle", +# "scaleoffset", +# "fletcher32", +# "fillvalue", +# ): +# expect_props.setdefault(p, getattr(original, p)) + +# # common properties with some name differences +# if h5py_to_zarr: +# expect_props.setdefault("fill_value", original.fillvalue) +# if zarr_to_h5py: +# expect_props.setdefault("fillvalue", original.fill_value) + +# # compare properties +# for k, v in expect_props.items(): +# assert v == getattr(copied, k) + +# # compare data +# assert_array_equal(original[:], copied[:]) + +# # compare attrs +# if without_attrs: +# for k in original.attrs.keys(): +# assert k not in copied.attrs +# else: +# if dest_h5py and "filters" in original.attrs: +# # special case in v3 (storing filters metadata under attributes) +# # we explicitly do not copy this info over to HDF5 +# original_attrs = original.attrs.asdict().copy() +# original_attrs.pop("filters") +# else: +# original_attrs = original.attrs +# assert sorted(original_attrs.items()) == sorted(copied.attrs.items()) + + +# def check_copied_group(original, copied, without_attrs=False, expect_props=None, shallow=False): +# # setup +# if expect_props is None: +# expect_props = dict() +# else: +# expect_props = expect_props.copy() + +# # compare children +# for k, v in original.items(): +# if hasattr(v, "shape"): +# assert k in copied +# check_copied_array(v, copied[k], without_attrs=without_attrs, expect_props=expect_props) +# elif shallow: +# assert k not in copied +# else: +# assert k in copied +# check_copied_group( +# v, +# copied[k], +# without_attrs=without_attrs, +# shallow=shallow, +# expect_props=expect_props, +# ) + +# # compare attrs +# if without_attrs: +# for k in original.attrs.keys(): +# assert k not in copied.attrs +# else: +# assert sorted(original.attrs.items()) == sorted(copied.attrs.items()) + + +# def test_copy_all(): +# """ +# https://github.com/zarr-developers/zarr-python/issues/269 + +# copy_all used to not copy attributes as `.keys()` does not return hidden `.zattrs`. + +# """ +# original_group = zarr.group(store=MemoryStore(), overwrite=True) +# original_group.attrs["info"] = "group attrs" +# original_subgroup = original_group.create_group("subgroup") +# original_subgroup.attrs["info"] = "sub attrs" + +# destination_group = zarr.group(store=MemoryStore(), overwrite=True) + +# # copy from memory to directory store +# copy_all( +# original_group, +# destination_group, +# dry_run=False, +# ) + +# assert "subgroup" in destination_group +# assert destination_group.attrs["info"] == "group attrs" +# assert destination_group.subgroup.attrs["info"] == "sub attrs" + + +# class TestCopy: +# @pytest.fixture(params=[False, True], ids=["zarr", "hdf5"]) +# def source(self, request, tmpdir): +# def prep_source(source): +# foo = source.create_group("foo") +# foo.attrs["experiment"] = "weird science" +# baz = foo.create_dataset("bar/baz", data=np.arange(100), chunks=(50,)) +# baz.attrs["units"] = "metres" +# if request.param: +# extra_kws = dict( +# compression="gzip", +# compression_opts=3, +# fillvalue=84, +# shuffle=True, +# fletcher32=True, +# ) +# else: +# extra_kws = dict(compressor=Zlib(3), order="F", fill_value=42, filters=[Adler32()]) +# source.create_dataset( +# "spam", +# data=np.arange(100, 200).reshape(20, 5), +# chunks=(10, 2), +# dtype="i2", +# **extra_kws, +# ) +# return source + +# if request.param: +# h5py = pytest.importorskip("h5py") +# fn = tmpdir.join("source.h5") +# with h5py.File(str(fn), mode="w") as h5f: +# yield prep_source(h5f) +# else: +# yield prep_source(group()) + +# @pytest.fixture(params=[False, True], ids=["zarr", "hdf5"]) +# def dest(self, request, tmpdir): +# if request.param: +# h5py = pytest.importorskip("h5py") +# fn = tmpdir.join("dest.h5") +# with h5py.File(str(fn), mode="w") as h5f: +# yield h5f +# else: +# yield group() + +# def test_copy_array(self, source, dest): +# # copy array with default options +# copy(source["foo/bar/baz"], dest) +# check_copied_array(source["foo/bar/baz"], dest["baz"]) +# copy(source["spam"], dest) +# check_copied_array(source["spam"], dest["spam"]) + +# def test_copy_bad_dest(self, source, dest): +# # try to copy to an array, dest must be a group +# dest = dest.create_dataset("eggs", shape=(100,)) +# with pytest.raises(ValueError): +# copy(source["foo/bar/baz"], dest) + +# def test_copy_array_name(self, source, dest): +# # copy array with name +# copy(source["foo/bar/baz"], dest, name="qux") +# assert "baz" not in dest +# check_copied_array(source["foo/bar/baz"], dest["qux"]) + +# def test_copy_array_create_options(self, source, dest): +# dest_h5py = dest.__module__.startswith("h5py.") + +# # copy array, provide creation options +# compressor = Zlib(9) +# create_kws = dict(chunks=(10,)) +# if dest_h5py: +# create_kws.update( +# compression="gzip", compression_opts=9, shuffle=True, fletcher32=True, fillvalue=42 +# ) +# else: +# create_kws.update(compressor=compressor, fill_value=42, order="F", filters=[Adler32()]) +# copy(source["foo/bar/baz"], dest, without_attrs=True, **create_kws) +# check_copied_array( +# source["foo/bar/baz"], dest["baz"], without_attrs=True, expect_props=create_kws +# ) + +# def test_copy_array_exists_array(self, source, dest): +# # copy array, dest array in the way +# dest.create_dataset("baz", shape=(10,)) + +# # raise +# with pytest.raises(CopyError): +# # should raise by default +# copy(source["foo/bar/baz"], dest) +# assert (10,) == dest["baz"].shape +# with pytest.raises(CopyError): +# copy(source["foo/bar/baz"], dest, if_exists="raise") +# assert (10,) == dest["baz"].shape + +# # skip +# copy(source["foo/bar/baz"], dest, if_exists="skip") +# assert (10,) == dest["baz"].shape + +# # replace +# copy(source["foo/bar/baz"], dest, if_exists="replace") +# check_copied_array(source["foo/bar/baz"], dest["baz"]) + +# # invalid option +# with pytest.raises(ValueError): +# copy(source["foo/bar/baz"], dest, if_exists="foobar") + +# def test_copy_array_exists_group(self, source, dest): +# # copy array, dest group in the way +# dest.create_group("baz") + +# # raise +# with pytest.raises(CopyError): +# copy(source["foo/bar/baz"], dest) +# assert not hasattr(dest["baz"], "shape") +# with pytest.raises(CopyError): +# copy(source["foo/bar/baz"], dest, if_exists="raise") +# assert not hasattr(dest["baz"], "shape") + +# # skip +# copy(source["foo/bar/baz"], dest, if_exists="skip") +# assert not hasattr(dest["baz"], "shape") + +# # replace +# copy(source["foo/bar/baz"], dest, if_exists="replace") +# check_copied_array(source["foo/bar/baz"], dest["baz"]) + +# def test_copy_array_skip_initialized(self, source, dest): +# dest_h5py = dest.__module__.startswith("h5py.") + +# dest.create_dataset("baz", shape=(100,), chunks=(10,), dtype="i8") +# assert not np.all(source["foo/bar/baz"][:] == dest["baz"][:]) + +# if dest_h5py: +# with pytest.raises(ValueError): +# # not available with copy to h5py +# copy(source["foo/bar/baz"], dest, if_exists="skip_initialized") + +# else: +# # copy array, dest array exists but not yet initialized +# copy(source["foo/bar/baz"], dest, if_exists="skip_initialized") +# check_copied_array(source["foo/bar/baz"], dest["baz"]) + +# # copy array, dest array exists and initialized, will be skipped +# dest["baz"][:] = np.arange(100, 200) +# copy(source["foo/bar/baz"], dest, if_exists="skip_initialized") +# assert_array_equal(np.arange(100, 200), dest["baz"][:]) +# assert not np.all(source["foo/bar/baz"][:] == dest["baz"][:]) + +# def test_copy_group(self, source, dest): +# # copy group, default options +# copy(source["foo"], dest) +# check_copied_group(source["foo"], dest["foo"]) + +# def test_copy_group_no_name(self, source, dest): +# with pytest.raises(TypeError): +# # need a name if copy root +# copy(source, dest) + +# copy(source, dest, name="root") +# check_copied_group(source, dest["root"]) + +# def test_copy_group_options(self, source, dest): +# # copy group, non-default options +# copy(source["foo"], dest, name="qux", without_attrs=True) +# assert "foo" not in dest +# check_copied_group(source["foo"], dest["qux"], without_attrs=True) + +# def test_copy_group_shallow(self, source, dest): +# # copy group, shallow +# copy(source, dest, name="eggs", shallow=True) +# check_copied_group(source, dest["eggs"], shallow=True) + +# def test_copy_group_exists_group(self, source, dest): +# # copy group, dest groups exist +# dest.create_group("foo/bar") +# copy(source["foo"], dest) +# check_copied_group(source["foo"], dest["foo"]) + +# def test_copy_group_exists_array(self, source, dest): +# # copy group, dest array in the way +# dest.create_dataset("foo/bar", shape=(10,)) + +# # raise +# with pytest.raises(CopyError): +# copy(source["foo"], dest) +# assert dest["foo/bar"].shape == (10,) +# with pytest.raises(CopyError): +# copy(source["foo"], dest, if_exists="raise") +# assert dest["foo/bar"].shape == (10,) + +# # skip +# copy(source["foo"], dest, if_exists="skip") +# assert dest["foo/bar"].shape == (10,) + +# # replace +# copy(source["foo"], dest, if_exists="replace") +# check_copied_group(source["foo"], dest["foo"]) + +# def test_copy_group_dry_run(self, source, dest): +# # dry run, empty destination +# n_copied, n_skipped, n_bytes_copied = copy( +# source["foo"], dest, dry_run=True, return_stats=True +# ) +# assert 0 == len(dest) +# assert 3 == n_copied +# assert 0 == n_skipped +# assert 0 == n_bytes_copied + +# # dry run, array exists in destination +# baz = np.arange(100, 200) +# dest.create_dataset("foo/bar/baz", data=baz) +# assert not np.all(source["foo/bar/baz"][:] == dest["foo/bar/baz"][:]) +# assert 1 == len(dest) + +# # raise +# with pytest.raises(CopyError): +# copy(source["foo"], dest, dry_run=True) +# assert 1 == len(dest) + +# # skip +# n_copied, n_skipped, n_bytes_copied = copy( +# source["foo"], dest, dry_run=True, if_exists="skip", return_stats=True +# ) +# assert 1 == len(dest) +# assert 2 == n_copied +# assert 1 == n_skipped +# assert 0 == n_bytes_copied +# assert_array_equal(baz, dest["foo/bar/baz"]) + +# # replace +# n_copied, n_skipped, n_bytes_copied = copy( +# source["foo"], dest, dry_run=True, if_exists="replace", return_stats=True +# ) +# assert 1 == len(dest) +# assert 3 == n_copied +# assert 0 == n_skipped +# assert 0 == n_bytes_copied +# assert_array_equal(baz, dest["foo/bar/baz"]) + +# def test_logging(self, source, dest, tmpdir): +# # callable log +# copy(source["foo"], dest, dry_run=True, log=print) + +# # file name +# fn = str(tmpdir.join("log_name")) +# copy(source["foo"], dest, dry_run=True, log=fn) + +# # file +# with tmpdir.join("log_file").open(mode="w") as f: +# copy(source["foo"], dest, dry_run=True, log=f) + +# # bad option +# with pytest.raises(TypeError): +# copy(source["foo"], dest, dry_run=True, log=True) + + +def test_open_positional_args_deprecated() -> None: + store = MemoryStore() + with pytest.warns(FutureWarning, match="pass"): + zarr.api.synchronous.open(store, "w", shape=(1,)) + + +def test_save_array_positional_args_deprecated() -> None: + store = MemoryStore() + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message="zarr_version is deprecated", category=DeprecationWarning + ) + with pytest.warns(FutureWarning, match="pass"): + save_array( + store, + np.ones( + 1, + ), + 3, + ) + + +def test_group_positional_args_deprecated() -> None: + store = MemoryStore() + with pytest.warns(FutureWarning, match="pass"): + group(store, True) + + +def test_open_group_positional_args_deprecated() -> None: + store = MemoryStore() + with pytest.warns(FutureWarning, match="pass"): + open_group(store, "w") + + +def test_open_falls_back_to_open_group() -> None: + # https://github.com/zarr-developers/zarr-python/issues/2309 + store = MemoryStore() + zarr.open_group(store, attributes={"key": "value"}) + + group = zarr.open(store) + assert isinstance(group, Group) + assert group.attrs == {"key": "value"} + + +async def test_open_falls_back_to_open_group_async(zarr_format: ZarrFormat) -> None: + # https://github.com/zarr-developers/zarr-python/issues/2309 + store = MemoryStore() + await zarr.api.asynchronous.open_group( + store, attributes={"key": "value"}, zarr_format=zarr_format + ) + + group = await zarr.api.asynchronous.open(store=store) + assert isinstance(group, zarr.core.group.AsyncGroup) + assert group.metadata.zarr_format == zarr_format + assert group.attrs == {"key": "value"} + + +@pytest.mark.parametrize("mode", ["r", "r+", "w", "a"]) +def test_open_modes_creates_group(tmp_path: pathlib.Path, mode: str) -> None: + # https://github.com/zarr-developers/zarr-python/issues/2490 + zarr_dir = tmp_path / f"mode-{mode}-test.zarr" + if mode in ["r", "r+"]: + # Expect FileNotFoundError to be raised if 'r' or 'r+' mode + with pytest.raises(FileNotFoundError): + zarr.open(store=zarr_dir, mode=mode) + else: + group = zarr.open(store=zarr_dir, mode=mode) + assert isinstance(group, Group) + + +async def test_metadata_validation_error() -> None: + with pytest.raises( + MetadataValidationError, + match="Invalid value for 'zarr_format'. Expected '2, 3, or None'. Got '3.0'.", + ): + await zarr.api.asynchronous.open_group(zarr_format="3.0") # type: ignore [arg-type] + + with pytest.raises( + MetadataValidationError, + match="Invalid value for 'zarr_format'. Expected '2, 3, or None'. Got '3.0'.", + ): + await zarr.api.asynchronous.open_array(shape=(1,), zarr_format="3.0") # type: ignore [arg-type] + + +@pytest.mark.parametrize( + "store", + ["local", "memory", "zip"], + indirect=True, +) +def test_open_array_with_mode_r_plus(store: Store, zarr_format: ZarrFormat) -> None: + # 'r+' means read/write (must exist) + with pytest.raises(FileNotFoundError): + zarr.open_array(store=store, mode="r+", zarr_format=zarr_format) + zarr.ones(store=store, shape=(3, 3), zarr_format=zarr_format) + z2 = zarr.open_array(store=store, mode="r+") + assert isinstance(z2, Array) + assert z2.metadata.zarr_format == zarr_format + result = z2[:] + assert isinstance(result, NDArrayLike) + assert (result == 1).all() + z2[:] = 3 + + +def test_api_exports() -> None: + """ + Test that the sync API and the async API export the same objects + """ + assert zarr.api.asynchronous.__all__ == zarr.api.synchronous.__all__ + + +@gpu_test +@pytest.mark.parametrize( + "store", + ["local", "memory", "zip"], + indirect=True, +) +@pytest.mark.parametrize("zarr_format", [None, 2, 3]) +def test_gpu_basic(store: Store, zarr_format: ZarrFormat | None) -> None: + import cupy as cp + + if zarr_format == 2: + # Without this, the zstd codec attempts to convert the cupy + # array to bytes. + compressors = None + else: + compressors = "auto" + + with zarr.config.enable_gpu(): + src = cp.random.uniform(size=(100, 100)) # allocate on the device + z = zarr.create_array( + store, + name="a", + shape=src.shape, + chunks=(10, 10), + dtype=src.dtype, + overwrite=True, + zarr_format=zarr_format, + compressors=compressors, + ) + z[:10, :10] = src[:10, :10] + + result = z[:10, :10] + # assert_array_equal doesn't check the type + assert isinstance(result, type(src)) + cp.testing.assert_array_equal(result, src[:10, :10]) + + +def test_v2_without_compressor() -> None: + # Make sure it's possible to set no compressor for v2 arrays + arr = zarr.create(store={}, shape=(1), dtype="uint8", zarr_format=2, compressor=None) + assert arr.compressors == () + + +def test_v2_with_v3_compressor() -> None: + # Check trying to create a v2 array with a v3 compressor fails + with pytest.raises( + ValueError, + match="Cannot use a BytesBytesCodec as a compressor for zarr v2 arrays. Use a numcodecs codec directly instead.", + ): + zarr.create( + store={}, shape=(1), dtype="uint8", zarr_format=2, compressor=zarr.codecs.BloscCodec() + ) + + +def add_empty_file(path: Path) -> Path: + fpath = path / "a.txt" + fpath.touch() + return fpath + + +@pytest.mark.parametrize("create_function", [create_array, from_array]) +@pytest.mark.parametrize("overwrite", [True, False]) +def test_no_overwrite_array(tmp_path: Path, create_function: Callable, overwrite: bool) -> None: # type:ignore[type-arg] + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + create_function(store=store, data=np.ones(shape=(1,)), overwrite=overwrite) + if overwrite: + assert not existing_fpath.exists() + else: + assert existing_fpath.exists() + + +@pytest.mark.parametrize("create_function", [create_group, group]) +@pytest.mark.parametrize("overwrite", [True, False]) +def test_no_overwrite_group(tmp_path: Path, create_function: Callable, overwrite: bool) -> None: # type:ignore[type-arg] + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + create_function(store=store, overwrite=overwrite) + if overwrite: + assert not existing_fpath.exists() + else: + assert existing_fpath.exists() + + +@pytest.mark.parametrize("open_func", [zarr.open, open_group]) +@pytest.mark.parametrize("mode", ["r", "r+", "a", "w", "w-"]) +def test_no_overwrite_open(tmp_path: Path, open_func: Callable, mode: str) -> None: # type:ignore[type-arg] + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + with contextlib.suppress(FileExistsError, FileNotFoundError, ValueError): + open_func(store=store, mode=mode) + if mode == "w": + assert not existing_fpath.exists() + else: + assert existing_fpath.exists() + + +def test_no_overwrite_load(tmp_path: Path) -> None: + store = zarr.storage.LocalStore(tmp_path) + existing_fpath = add_empty_file(tmp_path) + + assert existing_fpath.exists() + with contextlib.suppress(NotImplementedError): + zarr.load(store) + assert existing_fpath.exists() + + +@pytest.mark.parametrize( + "f", + [ + zarr.array, + zarr.create, + zarr.create_array, + zarr.ones, + zarr.ones_like, + zarr.empty, + zarr.empty_like, + zarr.full, + zarr.full_like, + zarr.zeros, + zarr.zeros_like, + ], +) +def test_auto_chunks(f: Callable[..., Array]) -> None: + # Make sure chunks are set automatically across the public API + # TODO: test shards with this test too + shape = (1000, 1000) + dtype = np.uint8 + kwargs = {"shape": shape, "dtype": dtype} + array = np.zeros(shape, dtype=dtype) + store = zarr.storage.MemoryStore() + + if f in [zarr.full, zarr.full_like]: + kwargs["fill_value"] = 0 + if f in [zarr.array]: + kwargs["data"] = array + if f in [zarr.empty_like, zarr.full_like, zarr.empty_like, zarr.ones_like, zarr.zeros_like]: + kwargs["a"] = array + if f in [zarr.create_array]: + kwargs["store"] = store + + a = f(**kwargs) + assert a.chunks == (500, 500) diff --git a/tests/test_array.py b/tests/test_array.py new file mode 100644 index 0000000000..28ea812967 --- /dev/null +++ b/tests/test_array.py @@ -0,0 +1,1772 @@ +import dataclasses +import inspect +import json +import math +import multiprocessing as mp +import pickle +import re +import sys +from itertools import accumulate +from typing import TYPE_CHECKING, Any, Literal +from unittest import mock + +import numcodecs +import numpy as np +import numpy.typing as npt +import pytest +from packaging.version import Version + +import zarr.api.asynchronous +import zarr.api.synchronous as sync_api +from tests.conftest import skip_object_dtype +from zarr import Array, AsyncArray, Group +from zarr.abc.store import Store +from zarr.codecs import ( + BytesCodec, + GzipCodec, + TransposeCodec, + ZstdCodec, +) +from zarr.core._info import ArrayInfo +from zarr.core.array import ( + CompressorsLike, + FiltersLike, + _parse_chunk_encoding_v2, + _parse_chunk_encoding_v3, + chunks_initialized, + create_array, +) +from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, default_buffer_prototype +from zarr.core.buffer.cpu import NDBuffer +from zarr.core.chunk_grids import _auto_partition +from zarr.core.chunk_key_encodings import ChunkKeyEncodingParams +from zarr.core.common import JSON, MemoryOrder, ZarrFormat +from zarr.core.dtype import get_data_type_from_native_dtype +from zarr.core.dtype.common import ENDIANNESS_STR, EndiannessStr +from zarr.core.dtype.npy.common import NUMPY_ENDIANNESS_STR, endianness_from_numpy_str +from zarr.core.dtype.npy.float import Float32, Float64 +from zarr.core.dtype.npy.int import Int16, UInt8 +from zarr.core.dtype.npy.string import VariableLengthUTF8 +from zarr.core.dtype.npy.structured import ( + Structured, +) +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64 +from zarr.core.dtype.wrapper import ZDType +from zarr.core.group import AsyncGroup +from zarr.core.indexing import BasicIndexer, ceildiv +from zarr.core.metadata.v2 import ArrayV2Metadata +from zarr.core.sync import sync +from zarr.errors import ContainsArrayError, ContainsGroupError +from zarr.storage import LocalStore, MemoryStore, StorePath + +from .test_dtype.conftest import zdtype_examples + +if TYPE_CHECKING: + from zarr.core.array_spec import ArrayConfigLike + from zarr.core.metadata.v3 import ArrayV3Metadata + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.parametrize("overwrite", [True, False]) +@pytest.mark.parametrize("extant_node", ["array", "group"]) +def test_array_creation_existing_node( + store: LocalStore | MemoryStore, + zarr_format: ZarrFormat, + overwrite: bool, + extant_node: Literal["array", "group"], +) -> None: + """ + Check that an existing array or group is handled as expected during array creation. + """ + spath = StorePath(store) + group = Group.from_store(spath, zarr_format=zarr_format) + expected_exception: type[ContainsArrayError | ContainsGroupError] + if extant_node == "array": + expected_exception = ContainsArrayError + _ = group.create_array("extant", shape=(10,), dtype="uint8") + elif extant_node == "group": + expected_exception = ContainsGroupError + _ = group.create_group("extant") + else: + raise AssertionError + + new_shape = (2, 2) + new_dtype = "float32" + + if overwrite: + if not store.supports_deletes: + pytest.skip("store does not support deletes") + arr_new = zarr.create_array( + spath / "extant", + shape=new_shape, + dtype=new_dtype, + overwrite=overwrite, + zarr_format=zarr_format, + ) + assert arr_new.shape == new_shape + assert arr_new.dtype == new_dtype + else: + with pytest.raises(expected_exception): + arr_new = zarr.create_array( + spath / "extant", + shape=new_shape, + dtype=new_dtype, + overwrite=overwrite, + zarr_format=zarr_format, + ) + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +async def test_create_creates_parents( + store: LocalStore | MemoryStore, zarr_format: ZarrFormat +) -> None: + # prepare a root node, with some data set + await zarr.api.asynchronous.open_group( + store=store, path="a", zarr_format=zarr_format, attributes={"key": "value"} + ) + + # create a child node with a couple intermediates + await zarr.api.asynchronous.create( + shape=(2, 2), store=store, path="a/b/c/d", zarr_format=zarr_format + ) + parts = ["a", "a/b", "a/b/c"] + + if zarr_format == 2: + files = [".zattrs", ".zgroup"] + else: + files = ["zarr.json"] + + expected = [f"{part}/{file}" for file in files for part in parts] + + if zarr_format == 2: + expected.extend([".zattrs", ".zgroup", "a/b/c/d/.zarray", "a/b/c/d/.zattrs"]) + else: + expected.extend(["zarr.json", "a/b/c/d/zarr.json"]) + + expected = sorted(expected) + + result = sorted([x async for x in store.list_prefix("")]) + + assert result == expected + + paths = ["a", "a/b", "a/b/c"] + for path in paths: + g = await zarr.api.asynchronous.open_group(store=store, path=path) + assert isinstance(g, AsyncGroup) + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +def test_array_name_properties_no_group( + store: LocalStore | MemoryStore, zarr_format: ZarrFormat +) -> None: + arr = zarr.create_array( + store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype=">i4" + ) + assert arr.path == "" + assert arr.name == "/" + assert arr.basename == "" + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +def test_array_name_properties_with_group( + store: LocalStore | MemoryStore, zarr_format: ZarrFormat +) -> None: + root = Group.from_store(store=store, zarr_format=zarr_format) + foo = root.create_array("foo", shape=(100,), chunks=(10,), dtype="i4") + assert foo.path == "foo" + assert foo.name == "/foo" + assert foo.basename == "foo" + + bar = root.create_group("bar") + spam = bar.create_array("spam", shape=(100,), chunks=(10,), dtype="i4") + + assert spam.path == "bar/spam" + assert spam.name == "/bar/spam" + assert spam.basename == "spam" + + +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("specifiy_fill_value", [True, False]) +@pytest.mark.parametrize( + "zdtype", zdtype_examples, ids=tuple(str(type(v)) for v in zdtype_examples) +) +def test_array_fill_value_default( + store: MemoryStore, specifiy_fill_value: bool, zdtype: ZDType[Any, Any] +) -> None: + """ + Test that creating an array with the fill_value parameter set to None, or unspecified, + results in the expected fill_value attribute of the array, i.e. the default value of the dtype + """ + shape = (10,) + if specifiy_fill_value: + arr = zarr.create_array( + store=store, + shape=shape, + dtype=zdtype, + zarr_format=3, + chunks=shape, + fill_value=None, + ) + else: + arr = zarr.create_array(store=store, shape=shape, dtype=zdtype, zarr_format=3, chunks=shape) + expected_fill_value = zdtype.default_scalar() + if isinstance(expected_fill_value, np.datetime64 | np.timedelta64): + if np.isnat(expected_fill_value): + assert np.isnat(arr.fill_value) + elif isinstance(expected_fill_value, np.floating | np.complexfloating): + if np.isnan(expected_fill_value): + assert np.isnan(arr.fill_value) + else: + assert arr.fill_value == expected_fill_value + # A simpler check would be to ensure that arr.fill_value.dtype == arr.dtype + # But for some numpy data types (namely, U), scalars might not have length. An empty string + # scalar from a `>U4` array would have dtype `>U`, and arr.fill_value.dtype == arr.dtype will fail. + + assert type(arr.fill_value) is type(np.array([arr.fill_value], dtype=arr.dtype)[0]) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize( + ("dtype_str", "fill_value"), + [("bool", True), ("uint8", 99), ("float32", -99.9), ("complex64", 3 + 4j)], +) +def test_array_v3_fill_value(store: MemoryStore, fill_value: int, dtype_str: str) -> None: + shape = (10,) + arr = zarr.create_array( + store=store, + shape=shape, + dtype=dtype_str, + zarr_format=3, + chunks=shape, + fill_value=fill_value, + ) + + assert arr.fill_value == np.dtype(dtype_str).type(fill_value) + assert arr.fill_value.dtype == arr.dtype + + +async def test_create_deprecated() -> None: + with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning, match=re.escape("Pass shape=(2, 2) as keyword args")): + await zarr.AsyncArray.create(MemoryStore(), (2, 2), dtype="f8") # type: ignore[call-overload] + with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning, match=re.escape("Pass shape=(2, 2) as keyword args")): + zarr.Array.create(MemoryStore(), (2, 2), dtype="f8") + + +def test_selection_positional_args_deprecated() -> None: + store = MemoryStore() + arr = zarr.create_array(store, shape=(2, 2), dtype="f8") + + with pytest.warns(FutureWarning, match="Pass out"): + arr.get_basic_selection(..., NDBuffer(array=np.empty((2, 2)))) + + with pytest.warns(FutureWarning, match="Pass fields"): + arr.set_basic_selection(..., 1, None) + + with pytest.warns(FutureWarning, match="Pass out"): + arr.get_orthogonal_selection(..., NDBuffer(array=np.empty((2, 2)))) + + with pytest.warns(FutureWarning, match="Pass"): + arr.set_orthogonal_selection(..., 1, None) + + with pytest.warns(FutureWarning, match="Pass"): + arr.get_mask_selection(np.zeros((2, 2), dtype=bool), NDBuffer(array=np.empty((0,)))) + + with pytest.warns(FutureWarning, match="Pass"): + arr.set_mask_selection(np.zeros((2, 2), dtype=bool), 1, None) + + with pytest.warns(FutureWarning, match="Pass"): + arr.get_coordinate_selection(([0, 1], [0, 1]), NDBuffer(array=np.empty((2,)))) + + with pytest.warns(FutureWarning, match="Pass"): + arr.set_coordinate_selection(([0, 1], [0, 1]), 1, None) + + with pytest.warns(FutureWarning, match="Pass"): + arr.get_block_selection((0, slice(None)), NDBuffer(array=np.empty((2, 2)))) + + with pytest.warns(FutureWarning, match="Pass"): + arr.set_block_selection((0, slice(None)), 1, None) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +async def test_array_v3_nan_fill_value(store: MemoryStore) -> None: + shape = (10,) + arr = zarr.create_array( + store=store, + shape=shape, + dtype=np.float64, + zarr_format=3, + chunks=shape, + fill_value=np.nan, + ) + arr[:] = np.nan + + assert np.isnan(arr.fill_value) + assert arr.fill_value.dtype == arr.dtype + # all fill value chunk is an empty chunk, and should not be written + assert len([a async for a in store.list_prefix("/")]) == 0 + + +@pytest.mark.parametrize("store", ["local"], indirect=["store"]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +async def test_serializable_async_array( + store: LocalStore | MemoryStore, zarr_format: ZarrFormat +) -> None: + expected = await zarr.api.asynchronous.create_array( + store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype="i4" + ) + # await expected.setitems(list(range(100))) + + p = pickle.dumps(expected) + actual = pickle.loads(p) + + assert actual == expected + # np.testing.assert_array_equal(await actual.getitem(slice(None)), await expected.getitem(slice(None))) + # TODO: uncomment the parts of this test that will be impacted by the config/prototype changes in flight + + +@pytest.mark.parametrize("store", ["local"], indirect=["store"]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +def test_serializable_sync_array(store: LocalStore, zarr_format: ZarrFormat) -> None: + expected = zarr.create_array( + store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype="i4" + ) + expected[:] = list(range(100)) + + p = pickle.dumps(expected) + actual = pickle.loads(p) + + assert actual == expected + np.testing.assert_array_equal(actual[:], expected[:]) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("zarr_format", [2, 3, "invalid"]) +def test_storage_transformers(store: MemoryStore, zarr_format: ZarrFormat | str) -> None: + """ + Test that providing an actual storage transformer produces a warning and otherwise passes through + """ + metadata_dict: dict[str, JSON] + if zarr_format == 3: + metadata_dict = { + "zarr_format": 3, + "node_type": "array", + "shape": (10,), + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, + "data_type": "uint8", + "chunk_key_encoding": {"name": "v2", "configuration": {"separator": "/"}}, + "codecs": (BytesCodec().to_dict(),), + "fill_value": 0, + "storage_transformers": ({"test": "should_raise"}), + } + else: + metadata_dict = { + "zarr_format": zarr_format, + "shape": (10,), + "chunks": (1,), + "dtype": "|u1", + "dimension_separator": ".", + "codecs": (BytesCodec().to_dict(),), + "fill_value": 0, + "order": "C", + "storage_transformers": ({"test": "should_raise"}), + } + if zarr_format == 3: + match = "Arrays with storage transformers are not supported in zarr-python at this time." + with pytest.raises(ValueError, match=match): + Array.from_dict(StorePath(store), data=metadata_dict) + elif zarr_format == 2: + # no warning + Array.from_dict(StorePath(store), data=metadata_dict) + else: + match = f"Invalid zarr_format: {zarr_format}. Expected 2 or 3" + with pytest.raises(ValueError, match=match): + Array.from_dict(StorePath(store), data=metadata_dict) + + +@pytest.mark.parametrize("test_cls", [Array, AsyncArray[Any]]) +@pytest.mark.parametrize("nchunks", [2, 5, 10]) +def test_nchunks(test_cls: type[Array] | type[AsyncArray[Any]], nchunks: int) -> None: + """ + Test that nchunks returns the number of chunks defined for the array. + """ + store = MemoryStore() + shape = 100 + arr = zarr.create_array(store, shape=(shape,), chunks=(ceildiv(shape, nchunks),), dtype="i4") + expected = nchunks + if test_cls == Array: + observed = arr.nchunks + else: + observed = arr._async_array.nchunks + assert observed == expected + + +@pytest.mark.parametrize("test_cls", [Array, AsyncArray[Any]]) +async def test_nchunks_initialized(test_cls: type[Array] | type[AsyncArray[Any]]) -> None: + """ + Test that nchunks_initialized accurately returns the number of stored chunks. + """ + store = MemoryStore() + arr = zarr.create_array(store, shape=(100,), chunks=(10,), dtype="i4") + + # write chunks one at a time + for idx, region in enumerate(arr._iter_chunk_regions()): + arr[region] = 1 + expected = idx + 1 + if test_cls == Array: + observed = arr.nchunks_initialized + else: + observed = await arr._async_array.nchunks_initialized() + assert observed == expected + + # delete chunks + for idx, key in enumerate(arr._iter_chunk_keys()): + sync(arr.store_path.store.delete(key)) + if test_cls == Array: + observed = arr.nchunks_initialized + else: + observed = await arr._async_array.nchunks_initialized() + expected = arr.nchunks - idx - 1 + assert observed == expected + + +@pytest.mark.parametrize("path", ["", "foo"]) +async def test_chunks_initialized(path: str) -> None: + """ + Test that chunks_initialized accurately returns the keys of stored chunks. + """ + store = MemoryStore() + arr = zarr.create_array(store, name=path, shape=(100,), chunks=(10,), dtype="i4") + + chunks_accumulated = tuple( + accumulate(tuple(tuple(v.split(" ")) for v in arr._iter_chunk_keys())) + ) + for keys, region in zip(chunks_accumulated, arr._iter_chunk_regions(), strict=False): + arr[region] = 1 + observed = sorted(await chunks_initialized(arr._async_array)) + expected = sorted(keys) + assert observed == expected + + +def test_nbytes_stored() -> None: + arr = zarr.create(shape=(100,), chunks=(10,), dtype="i4", codecs=[BytesCodec()]) + result = arr.nbytes_stored() + assert result == 502 # the size of the metadata document. This is a fragile test. + arr[:50] = 1 + result = arr.nbytes_stored() + assert result == 702 # the size with 5 chunks filled. + arr[50:] = 2 + result = arr.nbytes_stored() + assert result == 902 # the size with all chunks filled. + + +async def test_nbytes_stored_async() -> None: + arr = await zarr.api.asynchronous.create( + shape=(100,), chunks=(10,), dtype="i4", codecs=[BytesCodec()] + ) + result = await arr.nbytes_stored() + assert result == 502 # the size of the metadata document. This is a fragile test. + await arr.setitem(slice(50), 1) + result = await arr.nbytes_stored() + assert result == 702 # the size with 5 chunks filled. + await arr.setitem(slice(50, 100), 2) + result = await arr.nbytes_stored() + assert result == 902 # the size with all chunks filled. + + +@pytest.mark.parametrize("zarr_format", [2, 3]) +def test_update_attrs(zarr_format: ZarrFormat) -> None: + # regression test for https://github.com/zarr-developers/zarr-python/issues/2328 + store = MemoryStore() + arr = zarr.create_array( + store=store, shape=(5,), chunks=(5,), dtype="f8", zarr_format=zarr_format + ) + arr.attrs["foo"] = "bar" + assert arr.attrs["foo"] == "bar" + + arr2 = zarr.open_array(store=store, zarr_format=zarr_format) + assert arr2.attrs["foo"] == "bar" + + +@pytest.mark.parametrize(("chunks", "shards"), [((2, 2), None), ((2, 2), (4, 4))]) +class TestInfo: + def test_info_v2(self, chunks: tuple[int, int], shards: tuple[int, int] | None) -> None: + arr = zarr.create_array(store={}, shape=(8, 8), dtype="f8", chunks=chunks, zarr_format=2) + result = arr.info + expected = ArrayInfo( + _zarr_format=2, + _data_type=arr._async_array._zdtype, + _fill_value=arr.fill_value, + _shape=(8, 8), + _chunk_shape=chunks, + _shard_shape=None, + _order="C", + _read_only=False, + _store_type="MemoryStore", + _count_bytes=512, + _compressors=(numcodecs.Zstd(),), + ) + assert result == expected + + def test_info_v3(self, chunks: tuple[int, int], shards: tuple[int, int] | None) -> None: + arr = zarr.create_array(store={}, shape=(8, 8), dtype="f8", chunks=chunks, shards=shards) + result = arr.info + expected = ArrayInfo( + _zarr_format=3, + _data_type=arr._async_array._zdtype, + _fill_value=arr.fill_value, + _shape=(8, 8), + _chunk_shape=chunks, + _shard_shape=shards, + _order="C", + _read_only=False, + _store_type="MemoryStore", + _compressors=(ZstdCodec(),), + _serializer=BytesCodec(), + _count_bytes=512, + ) + assert result == expected + + def test_info_complete(self, chunks: tuple[int, int], shards: tuple[int, int] | None) -> None: + arr = zarr.create_array( + store={}, + shape=(8, 8), + dtype="f8", + chunks=chunks, + shards=shards, + compressors=(), + ) + result = arr.info_complete() + expected = ArrayInfo( + _zarr_format=3, + _data_type=arr._async_array._zdtype, + _fill_value=arr.fill_value, + _shape=(8, 8), + _chunk_shape=chunks, + _shard_shape=shards, + _order="C", + _read_only=False, + _store_type="MemoryStore", + _serializer=BytesCodec(), + _count_bytes=512, + _count_chunks_initialized=0, + _count_bytes_stored=521 if shards is None else 982, # the metadata? + ) + assert result == expected + + arr[:4, :4] = 10 + result = arr.info_complete() + if shards is None: + expected = dataclasses.replace( + expected, _count_chunks_initialized=4, _count_bytes_stored=649 + ) + else: + expected = dataclasses.replace( + expected, _count_chunks_initialized=1, _count_bytes_stored=1178 + ) + assert result == expected + + async def test_info_v2_async( + self, chunks: tuple[int, int], shards: tuple[int, int] | None + ) -> None: + arr = await zarr.api.asynchronous.create_array( + store={}, shape=(8, 8), dtype="f8", chunks=chunks, zarr_format=2 + ) + result = arr.info + expected = ArrayInfo( + _zarr_format=2, + _data_type=Float64(), + _fill_value=arr.metadata.fill_value, + _shape=(8, 8), + _chunk_shape=(2, 2), + _shard_shape=None, + _order="C", + _read_only=False, + _store_type="MemoryStore", + _count_bytes=512, + _compressors=(numcodecs.Zstd(),), + ) + assert result == expected + + async def test_info_v3_async( + self, chunks: tuple[int, int], shards: tuple[int, int] | None + ) -> None: + arr = await zarr.api.asynchronous.create_array( + store={}, + shape=(8, 8), + dtype="f8", + chunks=chunks, + shards=shards, + ) + result = arr.info + expected = ArrayInfo( + _zarr_format=3, + _data_type=arr._zdtype, + _fill_value=arr.metadata.fill_value, + _shape=(8, 8), + _chunk_shape=chunks, + _shard_shape=shards, + _order="C", + _read_only=False, + _store_type="MemoryStore", + _compressors=(ZstdCodec(),), + _serializer=BytesCodec(), + _count_bytes=512, + ) + assert result == expected + + async def test_info_complete_async( + self, chunks: tuple[int, int], shards: tuple[int, int] | None + ) -> None: + arr = await zarr.api.asynchronous.create_array( + store={}, + dtype="f8", + shape=(8, 8), + chunks=chunks, + shards=shards, + compressors=None, + ) + result = await arr.info_complete() + expected = ArrayInfo( + _zarr_format=3, + _data_type=arr._zdtype, + _fill_value=arr.metadata.fill_value, + _shape=(8, 8), + _chunk_shape=chunks, + _shard_shape=shards, + _order="C", + _read_only=False, + _store_type="MemoryStore", + _serializer=BytesCodec(), + _count_bytes=512, + _count_chunks_initialized=0, + _count_bytes_stored=521 if shards is None else 982, # the metadata? + ) + assert result == expected + + await arr.setitem((slice(4), slice(4)), 10) + result = await arr.info_complete() + if shards is None: + expected = dataclasses.replace( + expected, _count_chunks_initialized=4, _count_bytes_stored=553 + ) + else: + expected = dataclasses.replace( + expected, _count_chunks_initialized=1, _count_bytes_stored=1178 + ) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_resize_1d(store: MemoryStore, zarr_format: ZarrFormat) -> None: + z = zarr.create( + shape=105, chunks=10, dtype="i4", fill_value=0, store=store, zarr_format=zarr_format + ) + a = np.arange(105, dtype="i4") + z[:] = a + result = z[:] + assert isinstance(result, NDArrayLike) + assert (105,) == z.shape + assert (105,) == result.shape + assert np.dtype("i4") == z.dtype + assert np.dtype("i4") == result.dtype + assert (10,) == z.chunks + np.testing.assert_array_equal(a, result) + + z.resize(205) + result = z[:] + assert isinstance(result, NDArrayLike) + assert (205,) == z.shape + assert (205,) == result.shape + assert np.dtype("i4") == z.dtype + assert np.dtype("i4") == result.dtype + assert (10,) == z.chunks + np.testing.assert_array_equal(a, z[:105]) + np.testing.assert_array_equal(np.zeros(100, dtype="i4"), z[105:]) + + z.resize(55) + result = z[:] + assert isinstance(result, NDArrayLike) + assert (55,) == z.shape + assert (55,) == result.shape + assert np.dtype("i4") == z.dtype + assert np.dtype("i4") == result.dtype + assert (10,) == z.chunks + np.testing.assert_array_equal(a[:55], result) + + # via shape setter + new_shape = (105,) + z.shape = new_shape + result = z[:] + assert isinstance(result, NDArrayLike) + assert new_shape == z.shape + assert new_shape == result.shape + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_resize_2d(store: MemoryStore, zarr_format: ZarrFormat) -> None: + z = zarr.create( + shape=(105, 105), + chunks=(10, 10), + dtype="i4", + fill_value=0, + store=store, + zarr_format=zarr_format, + ) + a = np.arange(105 * 105, dtype="i4").reshape((105, 105)) + z[:] = a + result = z[:] + assert isinstance(result, NDArrayLike) + assert (105, 105) == z.shape + assert (105, 105) == result.shape + assert np.dtype("i4") == z.dtype + assert np.dtype("i4") == result.dtype + assert (10, 10) == z.chunks + np.testing.assert_array_equal(a, result) + + z.resize((205, 205)) + result = z[:] + assert isinstance(result, NDArrayLike) + assert (205, 205) == z.shape + assert (205, 205) == result.shape + assert np.dtype("i4") == z.dtype + assert np.dtype("i4") == result.dtype + assert (10, 10) == z.chunks + np.testing.assert_array_equal(a, z[:105, :105]) + np.testing.assert_array_equal(np.zeros((100, 205), dtype="i4"), z[105:, :]) + np.testing.assert_array_equal(np.zeros((205, 100), dtype="i4"), z[:, 105:]) + + z.resize((55, 55)) + result = z[:] + assert isinstance(result, NDArrayLike) + assert (55, 55) == z.shape + assert (55, 55) == result.shape + assert np.dtype("i4") == z.dtype + assert np.dtype("i4") == result.dtype + assert (10, 10) == z.chunks + np.testing.assert_array_equal(a[:55, :55], result) + + z.resize((55, 1)) + result = z[:] + assert isinstance(result, NDArrayLike) + assert (55, 1) == z.shape + assert (55, 1) == result.shape + assert np.dtype("i4") == z.dtype + assert np.dtype("i4") == result.dtype + assert (10, 10) == z.chunks + np.testing.assert_array_equal(a[:55, :1], result) + + z.resize((1, 55)) + result = z[:] + assert isinstance(result, NDArrayLike) + assert (1, 55) == z.shape + assert (1, 55) == result.shape + assert np.dtype("i4") == z.dtype + assert np.dtype("i4") == result.dtype + assert (10, 10) == z.chunks + np.testing.assert_array_equal(a[:1, :10], z[:, :10]) + np.testing.assert_array_equal(np.zeros((1, 55 - 10), dtype="i4"), z[:, 10:55]) + + # via shape setter + new_shape = (105, 105) + z.shape = new_shape + result = z[:] + assert isinstance(result, NDArrayLike) + assert new_shape == z.shape + assert new_shape == result.shape + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_append_1d(store: MemoryStore, zarr_format: ZarrFormat) -> None: + a = np.arange(105) + z = zarr.create(shape=a.shape, chunks=10, dtype=a.dtype, store=store, zarr_format=zarr_format) + z[:] = a + assert a.shape == z.shape + assert a.dtype == z.dtype + assert (10,) == z.chunks + np.testing.assert_array_equal(a, z[:]) + + b = np.arange(105, 205) + e = np.append(a, b) + assert z.shape == (105,) + z.append(b) + assert e.shape == z.shape + assert e.dtype == z.dtype + assert (10,) == z.chunks + np.testing.assert_array_equal(e, z[:]) + + # check append handles array-like + c = [1, 2, 3] + f = np.append(e, c) + z.append(c) + assert f.shape == z.shape + assert f.dtype == z.dtype + assert (10,) == z.chunks + np.testing.assert_array_equal(f, z[:]) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_append_2d(store: MemoryStore, zarr_format: ZarrFormat) -> None: + a = np.arange(105 * 105, dtype="i4").reshape((105, 105)) + z = zarr.create( + shape=a.shape, chunks=(10, 10), dtype=a.dtype, store=store, zarr_format=zarr_format + ) + z[:] = a + assert a.shape == z.shape + assert a.dtype == z.dtype + assert (10, 10) == z.chunks + actual = z[:] + np.testing.assert_array_equal(a, actual) + + b = np.arange(105 * 105, 2 * 105 * 105, dtype="i4").reshape((105, 105)) + e = np.append(a, b, axis=0) + z.append(b) + assert e.shape == z.shape + assert e.dtype == z.dtype + assert (10, 10) == z.chunks + actual = z[:] + np.testing.assert_array_equal(e, actual) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_append_2d_axis(store: MemoryStore, zarr_format: ZarrFormat) -> None: + a = np.arange(105 * 105, dtype="i4").reshape((105, 105)) + z = zarr.create( + shape=a.shape, chunks=(10, 10), dtype=a.dtype, store=store, zarr_format=zarr_format + ) + z[:] = a + assert a.shape == z.shape + assert a.dtype == z.dtype + assert (10, 10) == z.chunks + np.testing.assert_array_equal(a, z[:]) + + b = np.arange(105 * 105, 2 * 105 * 105, dtype="i4").reshape((105, 105)) + e = np.append(a, b, axis=1) + z.append(b, axis=1) + assert e.shape == z.shape + assert e.dtype == z.dtype + assert (10, 10) == z.chunks + np.testing.assert_array_equal(e, z[:]) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_append_bad_shape(store: MemoryStore, zarr_format: ZarrFormat) -> None: + a = np.arange(100) + z = zarr.create(shape=a.shape, chunks=10, dtype=a.dtype, store=store, zarr_format=zarr_format) + z[:] = a + b = a.reshape(10, 10) + with pytest.raises(ValueError): + z.append(b) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("write_empty_chunks", [True, False]) +@pytest.mark.parametrize("fill_value", [0, 5]) +def test_write_empty_chunks_behavior( + zarr_format: ZarrFormat, store: MemoryStore, write_empty_chunks: bool, fill_value: int +) -> None: + """ + Check that the write_empty_chunks value of the config is applied correctly. We expect that + when write_empty_chunks is True, writing chunks equal to the fill value will result in + those chunks appearing in the store. + + When write_empty_chunks is False, writing chunks that are equal to the fill value will result in + those chunks not being present in the store. In particular, they should be deleted if they were + already present. + """ + + arr = zarr.create_array( + store=store, + shape=(2,), + zarr_format=zarr_format, + dtype="i4", + fill_value=fill_value, + chunks=(1,), + config={"write_empty_chunks": write_empty_chunks}, + ) + + assert arr._async_array._config.write_empty_chunks == write_empty_chunks + + # initialize the store with some non-fill value chunks + arr[:] = fill_value + 1 + assert arr.nchunks_initialized == arr.nchunks + + arr[:] = fill_value + + if not write_empty_chunks: + assert arr.nchunks_initialized == 0 + else: + assert arr.nchunks_initialized == arr.nchunks + + +@pytest.mark.parametrize( + ("fill_value", "expected"), + [ + (np.nan * 1j, ["NaN", "NaN"]), + (np.nan, ["NaN", 0.0]), + (np.inf, ["Infinity", 0.0]), + (np.inf * 1j, ["NaN", "Infinity"]), + (-np.inf, ["-Infinity", 0.0]), + (math.inf, ["Infinity", 0.0]), + ], +) +async def test_special_complex_fill_values_roundtrip(fill_value: Any, expected: list[Any]) -> None: + store = MemoryStore() + zarr.create_array(store=store, shape=(1,), dtype=np.complex64, fill_value=fill_value) + content = await store.get("zarr.json", prototype=default_buffer_prototype()) + assert content is not None + actual = json.loads(content.to_bytes()) + assert actual["fill_value"] == expected + + +@pytest.mark.parametrize("shape", [(1,), (2, 3), (4, 5, 6)]) +@pytest.mark.parametrize("dtype", ["uint8", "float32"]) +@pytest.mark.parametrize("array_type", ["async", "sync"]) +async def test_nbytes( + shape: tuple[int, ...], dtype: str, array_type: Literal["async", "sync"] +) -> None: + """ + Test that the ``nbytes`` attribute of an Array or AsyncArray correctly reports the capacity of + the chunks of that array. + """ + store = MemoryStore() + arr = zarr.create_array(store=store, shape=shape, dtype=dtype, fill_value=0) + if array_type == "async": + assert arr._async_array.nbytes == np.prod(arr.shape) * arr.dtype.itemsize + else: + assert arr.nbytes == np.prod(arr.shape) * arr.dtype.itemsize + + +@pytest.mark.parametrize( + ("array_shape", "chunk_shape"), + [((256,), (2,))], +) +def test_auto_partition_auto_shards( + array_shape: tuple[int, ...], chunk_shape: tuple[int, ...] +) -> None: + """ + Test that automatically picking a shard size returns a tuple of 2 * the chunk shape for any axis + where there are 8 or more chunks. + """ + dtype = np.dtype("uint8") + expected_shards: tuple[int, ...] = () + for cs, a_len in zip(chunk_shape, array_shape, strict=False): + if a_len // cs >= 8: + expected_shards += (2 * cs,) + else: + expected_shards += (cs,) + + auto_shards, _ = _auto_partition( + array_shape=array_shape, + chunk_shape=chunk_shape, + shard_shape="auto", + item_size=dtype.itemsize, + ) + assert auto_shards == expected_shards + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +class TestCreateArray: + @staticmethod + def test_chunks_and_shards(store: Store) -> None: + spath = StorePath(store) + shape = (100, 100) + chunks = (5, 5) + shards = (10, 10) + + arr_v3 = zarr.create_array(store=spath / "v3", shape=shape, chunks=chunks, dtype="i4") + assert arr_v3.chunks == chunks + assert arr_v3.shards is None + + arr_v3_sharding = zarr.create_array( + store=spath / "v3_sharding", + shape=shape, + chunks=chunks, + shards=shards, + dtype="i4", + ) + assert arr_v3_sharding.chunks == chunks + assert arr_v3_sharding.shards == shards + + arr_v2 = zarr.create_array( + store=spath / "v2", shape=shape, chunks=chunks, zarr_format=2, dtype="i4" + ) + assert arr_v2.chunks == chunks + assert arr_v2.shards is None + + @staticmethod + @pytest.mark.parametrize("dtype", zdtype_examples) + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + def test_default_fill_value(dtype: ZDType[Any, Any], store: Store) -> None: + """ + Test that the fill value of an array is set to the default value for the dtype object + """ + a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype) + if isinstance(dtype, DateTime64 | TimeDelta64) and np.isnat(a.fill_value): + assert np.isnat(dtype.default_scalar()) + else: + assert a.fill_value == dtype.default_scalar() + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("dtype", zdtype_examples) + def test_dtype_forms(dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFormat) -> None: + """ + Test that the same array is produced from a ZDType instance, a numpy dtype, or a numpy string + """ + skip_object_dtype(dtype) + a = zarr.create_array( + store, name="a", shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format + ) + + b = zarr.create_array( + store, + name="b", + shape=(5,), + chunks=(5,), + dtype=dtype.to_native_dtype(), + zarr_format=zarr_format, + ) + assert a.dtype == b.dtype + + # Structured dtypes do not have a numpy string representation that uniquely identifies them + if not isinstance(dtype, Structured): + if isinstance(dtype, VariableLengthUTF8): + # in numpy 2.3, StringDType().str becomes the string 'StringDType()' which numpy + # does not accept as a string representation of the dtype. + c = zarr.create_array( + store, + name="c", + shape=(5,), + chunks=(5,), + dtype=dtype.to_native_dtype().char, + zarr_format=zarr_format, + ) + else: + c = zarr.create_array( + store, + name="c", + shape=(5,), + chunks=(5,), + dtype=dtype.to_native_dtype().str, + zarr_format=zarr_format, + ) + assert a.dtype == c.dtype + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("dtype", zdtype_examples) + def test_dtype_roundtrip( + dtype: ZDType[Any, Any], store: Store, zarr_format: ZarrFormat + ) -> None: + """ + Test that creating an array, then opening it, gets the same array. + """ + skip_object_dtype(dtype) + a = zarr.create_array(store, shape=(5,), chunks=(5,), dtype=dtype, zarr_format=zarr_format) + b = zarr.open_array(store) + assert a.dtype == b.dtype + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("dtype", ["uint8", "float32", "U3", "S4", "V1"]) + @pytest.mark.parametrize( + "compressors", + [ + "auto", + None, + (), + (ZstdCodec(level=3),), + (ZstdCodec(level=3), GzipCodec(level=0)), + ZstdCodec(level=3), + {"name": "zstd", "configuration": {"level": 3}}, + ({"name": "zstd", "configuration": {"level": 3}},), + ], + ) + @pytest.mark.parametrize( + "filters", + [ + "auto", + None, + (), + ( + TransposeCodec( + order=[ + 0, + ] + ), + ), + ( + TransposeCodec( + order=[ + 0, + ] + ), + TransposeCodec( + order=[ + 0, + ] + ), + ), + TransposeCodec( + order=[ + 0, + ] + ), + {"name": "transpose", "configuration": {"order": [0]}}, + ({"name": "transpose", "configuration": {"order": [0]}},), + ], + ) + @pytest.mark.parametrize(("chunks", "shards"), [((6,), None), ((3,), (6,))]) + async def test_v3_chunk_encoding( + store: MemoryStore, + compressors: CompressorsLike, + filters: FiltersLike, + dtype: str, + chunks: tuple[int, ...], + shards: tuple[int, ...] | None, + ) -> None: + """ + Test various possibilities for the compressors and filters parameter to create_array + """ + arr = await create_array( + store=store, + dtype=dtype, + shape=(12,), + chunks=chunks, + shards=shards, + zarr_format=3, + filters=filters, + compressors=compressors, + ) + filters_expected, _, compressors_expected = _parse_chunk_encoding_v3( + filters=filters, + compressors=compressors, + serializer="auto", + dtype=arr._zdtype, + ) + assert arr.filters == filters_expected + assert arr.compressors == compressors_expected + + @staticmethod + @pytest.mark.parametrize("name", ["v2", "default", "invalid"]) + @pytest.mark.parametrize("separator", [".", "/"]) + async def test_chunk_key_encoding( + name: str, separator: Literal[".", "/"], zarr_format: ZarrFormat, store: MemoryStore + ) -> None: + chunk_key_encoding = ChunkKeyEncodingParams(name=name, separator=separator) # type: ignore[typeddict-item] + error_msg = "" + if name == "invalid": + error_msg = "Unknown chunk key encoding." + if zarr_format == 2 and name == "default": + error_msg = "Invalid chunk key encoding. For Zarr format 2 arrays, the `name` field of the chunk key encoding must be 'v2'." + if error_msg: + with pytest.raises(ValueError, match=re.escape(error_msg)): + arr = await create_array( + store=store, + dtype="uint8", + shape=(10,), + chunks=(1,), + zarr_format=zarr_format, + chunk_key_encoding=chunk_key_encoding, + ) + else: + arr = await create_array( + store=store, + dtype="uint8", + shape=(10,), + chunks=(1,), + zarr_format=zarr_format, + chunk_key_encoding=chunk_key_encoding, + ) + if isinstance(arr.metadata, ArrayV2Metadata): + assert arr.metadata.dimension_separator == separator + + @staticmethod + @pytest.mark.parametrize( + ("kwargs", "error_msg"), + [ + ({"serializer": "bytes"}, "Zarr format 2 arrays do not support `serializer`."), + ({"dimension_names": ["test"]}, "Zarr format 2 arrays do not support dimension names."), + ], + ) + async def test_create_array_invalid_v2_arguments( + kwargs: dict[str, Any], error_msg: str, store: MemoryStore + ) -> None: + with pytest.raises(ValueError, match=re.escape(error_msg)): + await zarr.api.asynchronous.create_array( + store=store, dtype="uint8", shape=(10,), chunks=(1,), zarr_format=2, **kwargs + ) + + @staticmethod + @pytest.mark.parametrize( + ("kwargs", "error_msg"), + [ + ( + {"dimension_names": ["test"]}, + "dimension_names cannot be used for arrays with zarr_format 2.", + ), + ( + {"chunk_key_encoding": {"name": "default", "separator": "/"}}, + "chunk_key_encoding cannot be used for arrays with zarr_format 2. Use dimension_separator instead.", + ), + ( + {"codecs": "bytes"}, + "codecs cannot be used for arrays with zarr_format 2. Use filters and compressor instead.", + ), + ], + ) + async def test_create_invalid_v2_arguments( + kwargs: dict[str, Any], error_msg: str, store: MemoryStore + ) -> None: + with pytest.raises(ValueError, match=re.escape(error_msg)): + await zarr.api.asynchronous.create( + store=store, dtype="uint8", shape=(10,), chunks=(1,), zarr_format=2, **kwargs + ) + + @staticmethod + @pytest.mark.parametrize( + ("kwargs", "error_msg"), + [ + ( + {"chunk_shape": (1,), "chunks": (2,)}, + "Only one of chunk_shape or chunks can be provided.", + ), + ( + {"dimension_separator": "/"}, + "dimension_separator cannot be used for arrays with zarr_format 3. Use chunk_key_encoding instead.", + ), + ( + {"filters": []}, + "filters cannot be used for arrays with zarr_format 3. Use array-to-array codecs instead", + ), + ( + {"compressor": "blosc"}, + "compressor cannot be used for arrays with zarr_format 3. Use bytes-to-bytes codecs instead", + ), + ], + ) + async def test_invalid_v3_arguments( + kwargs: dict[str, Any], error_msg: str, store: MemoryStore + ) -> None: + kwargs.setdefault("chunks", (1,)) + with pytest.raises(ValueError, match=re.escape(error_msg)): + zarr.create(store=store, dtype="uint8", shape=(10,), zarr_format=3, **kwargs) + + @staticmethod + @pytest.mark.parametrize("dtype", ["uint8", "float32", "str", "U10", "S10", ">M8[10s]"]) + @pytest.mark.parametrize( + "compressors", + [ + "auto", + None, + numcodecs.Zstd(level=3), + (), + (numcodecs.Zstd(level=3),), + ], + ) + @pytest.mark.parametrize( + "filters", ["auto", None, numcodecs.GZip(level=1), (numcodecs.GZip(level=1),)] + ) + async def test_v2_chunk_encoding( + store: MemoryStore, compressors: CompressorsLike, filters: FiltersLike, dtype: str + ) -> None: + arr = await create_array( + store=store, + dtype=dtype, + shape=(10,), + zarr_format=2, + compressors=compressors, + filters=filters, + ) + filters_expected, compressor_expected = _parse_chunk_encoding_v2( + filters=filters, compressor=compressors, dtype=get_data_type_from_native_dtype(dtype) + ) + assert arr.metadata.zarr_format == 2 # guard for mypy + assert arr.metadata.compressor == compressor_expected + assert arr.metadata.filters == filters_expected + + # Normalize for property getters + compressor_expected = () if compressor_expected is None else (compressor_expected,) + filters_expected = () if filters_expected is None else filters_expected + + assert arr.compressors == compressor_expected + assert arr.filters == filters_expected + + @staticmethod + @pytest.mark.parametrize("dtype", [UInt8(), Float32(), VariableLengthUTF8()]) + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + async def test_default_filters_compressors( + store: MemoryStore, dtype: UInt8 | Float32 | VariableLengthUTF8, zarr_format: ZarrFormat + ) -> None: + """ + Test that the default ``filters`` and ``compressors`` are used when ``create_array`` is invoked with ``filters`` and ``compressors`` unspecified. + """ + + arr = await create_array( + store=store, + dtype=dtype, # type: ignore[arg-type] + shape=(10,), + zarr_format=zarr_format, + ) + + sig = inspect.signature(create_array) + + if zarr_format == 3: + expected_filters, expected_serializer, expected_compressors = _parse_chunk_encoding_v3( + compressors=sig.parameters["compressors"].default, + filters=sig.parameters["filters"].default, + serializer=sig.parameters["serializer"].default, + dtype=dtype, # type: ignore[arg-type] + ) + + elif zarr_format == 2: + default_filters, default_compressors = _parse_chunk_encoding_v2( + compressor=sig.parameters["compressors"].default, + filters=sig.parameters["filters"].default, + dtype=dtype, # type: ignore[arg-type] + ) + if default_filters is None: + expected_filters = () + else: + expected_filters = default_filters + if default_compressors is None: + expected_compressors = () + else: + expected_compressors = (default_compressors,) + expected_serializer = None + else: + raise ValueError(f"Invalid zarr_format: {zarr_format}") + + assert arr.filters == expected_filters + assert arr.serializer == expected_serializer + assert arr.compressors == expected_compressors + + @staticmethod + async def test_v2_no_shards(store: Store) -> None: + """ + Test that creating a Zarr v2 array with ``shard_shape`` set to a non-None value raises an error. + """ + msg = re.escape( + "Zarr format 2 arrays can only be created with `shard_shape` set to `None`. Got `shard_shape=(5,)` instead." + ) + with pytest.raises(ValueError, match=msg): + _ = await create_array( + store=store, + dtype="uint8", + shape=(10,), + shards=(5,), + zarr_format=2, + ) + + @staticmethod + @pytest.mark.parametrize("impl", ["sync", "async"]) + async def test_with_data(impl: Literal["sync", "async"], store: Store) -> None: + """ + Test that we can invoke ``create_array`` with a ``data`` parameter. + """ + data = np.arange(10) + name = "foo" + arr: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | Array + if impl == "sync": + arr = sync_api.create_array(store, name=name, data=data) + stored = arr[:] + elif impl == "async": + arr = await create_array(store, name=name, data=data, zarr_format=3) + stored = await arr._get_selection( + BasicIndexer(..., shape=arr.shape, chunk_grid=arr.metadata.chunk_grid), + prototype=default_buffer_prototype(), + ) + else: + raise ValueError(f"Invalid impl: {impl}") + + assert np.array_equal(stored, data) + + @staticmethod + async def test_with_data_invalid_params(store: Store) -> None: + """ + Test that failing to specify data AND shape / dtype results in a ValueError + """ + with pytest.raises(ValueError, match="shape was not specified"): + await create_array(store, data=None, shape=None, dtype=None) + + # we catch shape=None first, so specifying a dtype should raise the same exception as before + with pytest.raises(ValueError, match="shape was not specified"): + await create_array(store, data=None, shape=None, dtype="uint8") + + with pytest.raises(ValueError, match="dtype was not specified"): + await create_array(store, data=None, shape=(10, 10)) + + @staticmethod + async def test_data_ignored_params(store: Store) -> None: + """ + Test that specifying data AND shape AND dtype results in a ValueError + """ + data = np.arange(10) + with pytest.raises( + ValueError, match="The data parameter was used, but the shape parameter was also used." + ): + await create_array(store, data=data, shape=data.shape, dtype=None, overwrite=True) + + # we catch shape first, so specifying a dtype should raise the same warning as before + with pytest.raises( + ValueError, match="The data parameter was used, but the shape parameter was also used." + ): + await create_array(store, data=data, shape=data.shape, dtype=data.dtype, overwrite=True) + + with pytest.raises( + ValueError, match="The data parameter was used, but the dtype parameter was also used." + ): + await create_array(store, data=data, shape=None, dtype=data.dtype, overwrite=True) + + @staticmethod + @pytest.mark.parametrize("order", ["C", "F", None]) + @pytest.mark.parametrize("with_config", [True, False]) + def test_order( + order: MemoryOrder | None, + with_config: bool, + zarr_format: ZarrFormat, + store: MemoryStore, + ) -> None: + """ + Test that the arrays generated by array indexing have a memory order defined by the config order + value, and that for zarr v2 arrays, the ``order`` field in the array metadata is set correctly. + """ + config: ArrayConfigLike | None = {} + if order is None: + config = {} + expected = zarr.config.get("array.order") + else: + config = {"order": order} + expected = order + + if not with_config: + # Test without passing config parameter + config = None + + arr = zarr.create_array( + store=store, + shape=(2, 2), + zarr_format=zarr_format, + dtype="i4", + order=order, + config=config, + ) + assert arr.order == expected + if zarr_format == 2: + assert arr.metadata.zarr_format == 2 + assert arr.metadata.order == expected + + vals = np.asarray(arr) + if expected == "C": + assert vals.flags.c_contiguous + elif expected == "F": + assert vals.flags.f_contiguous + else: + raise AssertionError + + @staticmethod + @pytest.mark.parametrize("write_empty_chunks", [True, False]) + async def test_write_empty_chunks_config(write_empty_chunks: bool, store: Store) -> None: + """ + Test that the value of write_empty_chunks is sensitive to the global config when not set + explicitly + """ + with zarr.config.set({"array.write_empty_chunks": write_empty_chunks}): + arr = await create_array(store, shape=(2, 2), dtype="i4") + assert arr._config.write_empty_chunks == write_empty_chunks + + @staticmethod + @pytest.mark.parametrize("path", [None, "", "/", "/foo", "foo", "foo/bar"]) + async def test_name(store: Store, zarr_format: ZarrFormat, path: str | None) -> None: + arr = await create_array( + store, shape=(2, 2), dtype="i4", name=path, zarr_format=zarr_format + ) + if path is None: + expected_path = "" + elif path.startswith("/"): + expected_path = path.lstrip("/") + else: + expected_path = path + assert arr.path == expected_path + assert arr.name == "/" + expected_path + + # test that implicit groups were created + path_parts = expected_path.split("/") + if len(path_parts) > 1: + *parents, _ = ["", *accumulate(path_parts, lambda x, y: "/".join([x, y]))] # noqa: FLY002 + for parent_path in parents: + # this will raise if these groups were not created + _ = await zarr.api.asynchronous.open_group( + store=store, path=parent_path, zarr_format=zarr_format + ) + + @staticmethod + @pytest.mark.parametrize("endianness", ENDIANNESS_STR) + def test_default_endianness( + store: Store, zarr_format: ZarrFormat, endianness: EndiannessStr + ) -> None: + """ + Test that that endianness is correctly set when creating an array when not specifying a serializer + """ + dtype = Int16(endianness=endianness) + arr = zarr.create_array(store=store, shape=(1,), dtype=dtype, zarr_format=zarr_format) + byte_order: str = arr[:].dtype.byteorder # type: ignore[union-attr] + assert byte_order in NUMPY_ENDIANNESS_STR + assert endianness_from_numpy_str(byte_order) == endianness # type: ignore[arg-type] + + +@pytest.mark.parametrize("value", [1, 1.4, "a", b"a", np.array(1)]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_scalar_array(value: Any, zarr_format: ZarrFormat) -> None: + arr = zarr.array(value, zarr_format=zarr_format) + assert arr[...] == value + assert arr.shape == () + assert arr.ndim == 0 + assert isinstance(arr[()], NDArrayLikeOrScalar) + + +@pytest.mark.parametrize("store", ["local"], indirect=True) +@pytest.mark.parametrize("store2", ["local"], indirect=["store2"]) +@pytest.mark.parametrize("src_format", [2, 3]) +@pytest.mark.parametrize("new_format", [2, 3, None]) +async def test_creation_from_other_zarr_format( + store: Store, + store2: Store, + src_format: ZarrFormat, + new_format: ZarrFormat | None, +) -> None: + if src_format == 2: + src = zarr.create( + (50, 50), chunks=(10, 10), store=store, zarr_format=src_format, dimension_separator="/" + ) + else: + src = zarr.create( + (50, 50), + chunks=(10, 10), + store=store, + zarr_format=src_format, + chunk_key_encoding=("default", "."), + ) + + src[:] = np.arange(50 * 50).reshape((50, 50)) + result = zarr.from_array( + store=store2, + data=src, + zarr_format=new_format, + ) + np.testing.assert_array_equal(result[:], src[:]) + assert result.fill_value == src.fill_value + assert result.dtype == src.dtype + assert result.chunks == src.chunks + expected_format = src_format if new_format is None else new_format + assert result.metadata.zarr_format == expected_format + if src_format == new_format: + assert result.metadata == src.metadata + + result2 = zarr.array( + data=src, + store=store2, + overwrite=True, + zarr_format=new_format, + ) + np.testing.assert_array_equal(result2[:], src[:]) + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) +@pytest.mark.parametrize("store2", ["local", "memory", "zip"], indirect=["store2"]) +@pytest.mark.parametrize("src_chunks", [(40, 10), (11, 50)]) +@pytest.mark.parametrize("new_chunks", [(40, 10), (11, 50)]) +async def test_from_array( + store: Store, + store2: Store, + src_chunks: tuple[int, int], + new_chunks: tuple[int, int], + zarr_format: ZarrFormat, +) -> None: + src_fill_value = 2 + src_dtype = np.dtype("uint8") + src_attributes = None + + src = zarr.create( + (100, 10), + chunks=src_chunks, + dtype=src_dtype, + store=store, + fill_value=src_fill_value, + attributes=src_attributes, + ) + src[:] = np.arange(1000).reshape((100, 10)) + + new_fill_value = 3 + new_attributes: dict[str, JSON] = {"foo": "bar"} + + result = zarr.from_array( + data=src, + store=store2, + chunks=new_chunks, + fill_value=new_fill_value, + attributes=new_attributes, + ) + + np.testing.assert_array_equal(result[:], src[:]) + assert result.fill_value == new_fill_value + assert result.dtype == src_dtype + assert result.attrs == new_attributes + assert result.chunks == new_chunks + + +@pytest.mark.parametrize("store", ["local"], indirect=True) +@pytest.mark.parametrize("chunks", ["keep", "auto"]) +@pytest.mark.parametrize("write_data", [True, False]) +@pytest.mark.parametrize( + "src", + [ + np.arange(1000).reshape(10, 10, 10), + zarr.ones((10, 10, 10)), + 5, + [1, 2, 3], + [[1, 2, 3], [4, 5, 6]], + ], +) # add other npt.ArrayLike? +async def test_from_array_arraylike( + store: Store, + chunks: Literal["auto", "keep"] | tuple[int, int], + write_data: bool, + src: Array | npt.ArrayLike, +) -> None: + fill_value = 42 + result = zarr.from_array( + store, data=src, chunks=chunks, write_data=write_data, fill_value=fill_value + ) + if write_data: + np.testing.assert_array_equal(result[...], np.array(src)) + else: + np.testing.assert_array_equal(result[...], np.full_like(src, fill_value)) + + +async def test_orthogonal_set_total_slice() -> None: + """Ensure that a whole chunk overwrite does not read chunks""" + store = MemoryStore() + array = zarr.create_array(store, shape=(20, 20), chunks=(1, 2), dtype=int, fill_value=-1) + with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): + array[0, slice(4, 10)] = np.arange(6) + + array = zarr.create_array( + store, shape=(20, 21), chunks=(1, 2), dtype=int, fill_value=-1, overwrite=True + ) + with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): + array[0, :] = np.arange(21) + + with mock.patch("zarr.storage.MemoryStore.get", side_effect=RuntimeError): + array[:] = 1 + + +@pytest.mark.skipif( + Version(numcodecs.__version__) < Version("0.15.1"), + reason="codec configuration is overwritten on older versions. GH2800", +) +def test_roundtrip_numcodecs() -> None: + store = MemoryStore() + + compressors = [ + {"name": "numcodecs.shuffle", "configuration": {"elementsize": 2}}, + {"name": "numcodecs.zlib", "configuration": {"level": 4}}, + ] + filters = [ + { + "name": "numcodecs.fixedscaleoffset", + "configuration": { + "scale": 100.0, + "offset": 0.0, + "dtype": " Any: + return arr[index] + + +@pytest.mark.parametrize( + "method", + [ + pytest.param( + "fork", + marks=pytest.mark.skipif( + sys.platform in ("win32", "darwin"), reason="fork not supported on Windows or OSX" + ), + ), + "spawn", + pytest.param( + "forkserver", + marks=pytest.mark.skipif( + sys.platform == "win32", reason="forkserver not supported on Windows" + ), + ), + ], +) +@pytest.mark.parametrize("store", ["local"], indirect=True) +def test_multiprocessing(store: Store, method: Literal["fork", "spawn", "forkserver"]) -> None: + """ + Test that arrays can be pickled and indexed in child processes + """ + data = np.arange(100) + arr = zarr.create_array(store=store, data=data) + ctx = mp.get_context(method) + with ctx.Pool() as pool: + results = pool.starmap(_index_array, [(arr, slice(len(data)))]) + assert all(np.array_equal(r, data) for r in results) + + +async def test_sharding_coordinate_selection() -> None: + store = MemoryStore() + g = zarr.open_group(store, mode="w") + arr = g.create_array( + name="a", + shape=(2, 3, 4), + chunks=(1, 2, 2), + overwrite=True, + dtype=np.float32, + shards=(2, 4, 4), + ) + arr[:] = np.arange(2 * 3 * 4).reshape((2, 3, 4)) + result = arr[1, [0, 1]] # type: ignore[index] + assert isinstance(result, NDArrayLike) + assert (result == np.array([[12, 13, 14, 15], [16, 17, 18, 19]])).all() + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +def test_array_repr(store: Store) -> None: + shape = (2, 3, 4) + dtype = "uint8" + arr = zarr.create_array(store, shape=shape, dtype=dtype) + assert str(arr) == f"" diff --git a/tests/test_attributes.py b/tests/test_attributes.py new file mode 100644 index 0000000000..127b2dbc36 --- /dev/null +++ b/tests/test_attributes.py @@ -0,0 +1,84 @@ +import pytest + +import zarr.core +import zarr.core.attributes +import zarr.storage + + +def test_put() -> None: + store = zarr.storage.MemoryStore() + attrs = zarr.core.attributes.Attributes( + zarr.Group.from_store(store, attributes={"a": 1, "b": 2}) + ) + attrs.put({"a": 3, "c": 4}) + expected = {"a": 3, "c": 4} + assert dict(attrs) == expected + + +def test_asdict() -> None: + store = zarr.storage.MemoryStore() + attrs = zarr.core.attributes.Attributes( + zarr.Group.from_store(store, attributes={"a": 1, "b": 2}) + ) + result = attrs.asdict() + assert result == {"a": 1, "b": 2} + + +def test_update_attributes_preserves_existing() -> None: + """ + Test that `update_attributes` only updates the specified attributes + and preserves existing ones. + """ + store = zarr.storage.MemoryStore() + z = zarr.create(10, store=store, overwrite=True) + z.attrs["a"] = [] + z.attrs["b"] = 3 + assert dict(z.attrs) == {"a": [], "b": 3} + + z.update_attributes({"a": [3, 4], "c": 4}) + assert dict(z.attrs) == {"a": [3, 4], "b": 3, "c": 4} + + +def test_update_empty_attributes() -> None: + """ + Ensure updating when initial attributes are empty works. + """ + store = zarr.storage.MemoryStore() + z = zarr.create(10, store=store, overwrite=True) + assert dict(z.attrs) == {} + z.update_attributes({"a": [3, 4], "c": 4}) + assert dict(z.attrs) == {"a": [3, 4], "c": 4} + + +def test_update_no_changes() -> None: + """ + Ensure updating when no new or modified attributes does not alter existing ones. + """ + store = zarr.storage.MemoryStore() + z = zarr.create(10, store=store, overwrite=True) + z.attrs["a"] = [] + z.attrs["b"] = 3 + + z.update_attributes({}) + assert dict(z.attrs) == {"a": [], "b": 3} + + +@pytest.mark.parametrize("group", [True, False]) +def test_del_works(group: bool) -> None: + store = zarr.storage.MemoryStore() + z: zarr.Group | zarr.Array + if group: + z = zarr.create_group(store) + else: + z = zarr.create_array(store=store, shape=10, dtype=int) + assert dict(z.attrs) == {} + z.update_attributes({"a": [3, 4], "c": 4}) + del z.attrs["a"] + assert dict(z.attrs) == {"c": 4} + + z2: zarr.Group | zarr.Array + if group: + z2 = zarr.open_group(store) + else: + z2 = zarr.open_array(store) + assert dict(z2.attrs) == {"c": 4} diff --git a/tests/test_buffer.py b/tests/test_buffer.py new file mode 100644 index 0000000000..93b116e908 --- /dev/null +++ b/tests/test_buffer.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +import pytest + +import zarr +from zarr.abc.buffer import ArrayLike, BufferPrototype, NDArrayLike +from zarr.buffer import cpu, gpu +from zarr.codecs.blosc import BloscCodec +from zarr.codecs.crc32c_ import Crc32cCodec +from zarr.codecs.gzip import GzipCodec +from zarr.codecs.transpose import TransposeCodec +from zarr.codecs.zstd import ZstdCodec +from zarr.storage import MemoryStore, StorePath +from zarr.testing.buffer import ( + NDBufferUsingTestNDArrayLike, + StoreExpectingTestBuffer, + TestBuffer, + TestNDArrayLike, +) +from zarr.testing.utils import gpu_test + +if TYPE_CHECKING: + import types + +try: + import cupy as cp +except ImportError: + cp = None + + +import zarr.api.asynchronous + +if TYPE_CHECKING: + import types + + +def test_nd_array_like(xp: types.ModuleType) -> None: + ary = xp.arange(10) + assert isinstance(ary, ArrayLike) + assert isinstance(ary, NDArrayLike) + + +@pytest.mark.asyncio +async def test_async_array_prototype() -> None: + """Test the use of a custom buffer prototype""" + + expect = np.zeros((9, 9), dtype="uint16", order="F") + a = await zarr.api.asynchronous.create_array( + StorePath(StoreExpectingTestBuffer()) / "test_async_array_prototype", + shape=expect.shape, + chunks=(5, 5), + dtype=expect.dtype, + fill_value=0, + ) + expect[1:4, 3:6] = np.ones((3, 3)) + + my_prototype = BufferPrototype(buffer=TestBuffer, nd_buffer=NDBufferUsingTestNDArrayLike) + + await a.setitem( + selection=(slice(1, 4), slice(3, 6)), + value=np.ones((3, 3)), + prototype=my_prototype, + ) + got = await a.getitem(selection=(slice(0, 9), slice(0, 9)), prototype=my_prototype) + # ignoring a mypy error here that TestNDArrayLike doesn't meet the NDArrayLike protocol + # The test passes, so it clearly does. + assert isinstance(got, TestNDArrayLike) + assert np.array_equal(expect, got) # type: ignore[unreachable] + + +@gpu_test +@pytest.mark.asyncio +async def test_async_array_gpu_prototype() -> None: + """Test the use of the GPU buffer prototype""" + + expect = cp.zeros((9, 9), dtype="uint16", order="F") + a = await zarr.api.asynchronous.create_array( + StorePath(MemoryStore()) / "test_async_array_gpu_prototype", + shape=expect.shape, + chunks=(5, 5), + dtype=expect.dtype, + fill_value=0, + ) + expect[1:4, 3:6] = cp.ones((3, 3)) + + await a.setitem( + selection=(slice(1, 4), slice(3, 6)), + value=cp.ones((3, 3)), + prototype=gpu.buffer_prototype, + ) + got = await a.getitem(selection=(slice(0, 9), slice(0, 9)), prototype=gpu.buffer_prototype) + assert isinstance(got, cp.ndarray) + assert cp.array_equal(expect, got) + + +@pytest.mark.asyncio +async def test_codecs_use_of_prototype() -> None: + expect = np.zeros((10, 10), dtype="uint16", order="F") + a = await zarr.api.asynchronous.create_array( + StorePath(StoreExpectingTestBuffer()) / "test_codecs_use_of_prototype", + shape=expect.shape, + chunks=(5, 5), + dtype=expect.dtype, + fill_value=0, + compressors=[BloscCodec(), Crc32cCodec(), GzipCodec(), ZstdCodec()], + filters=[TransposeCodec(order=(1, 0))], + ) + expect[:] = np.arange(100).reshape(10, 10) + + my_prototype = BufferPrototype(buffer=TestBuffer, nd_buffer=NDBufferUsingTestNDArrayLike) + + await a.setitem( + selection=(slice(0, 10), slice(0, 10)), + value=expect[:], + prototype=my_prototype, + ) + got = await a.getitem(selection=(slice(0, 10), slice(0, 10)), prototype=my_prototype) + # ignoring a mypy error here that TestNDArrayLike doesn't meet the NDArrayLike protocol + # The test passes, so it clearly does. + assert isinstance(got, TestNDArrayLike) + assert np.array_equal(expect, got) # type: ignore[unreachable] + + +@gpu_test +@pytest.mark.asyncio +async def test_codecs_use_of_gpu_prototype() -> None: + expect = cp.zeros((10, 10), dtype="uint16", order="F") + a = await zarr.api.asynchronous.create_array( + StorePath(MemoryStore()) / "test_codecs_use_of_gpu_prototype", + shape=expect.shape, + chunks=(5, 5), + dtype=expect.dtype, + fill_value=0, + compressors=[BloscCodec(), Crc32cCodec(), GzipCodec(), ZstdCodec()], + filters=[TransposeCodec(order=(1, 0))], + ) + expect[:] = cp.arange(100).reshape(10, 10) + + await a.setitem( + selection=(slice(0, 10), slice(0, 10)), + value=expect[:], + prototype=gpu.buffer_prototype, + ) + got = await a.getitem(selection=(slice(0, 10), slice(0, 10)), prototype=gpu.buffer_prototype) + assert isinstance(got, cp.ndarray) + assert cp.array_equal(expect, got) + + +@gpu_test +@pytest.mark.asyncio +async def test_sharding_use_of_gpu_prototype() -> None: + with zarr.config.enable_gpu(): + expect = cp.zeros((10, 10), dtype="uint16", order="F") + + a = await zarr.api.asynchronous.create_array( + StorePath(MemoryStore()) / "test_codecs_use_of_gpu_prototype", + shape=expect.shape, + chunks=(5, 5), + shards=(10, 10), + dtype=expect.dtype, + fill_value=0, + ) + expect[:] = cp.arange(100).reshape(10, 10) + + await a.setitem( + selection=(slice(0, 10), slice(0, 10)), + value=expect[:], + prototype=gpu.buffer_prototype, + ) + got = await a.getitem( + selection=(slice(0, 10), slice(0, 10)), prototype=gpu.buffer_prototype + ) + assert isinstance(got, cp.ndarray) + assert cp.array_equal(expect, got) + + +def test_numpy_buffer_prototype() -> None: + buffer = cpu.buffer_prototype.buffer.create_zero_length() + ndbuffer = cpu.buffer_prototype.nd_buffer.create(shape=(1, 2), dtype=np.dtype("int64")) + assert isinstance(buffer.as_array_like(), np.ndarray) + assert isinstance(ndbuffer.as_ndarray_like(), np.ndarray) + with pytest.raises(ValueError, match="Buffer does not contain a single scalar value"): + ndbuffer.as_scalar() + + +@gpu_test +def test_gpu_buffer_prototype() -> None: + buffer = gpu.buffer_prototype.buffer.create_zero_length() + ndbuffer = gpu.buffer_prototype.nd_buffer.create(shape=(1, 2), dtype=cp.dtype("int64")) + assert isinstance(buffer.as_array_like(), cp.ndarray) + assert isinstance(ndbuffer.as_ndarray_like(), cp.ndarray) + with pytest.raises(ValueError, match="Buffer does not contain a single scalar value"): + ndbuffer.as_scalar() + + +# TODO: the same test for other buffer classes +def test_cpu_buffer_as_scalar() -> None: + buf = cpu.buffer_prototype.nd_buffer.create(shape=(), dtype="int64") + assert buf.as_scalar() == buf.as_ndarray_like()[()] # type: ignore[index] diff --git a/tests/test_chunk_grids.py b/tests/test_chunk_grids.py new file mode 100644 index 0000000000..4c69c483ae --- /dev/null +++ b/tests/test_chunk_grids.py @@ -0,0 +1,54 @@ +from typing import Any + +import numpy as np +import pytest + +from zarr.core.chunk_grids import _guess_chunks, normalize_chunks + + +@pytest.mark.parametrize( + "shape", [(0,), (0,) * 2, (1, 2, 0, 4, 5), (10, 0), (10,), (100,) * 3, (1000000,), (10000,) * 2] +) +@pytest.mark.parametrize("itemsize", [1, 2, 4]) +def test_guess_chunks(shape: tuple[int, ...], itemsize: int) -> None: + chunks = _guess_chunks(shape, itemsize) + chunk_size = np.prod(chunks) * itemsize + assert isinstance(chunks, tuple) + assert len(chunks) == len(shape) + assert chunk_size < (64 * 1024 * 1024) + # doesn't make any sense to allow chunks to have zero length dimension + assert all(0 < c <= max(s, 1) for c, s in zip(chunks, shape, strict=False)) + + +@pytest.mark.parametrize( + ("chunks", "shape", "typesize", "expected"), + [ + ((10,), (100,), 1, (10,)), + ([10], (100,), 1, (10,)), + (10, (100,), 1, (10,)), + ((10, 10), (100, 10), 1, (10, 10)), + (10, (100, 10), 1, (10, 10)), + ((10, None), (100, 10), 1, (10, 10)), + (30, (100, 20, 10), 1, (30, 30, 30)), + ((30,), (100, 20, 10), 1, (30, 20, 10)), + ((30, None), (100, 20, 10), 1, (30, 20, 10)), + ((30, None, None), (100, 20, 10), 1, (30, 20, 10)), + ((30, 20, None), (100, 20, 10), 1, (30, 20, 10)), + ((30, 20, 10), (100, 20, 10), 1, (30, 20, 10)), + # auto chunking + (None, (100,), 1, (100,)), + (-1, (100,), 1, (100,)), + ((30, -1, None), (100, 20, 10), 1, (30, 20, 10)), + ], +) +def test_normalize_chunks( + chunks: Any, shape: tuple[int, ...], typesize: int, expected: tuple[int, ...] +) -> None: + assert expected == normalize_chunks(chunks, shape, typesize) + + +def test_normalize_chunks_errors() -> None: + with pytest.raises(ValueError): + normalize_chunks("foo", (100,), 1) + with pytest.raises(ValueError): + normalize_chunks((100, 10), (100,), 1) diff --git a/tests/test_codec_entrypoints.py b/tests/test_codec_entrypoints.py new file mode 100644 index 0000000000..e1ef027dd4 --- /dev/null +++ b/tests/test_codec_entrypoints.py @@ -0,0 +1,50 @@ +import os.path +import sys +from collections.abc import Generator + +import pytest + +import zarr.registry +from zarr import config + +here = os.path.abspath(os.path.dirname(__file__)) + + +@pytest.fixture +def set_path() -> Generator[None, None, None]: + sys.path.append(here) + zarr.registry._collect_entrypoints() + yield + sys.path.remove(here) + registries = zarr.registry._collect_entrypoints() + for registry in registries: + registry.lazy_load_list.clear() + config.reset() + + +@pytest.mark.usefixtures("set_path") +@pytest.mark.parametrize("codec_name", ["TestEntrypointCodec", "TestEntrypointGroup.Codec"]) +def test_entrypoint_codec(codec_name: str) -> None: + config.set({"codecs.test": "package_with_entrypoint." + codec_name}) + cls_test = zarr.registry.get_codec_class("test") + assert cls_test.__qualname__ == codec_name + + +@pytest.mark.usefixtures("set_path") +def test_entrypoint_pipeline() -> None: + config.set({"codec_pipeline.path": "package_with_entrypoint.TestEntrypointCodecPipeline"}) + cls = zarr.registry.get_pipeline_class() + assert cls.__name__ == "TestEntrypointCodecPipeline" + + +@pytest.mark.usefixtures("set_path") +@pytest.mark.parametrize("buffer_name", ["TestEntrypointBuffer", "TestEntrypointGroup.Buffer"]) +def test_entrypoint_buffer(buffer_name: str) -> None: + config.set( + { + "buffer": "package_with_entrypoint." + buffer_name, + "ndbuffer": "package_with_entrypoint.TestEntrypointNDBuffer", + } + ) + assert zarr.registry.get_buffer_class().__qualname__ == buffer_name + assert zarr.registry.get_ndbuffer_class().__name__ == "TestEntrypointNDBuffer" diff --git a/tests/test_codecs/__init__.py b/tests/test_codecs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_codecs/test_blosc.py b/tests/test_codecs/test_blosc.py new file mode 100644 index 0000000000..6e6e9df383 --- /dev/null +++ b/tests/test_codecs/test_blosc.py @@ -0,0 +1,75 @@ +import json + +import numcodecs +import numpy as np +import pytest +from packaging.version import Version + +import zarr +from zarr.abc.store import Store +from zarr.codecs import BloscCodec +from zarr.core.buffer import default_buffer_prototype +from zarr.storage import StorePath + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +@pytest.mark.parametrize("dtype", ["uint8", "uint16"]) +async def test_blosc_evolve(store: Store, dtype: str) -> None: + typesize = np.dtype(dtype).itemsize + path = "blosc_evolve" + spath = StorePath(store, path) + await zarr.api.asynchronous.create_array( + spath, + shape=(16, 16), + chunks=(16, 16), + dtype=dtype, + fill_value=0, + compressors=BloscCodec(), + ) + buf = await store.get(f"{path}/zarr.json", prototype=default_buffer_prototype()) + assert buf is not None + zarr_json = json.loads(buf.to_bytes()) + blosc_configuration_json = zarr_json["codecs"][1]["configuration"] + assert blosc_configuration_json["typesize"] == typesize + if typesize == 1: + assert blosc_configuration_json["shuffle"] == "bitshuffle" + else: + assert blosc_configuration_json["shuffle"] == "shuffle" + + path2 = "blosc_evolve_sharding" + spath2 = StorePath(store, path2) + await zarr.api.asynchronous.create_array( + spath2, + shape=(16, 16), + chunks=(16, 16), + shards=(16, 16), + dtype=dtype, + fill_value=0, + compressors=BloscCodec(), + ) + buf = await store.get(f"{path2}/zarr.json", prototype=default_buffer_prototype()) + assert buf is not None + zarr_json = json.loads(buf.to_bytes()) + blosc_configuration_json = zarr_json["codecs"][0]["configuration"]["codecs"][1]["configuration"] + assert blosc_configuration_json["typesize"] == typesize + if typesize == 1: + assert blosc_configuration_json["shuffle"] == "bitshuffle" + else: + assert blosc_configuration_json["shuffle"] == "shuffle" + + +async def test_typesize() -> None: + a = np.arange(1000000, dtype=np.uint64) + codecs = [zarr.codecs.BytesCodec(), zarr.codecs.BloscCodec()] + z = zarr.array(a, chunks=(10000), codecs=codecs) + data = await z.store.get("c/0", prototype=default_buffer_prototype()) + assert data is not None + bytes = data.to_bytes() + size = len(bytes) + msg = f"Blosc size mismatch. First 10 bytes: {bytes[:20]!r} and last 10 bytes: {bytes[-20:]!r}" + if Version(numcodecs.__version__) >= Version("0.16.0"): + expected_size = 402 + assert size == expected_size, msg + else: + expected_size = 10216 + assert size == expected_size, msg diff --git a/tests/test_codecs/test_codecs.py b/tests/test_codecs/test_codecs.py new file mode 100644 index 0000000000..468f395254 --- /dev/null +++ b/tests/test_codecs/test_codecs.py @@ -0,0 +1,415 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest + +import zarr +import zarr.api +import zarr.api.asynchronous +from zarr import Array, AsyncArray, config +from zarr.codecs import ( + BytesCodec, + GzipCodec, + ShardingCodec, + TransposeCodec, +) +from zarr.core.buffer import default_buffer_prototype +from zarr.core.indexing import BasicSelection, morton_order_iter +from zarr.core.metadata.v3 import ArrayV3Metadata +from zarr.storage import StorePath + +if TYPE_CHECKING: + from zarr.abc.store import Store + from zarr.core.buffer.core import NDArrayLikeOrScalar + from zarr.core.common import ChunkCoords, MemoryOrder + + +@dataclass(frozen=True) +class _AsyncArrayProxy: + array: AsyncArray[Any] + + def __getitem__(self, selection: BasicSelection) -> _AsyncArraySelectionProxy: + return _AsyncArraySelectionProxy(self.array, selection) + + +@dataclass(frozen=True) +class _AsyncArraySelectionProxy: + array: AsyncArray[Any] + selection: BasicSelection + + async def get(self) -> NDArrayLikeOrScalar: + return await self.array.getitem(self.selection) + + async def set(self, value: np.ndarray[Any, Any]) -> None: + return await self.array.setitem(self.selection, value) + + +def order_from_dim(order: MemoryOrder, ndim: int) -> tuple[int, ...]: + if order == "F": + return tuple(ndim - x - 1 for x in range(ndim)) + else: + return tuple(range(ndim)) + + +def test_sharding_pickle() -> None: + """ + Test that sharding codecs can be pickled + """ + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +@pytest.mark.parametrize("input_order", ["F", "C"]) +@pytest.mark.parametrize("store_order", ["F", "C"]) +@pytest.mark.parametrize("runtime_write_order", ["F", "C"]) +@pytest.mark.parametrize("runtime_read_order", ["F", "C"]) +@pytest.mark.parametrize("with_sharding", [True, False]) +async def test_order( + store: Store, + input_order: MemoryOrder, + store_order: MemoryOrder, + runtime_write_order: MemoryOrder, + runtime_read_order: MemoryOrder, + with_sharding: bool, +) -> None: + data = np.arange(0, 256, dtype="uint16").reshape((32, 8), order=input_order) + path = "order" + spath = StorePath(store, path=path) + + a = await zarr.api.asynchronous.create_array( + spath, + shape=data.shape, + chunks=(16, 8) if with_sharding else (32, 8), + shards=(32, 8) if with_sharding else None, + dtype=data.dtype, + fill_value=0, + chunk_key_encoding={"name": "v2", "separator": "."}, + filters=[TransposeCodec(order=order_from_dim(store_order, data.ndim))], + config={"order": runtime_write_order}, + ) + + await _AsyncArrayProxy(a)[:, :].set(data) + read_data = await _AsyncArrayProxy(a)[:, :].get() + assert np.array_equal(data, read_data) + + with config.set({"array.order": runtime_read_order}): + a = await AsyncArray.open( + spath, + ) + read_data = await _AsyncArrayProxy(a)[:, :].get() + assert np.array_equal(data, read_data) + + assert isinstance(read_data, np.ndarray) + if runtime_read_order == "F": + assert read_data.flags["F_CONTIGUOUS"] + assert not read_data.flags["C_CONTIGUOUS"] + else: + assert not read_data.flags["F_CONTIGUOUS"] + assert read_data.flags["C_CONTIGUOUS"] + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +@pytest.mark.parametrize("input_order", ["F", "C"]) +@pytest.mark.parametrize("runtime_write_order", ["F", "C"]) +@pytest.mark.parametrize("runtime_read_order", ["F", "C"]) +@pytest.mark.parametrize("with_sharding", [True, False]) +def test_order_implicit( + store: Store, + input_order: MemoryOrder, + runtime_write_order: MemoryOrder, + runtime_read_order: MemoryOrder, + with_sharding: bool, +) -> None: + data = np.arange(0, 256, dtype="uint16").reshape((16, 16), order=input_order) + path = "order_implicit" + spath = StorePath(store, path) + + with config.set({"array.order": runtime_write_order}): + a = zarr.create_array( + spath, + shape=data.shape, + chunks=(8, 8) if with_sharding else (16, 16), + shards=(16, 16) if with_sharding else None, + dtype=data.dtype, + fill_value=0, + ) + + a[:, :] = data + + with config.set({"array.order": runtime_read_order}): + a = Array.open(spath) + read_data = a[:, :] + assert np.array_equal(data, read_data) + + assert isinstance(read_data, np.ndarray) + if runtime_read_order == "F": + assert read_data.flags["F_CONTIGUOUS"] + assert not read_data.flags["C_CONTIGUOUS"] + else: + assert not read_data.flags["F_CONTIGUOUS"] + assert read_data.flags["C_CONTIGUOUS"] + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_open(store: Store) -> None: + spath = StorePath(store) + a = zarr.create_array( + spath, + shape=(16, 16), + chunks=(16, 16), + dtype="int32", + fill_value=0, + ) + b = Array.open(spath) + assert a.metadata == b.metadata + + +def test_morton() -> None: + assert list(morton_order_iter((2, 2))) == [(0, 0), (1, 0), (0, 1), (1, 1)] + assert list(morton_order_iter((2, 2, 2))) == [ + (0, 0, 0), + (1, 0, 0), + (0, 1, 0), + (1, 1, 0), + (0, 0, 1), + (1, 0, 1), + (0, 1, 1), + (1, 1, 1), + ] + assert list(morton_order_iter((2, 2, 2, 2))) == [ + (0, 0, 0, 0), + (1, 0, 0, 0), + (0, 1, 0, 0), + (1, 1, 0, 0), + (0, 0, 1, 0), + (1, 0, 1, 0), + (0, 1, 1, 0), + (1, 1, 1, 0), + (0, 0, 0, 1), + (1, 0, 0, 1), + (0, 1, 0, 1), + (1, 1, 0, 1), + (0, 0, 1, 1), + (1, 0, 1, 1), + (0, 1, 1, 1), + (1, 1, 1, 1), + ] + + +@pytest.mark.parametrize( + "shape", + [ + [2, 2, 2], + [5, 2], + [2, 5], + [2, 9, 2], + [3, 2, 12], + [2, 5, 1], + [4, 3, 6, 2, 7], + [3, 2, 1, 6, 4, 5, 2], + ], +) +def test_morton2(shape: ChunkCoords) -> None: + order = list(morton_order_iter(shape)) + for i, x in enumerate(order): + assert x not in order[:i] # no duplicates + assert all(x[j] < shape[j] for j in range(len(shape))) # all indices are within bounds + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_write_partial_chunks(store: Store) -> None: + data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) + spath = StorePath(store) + a = zarr.create_array( + spath, + shape=data.shape, + chunks=(20, 20), + dtype=data.dtype, + fill_value=1, + ) + a[0:16, 0:16] = data + assert np.array_equal(a[0:16, 0:16], data) + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +async def test_delete_empty_chunks(store: Store) -> None: + data = np.ones((16, 16)) + path = "delete_empty_chunks" + spath = StorePath(store, path) + a = await zarr.api.asynchronous.create_array( + spath, + shape=data.shape, + chunks=(32, 32), + dtype=data.dtype, + fill_value=1, + ) + await _AsyncArrayProxy(a)[:16, :16].set(np.zeros((16, 16))) + await _AsyncArrayProxy(a)[:16, :16].set(data) + assert np.array_equal(await _AsyncArrayProxy(a)[:16, :16].get(), data) + assert await store.get(f"{path}/c0/0", prototype=default_buffer_prototype()) is None + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +async def test_dimension_names(store: Store) -> None: + data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) + path = "dimension_names" + spath = StorePath(store, path) + await zarr.api.asynchronous.create_array( + spath, + shape=data.shape, + chunks=(16, 16), + dtype=data.dtype, + fill_value=0, + dimension_names=("x", "y"), + ) + + assert isinstance( + meta := (await zarr.api.asynchronous.open_array(store=spath)).metadata, ArrayV3Metadata + ) + assert meta.dimension_names == ( + "x", + "y", + ) + path2 = "dimension_names2" + spath2 = StorePath(store, path2) + await zarr.api.asynchronous.create_array( + spath2, + shape=data.shape, + chunks=(16, 16), + dtype=data.dtype, + fill_value=0, + ) + + assert isinstance(meta := (await AsyncArray.open(spath2)).metadata, ArrayV3Metadata) + assert meta.dimension_names is None + zarr_json_buffer = await store.get(f"{path2}/zarr.json", prototype=default_buffer_prototype()) + assert zarr_json_buffer is not None + assert "dimension_names" not in json.loads(zarr_json_buffer.to_bytes()) + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_invalid_metadata(store: Store) -> None: + spath2 = StorePath(store, "invalid_codec_order") + with pytest.raises(TypeError): + Array.create( + spath2, + shape=(16, 16), + chunk_shape=(16, 16), + dtype=np.dtype("uint8"), + fill_value=0, + codecs=[ + BytesCodec(), + TransposeCodec(order=order_from_dim("F", 2)), + ], + ) + spath3 = StorePath(store, "invalid_order") + with pytest.raises(TypeError): + Array.create( + spath3, + shape=(16, 16), + chunk_shape=(16, 16), + dtype=np.dtype("uint8"), + fill_value=0, + codecs=[ + TransposeCodec(order="F"), # type: ignore[arg-type] + BytesCodec(), + ], + ) + spath4 = StorePath(store, "invalid_missing_bytes_codec") + with pytest.raises(ValueError): + Array.create( + spath4, + shape=(16, 16), + chunk_shape=(16, 16), + dtype=np.dtype("uint8"), + fill_value=0, + codecs=[ + TransposeCodec(order=order_from_dim("F", 2)), + ], + ) + spath5 = StorePath(store, "invalid_inner_chunk_shape") + with pytest.raises(ValueError): + Array.create( + spath5, + shape=(16, 16), + chunk_shape=(16, 16), + dtype=np.dtype("uint8"), + fill_value=0, + codecs=[ + ShardingCodec(chunk_shape=(8,)), + ], + ) + spath6 = StorePath(store, "invalid_inner_chunk_shape") + with pytest.raises(ValueError): + Array.create( + spath6, + shape=(16, 16), + chunk_shape=(16, 16), + dtype=np.dtype("uint8"), + fill_value=0, + codecs=[ + ShardingCodec(chunk_shape=(8, 7)), + ], + ) + spath7 = StorePath(store, "warning_inefficient_codecs") + with pytest.warns(UserWarning): + Array.create( + spath7, + shape=(16, 16), + chunk_shape=(16, 16), + dtype=np.dtype("uint8"), + fill_value=0, + codecs=[ + ShardingCodec(chunk_shape=(8, 8)), + GzipCodec(), + ], + ) + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_invalid_metadata_create_array(store: Store) -> None: + spath = StorePath(store, "warning_inefficient_codecs") + with pytest.warns(UserWarning): + zarr.create_array( + spath, + shape=(16, 16), + chunks=(16, 16), + dtype=np.dtype("uint8"), + fill_value=0, + serializer=ShardingCodec(chunk_shape=(8, 8)), + compressors=[ + GzipCodec(), + ], + ) + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +async def test_resize(store: Store) -> None: + data = np.zeros((16, 18), dtype="uint16") + path = "resize" + spath = StorePath(store, path) + a = await zarr.api.asynchronous.create_array( + spath, + shape=data.shape, + chunks=(10, 10), + dtype=data.dtype, + chunk_key_encoding={"name": "v2", "separator": "."}, + fill_value=1, + ) + + await _AsyncArrayProxy(a)[:16, :18].set(data) + assert await store.get(f"{path}/1.1", prototype=default_buffer_prototype()) is not None + assert await store.get(f"{path}/0.0", prototype=default_buffer_prototype()) is not None + assert await store.get(f"{path}/0.1", prototype=default_buffer_prototype()) is not None + assert await store.get(f"{path}/1.0", prototype=default_buffer_prototype()) is not None + + await a.resize((10, 12)) + assert a.metadata.shape == (10, 12) + assert a.shape == (10, 12) + assert await store.get(f"{path}/0.0", prototype=default_buffer_prototype()) is not None + assert await store.get(f"{path}/0.1", prototype=default_buffer_prototype()) is not None + assert await store.get(f"{path}/1.0", prototype=default_buffer_prototype()) is None + assert await store.get(f"{path}/1.1", prototype=default_buffer_prototype()) is None diff --git a/tests/test_codecs/test_endian.py b/tests/test_codecs/test_endian.py new file mode 100644 index 0000000000..ab64afb1b8 --- /dev/null +++ b/tests/test_codecs/test_endian.py @@ -0,0 +1,60 @@ +from typing import Literal + +import numpy as np +import pytest + +import zarr +from zarr.abc.store import Store +from zarr.codecs import BytesCodec +from zarr.storage import StorePath + +from .test_codecs import _AsyncArrayProxy + + +@pytest.mark.filterwarnings("ignore:The endianness of the requested serializer") +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +@pytest.mark.parametrize("endian", ["big", "little"]) +async def test_endian(store: Store, endian: Literal["big", "little"]) -> None: + data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) + path = "endian" + spath = StorePath(store, path) + a = await zarr.api.asynchronous.create_array( + spath, + shape=data.shape, + chunks=(16, 16), + dtype=data.dtype, + fill_value=0, + chunk_key_encoding={"name": "v2", "separator": "."}, + serializer=BytesCodec(endian=endian), + ) + + await _AsyncArrayProxy(a)[:, :].set(data) + readback_data = await _AsyncArrayProxy(a)[:, :].get() + assert np.array_equal(data, readback_data) + + +@pytest.mark.filterwarnings("ignore:The endianness of the requested serializer") +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +@pytest.mark.parametrize("dtype_input_endian", [">u2", "u2", " None: + data = np.arange(0, 256, dtype=dtype_input_endian).reshape((16, 16)) + path = "endian" + spath = StorePath(store, path) + a = await zarr.api.asynchronous.create_array( + spath, + shape=data.shape, + chunks=(16, 16), + dtype="uint16", + fill_value=0, + chunk_key_encoding={"name": "v2", "separator": "."}, + serializer=BytesCodec(endian=dtype_store_endian), + ) + + await _AsyncArrayProxy(a)[:, :].set(data) + readback_data = await _AsyncArrayProxy(a)[:, :].get() + assert np.array_equal(data, readback_data) diff --git a/tests/test_codecs/test_gzip.py b/tests/test_codecs/test_gzip.py new file mode 100644 index 0000000000..4753036c87 --- /dev/null +++ b/tests/test_codecs/test_gzip.py @@ -0,0 +1,24 @@ +import numpy as np +import pytest + +import zarr +from zarr.abc.store import Store +from zarr.codecs import GzipCodec +from zarr.storage import StorePath + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_gzip(store: Store) -> None: + data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) + + a = zarr.create_array( + StorePath(store), + shape=data.shape, + chunks=(16, 16), + dtype=data.dtype, + fill_value=0, + compressors=GzipCodec(), + ) + + a[:, :] = data + assert np.array_equal(data, a[:, :]) diff --git a/tests/test_codecs/test_sharding.py b/tests/test_codecs/test_sharding.py new file mode 100644 index 0000000000..403fd80e81 --- /dev/null +++ b/tests/test_codecs/test_sharding.py @@ -0,0 +1,483 @@ +import pickle +from typing import Any + +import numpy as np +import numpy.typing as npt +import pytest + +import zarr +import zarr.api +import zarr.api.asynchronous +from zarr import Array +from zarr.abc.store import Store +from zarr.codecs import ( + BloscCodec, + ShardingCodec, + ShardingCodecIndexLocation, + TransposeCodec, +) +from zarr.core.buffer import NDArrayLike, default_buffer_prototype +from zarr.storage import StorePath + +from ..conftest import ArrayRequest +from .test_codecs import _AsyncArrayProxy, order_from_dim + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize("index_location", ["start", "end"]) +@pytest.mark.parametrize( + "array_fixture", + [ + ArrayRequest(shape=(128,) * 1, dtype="uint8", order="C"), + ArrayRequest(shape=(128,) * 2, dtype="uint8", order="C"), + ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), + ], + indirect=["array_fixture"], +) +@pytest.mark.parametrize("offset", [0, 10]) +def test_sharding( + store: Store, + array_fixture: npt.NDArray[Any], + index_location: ShardingCodecIndexLocation, + offset: int, +) -> None: + """ + Test that we can create an array with a sharding codec, write data to that array, and get + the same data out via indexing. + """ + data = array_fixture + spath = StorePath(store) + + arr = zarr.create_array( + spath, + shape=tuple(s + offset for s in data.shape), + chunks=(32,) * data.ndim, + shards={"shape": (64,) * data.ndim, "index_location": index_location}, + dtype=data.dtype, + fill_value=6, + filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], + compressors=BloscCodec(cname="lz4"), + ) + write_region = tuple(slice(offset, None) for dim in range(data.ndim)) + arr[write_region] = data + + if offset > 0: + empty_region = tuple(slice(0, offset) for dim in range(data.ndim)) + assert np.all(arr[empty_region] == arr.metadata.fill_value) + + read_data = arr[write_region] + assert isinstance(read_data, NDArrayLike) + assert data.shape == read_data.shape + assert np.array_equal(data, read_data) + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize("index_location", ["start", "end"]) +@pytest.mark.parametrize("offset", [0, 10]) +def test_sharding_scalar( + store: Store, + index_location: ShardingCodecIndexLocation, + offset: int, +) -> None: + """ + Test that we can create an array with a sharding codec, write data to that array, and get + the same data out via indexing. + """ + spath = StorePath(store) + + arr = zarr.create_array( + spath, + shape=(128, 128), + chunks=(32, 32), + shards={"shape": (64, 64), "index_location": index_location}, + dtype="uint8", + fill_value=6, + filters=[TransposeCodec(order=order_from_dim("F", 2))], + compressors=BloscCodec(cname="lz4"), + ) + arr[:16, :16] = 10 # intentionally write partial chunks + read_data = arr[:16, :16] + np.testing.assert_array_equal(read_data, 10) + + +@pytest.mark.parametrize("index_location", ["start", "end"]) +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize( + "array_fixture", + [ + ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), + ], + indirect=["array_fixture"], +) +def test_sharding_partial( + store: Store, array_fixture: npt.NDArray[Any], index_location: ShardingCodecIndexLocation +) -> None: + data = array_fixture + spath = StorePath(store) + a = zarr.create_array( + spath, + shape=tuple(a + 10 for a in data.shape), + chunks=(32, 32, 32), + shards={"shape": (64, 64, 64), "index_location": index_location}, + compressors=BloscCodec(cname="lz4"), + filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], + dtype=data.dtype, + fill_value=0, + ) + + a[10:, 10:, 10:] = data + + read_data = a[0:10, 0:10, 0:10] + assert np.all(read_data == 0) + + read_data = a[10:, 10:, 10:] + assert isinstance(read_data, NDArrayLike) + assert data.shape == read_data.shape + assert np.array_equal(data, read_data) + + +@pytest.mark.parametrize("index_location", ["start", "end"]) +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize( + "array_fixture", + [ + ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), + ], + indirect=["array_fixture"], +) +def test_sharding_partial_readwrite( + store: Store, array_fixture: npt.NDArray[Any], index_location: ShardingCodecIndexLocation +) -> None: + data = array_fixture + spath = StorePath(store) + a = zarr.create_array( + spath, + shape=data.shape, + chunks=(1, data.shape[1], data.shape[2]), + shards={"shape": data.shape, "index_location": index_location}, + dtype=data.dtype, + fill_value=0, + filters=None, + compressors=None, + ) + + a[:] = data + + for x in range(data.shape[0]): + read_data = a[x, :, :] + assert np.array_equal(data[x], read_data) + + +@pytest.mark.parametrize( + "array_fixture", + [ + ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), + ], + indirect=["array_fixture"], +) +@pytest.mark.parametrize("index_location", ["start", "end"]) +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +def test_sharding_partial_read( + store: Store, array_fixture: npt.NDArray[Any], index_location: ShardingCodecIndexLocation +) -> None: + data = array_fixture + spath = StorePath(store) + a = zarr.create_array( + spath, + shape=tuple(a + 10 for a in data.shape), + chunks=(32, 32, 32), + shards={"shape": (64, 64, 64), "index_location": index_location}, + compressors=BloscCodec(cname="lz4"), + filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], + dtype=data.dtype, + fill_value=1, + ) + + read_data = a[0:10, 0:10, 0:10] + assert np.all(read_data == 1) + + +@pytest.mark.parametrize( + "array_fixture", + [ + ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), + ], + indirect=["array_fixture"], +) +@pytest.mark.parametrize("index_location", ["start", "end"]) +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +def test_sharding_partial_overwrite( + store: Store, array_fixture: npt.NDArray[Any], index_location: ShardingCodecIndexLocation +) -> None: + data = array_fixture[:10, :10, :10] + spath = StorePath(store) + a = zarr.create_array( + spath, + shape=tuple(a + 10 for a in data.shape), + chunks=(32, 32, 32), + shards={"shape": (64, 64, 64), "index_location": index_location}, + compressors=BloscCodec(cname="lz4"), + filters=[TransposeCodec(order=order_from_dim("F", data.ndim))], + dtype=data.dtype, + fill_value=1, + ) + + a[:10, :10, :10] = data + + read_data = a[0:10, 0:10, 0:10] + assert np.array_equal(data, read_data) + + data += 10 + a[:10, :10, :10] = data + read_data = a[0:10, 0:10, 0:10] + assert np.array_equal(data, read_data) + + +@pytest.mark.parametrize( + "array_fixture", + [ + ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), + ], + indirect=["array_fixture"], +) +@pytest.mark.parametrize( + "outer_index_location", + ["start", "end"], +) +@pytest.mark.parametrize( + "inner_index_location", + ["start", "end"], +) +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +def test_nested_sharding( + store: Store, + array_fixture: npt.NDArray[Any], + outer_index_location: ShardingCodecIndexLocation, + inner_index_location: ShardingCodecIndexLocation, +) -> None: + data = array_fixture + spath = StorePath(store) + a = Array.create( + spath, + shape=data.shape, + chunk_shape=(64, 64, 64), + dtype=data.dtype, + fill_value=0, + codecs=[ + ShardingCodec( + chunk_shape=(32, 32, 32), + codecs=[ + ShardingCodec(chunk_shape=(16, 16, 16), index_location=inner_index_location) + ], + index_location=outer_index_location, + ) + ], + ) + + a[:, :, :] = data + + read_data = a[0 : data.shape[0], 0 : data.shape[1], 0 : data.shape[2]] + assert isinstance(read_data, NDArrayLike) + assert data.shape == read_data.shape + assert np.array_equal(data, read_data) + + +@pytest.mark.parametrize( + "array_fixture", + [ + ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), + ], + indirect=["array_fixture"], +) +@pytest.mark.parametrize( + "outer_index_location", + ["start", "end"], +) +@pytest.mark.parametrize( + "inner_index_location", + ["start", "end"], +) +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +def test_nested_sharding_create_array( + store: Store, + array_fixture: npt.NDArray[Any], + outer_index_location: ShardingCodecIndexLocation, + inner_index_location: ShardingCodecIndexLocation, +) -> None: + data = array_fixture + spath = StorePath(store) + a = zarr.create_array( + spath, + shape=data.shape, + chunks=(32, 32, 32), + dtype=data.dtype, + fill_value=0, + serializer=ShardingCodec( + chunk_shape=(32, 32, 32), + codecs=[ShardingCodec(chunk_shape=(16, 16, 16), index_location=inner_index_location)], + index_location=outer_index_location, + ), + filters=None, + compressors=None, + ) + print(a.metadata.to_dict()) + + a[:, :, :] = data + + read_data = a[0 : data.shape[0], 0 : data.shape[1], 0 : data.shape[2]] + assert isinstance(read_data, NDArrayLike) + assert data.shape == read_data.shape + assert np.array_equal(data, read_data) + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +def test_open_sharding(store: Store) -> None: + path = "open_sharding" + spath = StorePath(store, path) + a = zarr.create_array( + spath, + shape=(16, 16), + chunks=(8, 8), + shards=(16, 16), + filters=[TransposeCodec(order=order_from_dim("F", 2))], + compressors=BloscCodec(), + dtype="int32", + fill_value=0, + ) + b = Array.open(spath) + assert a.metadata == b.metadata + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +def test_write_partial_sharded_chunks(store: Store) -> None: + data = np.arange(0, 16 * 16, dtype="uint16").reshape((16, 16)) + spath = StorePath(store) + a = zarr.create_array( + spath, + shape=(40, 40), + chunks=(10, 10), + shards=(20, 20), + dtype=data.dtype, + compressors=BloscCodec(), + fill_value=1, + ) + a[0:16, 0:16] = data + assert np.array_equal(a[0:16, 0:16], data) + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +async def test_delete_empty_shards(store: Store) -> None: + if not store.supports_deletes: + pytest.skip("store does not support deletes") + path = "delete_empty_shards" + spath = StorePath(store, path) + a = await zarr.api.asynchronous.create_array( + spath, + shape=(16, 16), + chunks=(8, 8), + shards=(8, 16), + dtype="uint16", + compressors=None, + fill_value=1, + ) + print(a.metadata.to_dict()) + await _AsyncArrayProxy(a)[:, :].set(np.zeros((16, 16))) + await _AsyncArrayProxy(a)[8:, :].set(np.ones((8, 16))) + await _AsyncArrayProxy(a)[:, 8:].set(np.ones((16, 8))) + # chunk (0, 0) is full + # chunks (0, 1), (1, 0), (1, 1) are empty + # shard (0, 0) is half-full + # shard (1, 0) is empty + + data = np.ones((16, 16), dtype="uint16") + data[:8, :8] = 0 + assert np.array_equal(data, await _AsyncArrayProxy(a)[:, :].get()) + assert await store.get(f"{path}/c/1/0", prototype=default_buffer_prototype()) is None + chunk_bytes = await store.get(f"{path}/c/0/0", prototype=default_buffer_prototype()) + assert chunk_bytes is not None + assert len(chunk_bytes) == 16 * 2 + 8 * 8 * 2 + 4 + + +def test_pickle() -> None: + codec = ShardingCodec(chunk_shape=(8, 8)) + assert pickle.loads(pickle.dumps(codec)) == codec + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +@pytest.mark.parametrize( + "index_location", [ShardingCodecIndexLocation.start, ShardingCodecIndexLocation.end] +) +async def test_sharding_with_empty_inner_chunk( + store: Store, index_location: ShardingCodecIndexLocation +) -> None: + data = np.arange(0, 16 * 16, dtype="uint32").reshape((16, 16)) + fill_value = 1 + + path = f"sharding_with_empty_inner_chunk_{index_location}" + spath = StorePath(store, path) + a = await zarr.api.asynchronous.create_array( + spath, + shape=(16, 16), + chunks=(4, 4), + shards={"shape": (8, 8), "index_location": index_location}, + dtype="uint32", + fill_value=fill_value, + ) + data[:4, :4] = fill_value + await a.setitem(..., data) + print("read data") + data_read = await a.getitem(...) + assert np.array_equal(data_read, data) + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +@pytest.mark.parametrize( + "index_location", + [ShardingCodecIndexLocation.start, ShardingCodecIndexLocation.end], +) +@pytest.mark.parametrize("chunks_per_shard", [(5, 2), (2, 5), (5, 5)]) +async def test_sharding_with_chunks_per_shard( + store: Store, index_location: ShardingCodecIndexLocation, chunks_per_shard: tuple[int] +) -> None: + chunk_shape = (2, 1) + shape = tuple(x * y for x, y in zip(chunks_per_shard, chunk_shape, strict=False)) + data = np.ones(np.prod(shape), dtype="int32").reshape(shape) + fill_value = 42 + + path = f"test_sharding_with_chunks_per_shard_{index_location}" + spath = StorePath(store, path) + a = zarr.create_array( + spath, + shape=shape, + chunks=chunk_shape, + shards={"shape": shape, "index_location": index_location}, + dtype="int32", + fill_value=fill_value, + ) + a[...] = data + data_read = a[...] + assert np.array_equal(data_read, data) + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_invalid_metadata(store: Store) -> None: + spath1 = StorePath(store, "invalid_inner_chunk_shape") + with pytest.raises(ValueError): + zarr.create_array( + spath1, + shape=(16, 16), + shards=(16, 16), + chunks=(8,), + dtype=np.dtype("uint8"), + fill_value=0, + ) + spath2 = StorePath(store, "invalid_inner_chunk_shape") + with pytest.raises(ValueError): + zarr.create_array( + spath2, + shape=(16, 16), + shards=(16, 16), + chunks=(8, 7), + dtype=np.dtype("uint8"), + fill_value=0, + ) diff --git a/tests/test_codecs/test_transpose.py b/tests/test_codecs/test_transpose.py new file mode 100644 index 0000000000..06ec668ad3 --- /dev/null +++ b/tests/test_codecs/test_transpose.py @@ -0,0 +1,95 @@ +import numpy as np +import pytest + +import zarr +from zarr import AsyncArray, config +from zarr.abc.store import Store +from zarr.codecs import TransposeCodec +from zarr.core.common import MemoryOrder +from zarr.storage import StorePath + +from .test_codecs import _AsyncArrayProxy + + +@pytest.mark.parametrize("input_order", ["F", "C"]) +@pytest.mark.parametrize("runtime_write_order", ["F", "C"]) +@pytest.mark.parametrize("runtime_read_order", ["F", "C"]) +@pytest.mark.parametrize("with_sharding", [True, False]) +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +async def test_transpose( + store: Store, + input_order: MemoryOrder, + runtime_write_order: MemoryOrder, + runtime_read_order: MemoryOrder, + with_sharding: bool, +) -> None: + data = np.arange(0, 256, dtype="uint16").reshape((1, 32, 8), order=input_order) + spath = StorePath(store, path="transpose") + with config.set({"array.order": runtime_write_order}): + a = await zarr.api.asynchronous.create_array( + spath, + shape=data.shape, + chunks=(1, 16, 8) if with_sharding else (1, 32, 8), + shards=(1, 32, 8) if with_sharding else None, + dtype=data.dtype, + fill_value=0, + chunk_key_encoding={"name": "v2", "separator": "."}, + filters=[TransposeCodec(order=(2, 1, 0))], + ) + + await _AsyncArrayProxy(a)[:, :].set(data) + read_data = await _AsyncArrayProxy(a)[:, :].get() + assert np.array_equal(data, read_data) + + with config.set({"array.order": runtime_read_order}): + a = await AsyncArray.open( + spath, + ) + read_data = await _AsyncArrayProxy(a)[:, :].get() + assert np.array_equal(data, read_data) + + assert isinstance(read_data, np.ndarray) + if runtime_read_order == "F": + assert read_data.flags["F_CONTIGUOUS"] + assert not read_data.flags["C_CONTIGUOUS"] + else: + assert not read_data.flags["F_CONTIGUOUS"] + assert read_data.flags["C_CONTIGUOUS"] + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +@pytest.mark.parametrize("order", [[1, 2, 0], [1, 2, 3, 0], [3, 2, 4, 0, 1]]) +def test_transpose_non_self_inverse(store: Store, order: list[int]) -> None: + shape = [i + 3 for i in range(len(order))] + data = np.arange(0, np.prod(shape), dtype="uint16").reshape(shape) + spath = StorePath(store, "transpose_non_self_inverse") + a = zarr.create_array( + spath, + shape=data.shape, + chunks=data.shape, + dtype=data.dtype, + fill_value=0, + filters=[TransposeCodec(order=order)], + ) + a[:, :] = data + read_data = a[:, :] + assert np.array_equal(data, read_data) + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_transpose_invalid( + store: Store, +) -> None: + data = np.arange(0, 256, dtype="uint16").reshape((1, 32, 8)) + spath = StorePath(store, "transpose_invalid") + for order in [(1, 0), (3, 2, 1), (3, 3, 1), "F", "C"]: + with pytest.raises((ValueError, TypeError)): + zarr.create_array( + spath, + shape=data.shape, + chunks=(1, 32, 8), + dtype=data.dtype, + fill_value=0, + chunk_key_encoding={"name": "v2", "separator": "."}, + filters=[TransposeCodec(order=order)], # type: ignore[arg-type] + ) diff --git a/tests/test_codecs/test_vlen.py b/tests/test_codecs/test_vlen.py new file mode 100644 index 0000000000..6fe1863464 --- /dev/null +++ b/tests/test_codecs/test_vlen.py @@ -0,0 +1,64 @@ +from typing import Any + +import numpy as np +import pytest + +import zarr +from zarr import Array +from zarr.abc.codec import Codec +from zarr.abc.store import Store +from zarr.codecs import ZstdCodec +from zarr.core.dtype import get_data_type_from_native_dtype +from zarr.core.dtype.npy.string import _NUMPY_SUPPORTS_VLEN_STRING +from zarr.core.metadata.v3 import ArrayV3Metadata +from zarr.storage import StorePath + +numpy_str_dtypes: list[type | str | None] = [None, str, "str", np.dtypes.StrDType, "S", "U"] +expected_array_string_dtype: np.dtype[Any] +if _NUMPY_SUPPORTS_VLEN_STRING: + numpy_str_dtypes.append(np.dtypes.StringDType) + expected_array_string_dtype = np.dtypes.StringDType() +else: + expected_array_string_dtype = np.dtype("O") + + +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +@pytest.mark.parametrize("store", ["memory", "local"], indirect=["store"]) +@pytest.mark.parametrize("dtype", numpy_str_dtypes) +@pytest.mark.parametrize("as_object_array", [False, True]) +@pytest.mark.parametrize("compressor", [None, ZstdCodec()]) +def test_vlen_string( + store: Store, dtype: np.dtype[Any] | None, as_object_array: bool, compressor: Codec | None +) -> None: + strings = ["hello", "world", "this", "is", "a", "test"] + data = np.array(strings, dtype=dtype).reshape((2, 3)) + + sp = StorePath(store, path="string") + a = zarr.create_array( + sp, + shape=data.shape, + chunks=data.shape, + dtype=data.dtype, + fill_value="", + compressors=compressor, + ) + assert isinstance(a.metadata, ArrayV3Metadata) # needed for mypy + + # should also work if input array is an object array, provided we explicitly specified + # a stringlike dtype when creating the Array + if as_object_array: + data_obj = data.astype("O") + + a[:, :] = data_obj + else: + a[:, :] = data + assert np.array_equal(data, a[:, :]) + assert a.metadata.data_type == get_data_type_from_native_dtype(data.dtype) + assert a.dtype == data.dtype + + # test round trip + b = Array.open(sp) + assert isinstance(b.metadata, ArrayV3Metadata) # needed for mypy + assert np.array_equal(data, b[:, :]) + assert b.metadata.data_type == get_data_type_from_native_dtype(data.dtype) + assert a.dtype == data.dtype diff --git a/tests/test_codecs/test_zstd.py b/tests/test_codecs/test_zstd.py new file mode 100644 index 0000000000..6068f53443 --- /dev/null +++ b/tests/test_codecs/test_zstd.py @@ -0,0 +1,25 @@ +import numpy as np +import pytest + +import zarr +from zarr.abc.store import Store +from zarr.codecs import ZstdCodec +from zarr.storage import StorePath + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +@pytest.mark.parametrize("checksum", [True, False]) +def test_zstd(store: Store, checksum: bool) -> None: + data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) + + a = zarr.create_array( + StorePath(store, path="zstd"), + shape=data.shape, + chunks=(16, 16), + dtype=data.dtype, + fill_value=0, + compressors=ZstdCodec(level=0, checksum=checksum), + ) + + a[:, :] = data + assert np.array_equal(data, a[:, :]) diff --git a/tests/test_common.py b/tests/test_common.py new file mode 100644 index 0000000000..c28723d1a8 --- /dev/null +++ b/tests/test_common.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + from typing import Any, Literal + +import numpy as np +import pytest + +from zarr.core.common import parse_name, parse_shapelike, product +from zarr.core.config import parse_indexing_order + + +@pytest.mark.parametrize("data", [(0, 0, 0, 0), (1, 3, 4, 5, 6), (2, 4)]) +def test_product(data: tuple[int, ...]) -> None: + assert product(data) == np.prod(data) + + +# todo: test +def test_concurrent_map() -> None: ... + + +# todo: test +def test_to_thread() -> None: ... + + +# todo: test +def test_enum_names() -> None: ... + + +# todo: test +def test_parse_enum() -> None: ... + + +@pytest.mark.parametrize("data", [("foo", "bar"), (10, 11)]) +def test_parse_name_invalid(data: tuple[Any, Any]) -> None: + observed, expected = data + if isinstance(observed, str): + with pytest.raises(ValueError, match=f"Expected '{expected}'. Got {observed} instead."): + parse_name(observed, expected) + else: + with pytest.raises( + TypeError, match=f"Expected a string, got an instance of {type(observed)}." + ): + parse_name(observed, expected) + + +@pytest.mark.parametrize("data", [("foo", "foo"), ("10", "10")]) +def test_parse_name_valid(data: tuple[Any, Any]) -> None: + observed, expected = data + assert parse_name(observed, expected) == observed + + +@pytest.mark.parametrize("data", [0, 1, "hello", "f"]) +def test_parse_indexing_order_invalid(data: Any) -> None: + with pytest.raises(ValueError, match="Expected one of"): + parse_indexing_order(data) + + +@pytest.mark.parametrize("data", ["C", "F"]) +def parse_indexing_order_valid(data: Literal["C", "F"]) -> None: + assert parse_indexing_order(data) == data + + +@pytest.mark.parametrize("data", [lambda v: v, slice(None)]) +def test_parse_shapelike_invalid_single_type(data: Any) -> None: + """ + Test that we get the expected error message when passing in a value that is not an integer + or an iterable of integers. + """ + with pytest.raises(TypeError, match="Expected an integer or an iterable of integers."): + parse_shapelike(data) + + +def test_parse_shapelike_invalid_single_value() -> None: + """ + Test that we get the expected error message when passing in a negative integer. + """ + with pytest.raises(ValueError, match="Expected a non-negative integer."): + parse_shapelike(-1) + + +@pytest.mark.parametrize("data", ["shape", ("0", 1, 2, 3), {"0": "0"}, ((1, 2), (2, 2)), (4.0, 2)]) +def test_parse_shapelike_invalid_iterable_types(data: Any) -> None: + """ + Test that we get the expected error message when passing in an iterable containing + non-integer elements + """ + with pytest.raises(TypeError, match="Expected an iterable of integers"): + parse_shapelike(data) + + +@pytest.mark.parametrize("data", [(1, 2, 3, -1), (-10,)]) +def test_parse_shapelike_invalid_iterable_values(data: Any) -> None: + """ + Test that we get the expected error message when passing in an iterable containing negative + integers + """ + with pytest.raises(ValueError, match="Expected all values to be non-negative."): + parse_shapelike(data) + + +@pytest.mark.parametrize("data", [range(10), [0, 1, 2, 3], (3, 4, 5), ()]) +def test_parse_shapelike_valid(data: Iterable[int]) -> None: + assert parse_shapelike(data) == tuple(data) + + +# todo: more dtypes +@pytest.mark.parametrize("data", [("uint8", np.uint8), ("float64", np.float64)]) +def parse_dtype(data: tuple[str, np.dtype[Any]]) -> None: + unparsed, parsed = data + assert parse_dtype(unparsed) == parsed + + +# todo: figure out what it means to test this +def test_parse_fill_value() -> None: ... diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000000..ed778a02ae --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,351 @@ +import os +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any +from unittest import mock +from unittest.mock import Mock + +import numpy as np +import pytest + +import zarr +import zarr.api +from zarr import zeros +from zarr.abc.codec import CodecPipeline +from zarr.abc.store import ByteSetter, Store +from zarr.codecs import ( + BloscCodec, + BytesCodec, + Crc32cCodec, + GzipCodec, + ShardingCodec, +) +from zarr.core.array import create_array +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import NDBuffer +from zarr.core.buffer.core import Buffer +from zarr.core.codec_pipeline import BatchedCodecPipeline +from zarr.core.config import BadConfigError, config +from zarr.core.dtype import Int8, VariableLengthUTF8 +from zarr.core.indexing import SelectorTuple +from zarr.registry import ( + fully_qualified_name, + get_buffer_class, + get_codec_class, + get_ndbuffer_class, + get_pipeline_class, + register_buffer, + register_codec, + register_ndbuffer, + register_pipeline, +) +from zarr.storage import MemoryStore +from zarr.testing.buffer import ( + NDBufferUsingTestNDArrayLike, + StoreExpectingTestBuffer, + TestBuffer, + TestNDArrayLike, +) + +if TYPE_CHECKING: + from zarr.core.dtype.wrapper import ZDType + + +def test_config_defaults_set() -> None: + # regression test for available defaults + assert ( + config.defaults + == [ + { + "default_zarr_format": 3, + "array": { + "order": "C", + "write_empty_chunks": False, + "v2_default_compressor": { + "default": {"id": "zstd", "level": 0, "checksum": False}, + "variable-length-string": {"id": "zstd", "level": 0, "checksum": False}, + }, + "v2_default_filters": { + "default": None, + "variable-length-string": [{"id": "vlen-utf8"}], + }, + "v3_default_filters": {"default": [], "variable-length-string": []}, + "v3_default_serializer": { + "default": {"name": "bytes", "configuration": {"endian": "little"}}, + "variable-length-string": {"name": "vlen-utf8"}, + }, + "v3_default_compressors": { + "default": [ + {"name": "zstd", "configuration": {"level": 0, "checksum": False}}, + ], + "variable-length-string": [ + {"name": "zstd", "configuration": {"level": 0, "checksum": False}} + ], + }, + }, + "async": {"concurrency": 10, "timeout": None}, + "threading": {"max_workers": None}, + "json_indent": 2, + "codec_pipeline": { + "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", + "batch_size": 1, + }, + "codecs": { + "blosc": "zarr.codecs.blosc.BloscCodec", + "gzip": "zarr.codecs.gzip.GzipCodec", + "zstd": "zarr.codecs.zstd.ZstdCodec", + "bytes": "zarr.codecs.bytes.BytesCodec", + "endian": "zarr.codecs.bytes.BytesCodec", # compatibility with earlier versions of ZEP1 + "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", + "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", + "transpose": "zarr.codecs.transpose.TransposeCodec", + "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", + "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", + }, + "buffer": "zarr.buffer.cpu.Buffer", + "ndbuffer": "zarr.buffer.cpu.NDBuffer", + } + ] + ) + assert config.get("array.order") == "C" + assert config.get("async.concurrency") == 10 + assert config.get("async.timeout") is None + assert config.get("codec_pipeline.batch_size") == 1 + assert config.get("json_indent") == 2 + + +@pytest.mark.parametrize( + ("key", "old_val", "new_val"), + [("array.order", "C", "F"), ("async.concurrency", 10, 20), ("json_indent", 2, 0)], +) +def test_config_defaults_can_be_overridden(key: str, old_val: Any, new_val: Any) -> None: + assert config.get(key) == old_val + with config.set({key: new_val}): + assert config.get(key) == new_val + + +def test_fully_qualified_name() -> None: + class MockClass: + pass + + assert ( + fully_qualified_name(MockClass) + == "tests.test_config.test_fully_qualified_name..MockClass" + ) + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_config_codec_pipeline_class(store: Store) -> None: + # has default value + assert get_pipeline_class().__name__ != "" + + config.set({"codec_pipeline.name": "zarr.core.codec_pipeline.BatchedCodecPipeline"}) + assert get_pipeline_class() == zarr.core.codec_pipeline.BatchedCodecPipeline + + _mock = Mock() + + class MockCodecPipeline(BatchedCodecPipeline): + async def write( + self, + batch_info: Iterable[tuple[ByteSetter, ArraySpec, SelectorTuple, SelectorTuple, bool]], + value: NDBuffer, + drop_axes: tuple[int, ...] = (), + ) -> None: + _mock.call() + + register_pipeline(MockCodecPipeline) + config.set({"codec_pipeline.path": fully_qualified_name(MockCodecPipeline)}) + + assert get_pipeline_class() == MockCodecPipeline + + # test if codec is used + arr = zarr.create_array( + store=store, + shape=(100,), + chunks=(10,), + zarr_format=3, + dtype="i4", + ) + arr[:] = range(100) + + _mock.call.assert_called() + + with pytest.raises(BadConfigError): + config.set({"codec_pipeline.path": "wrong_name"}) + get_pipeline_class() + + class MockEnvCodecPipeline(CodecPipeline): + pass + + register_pipeline(MockEnvCodecPipeline) # type: ignore[type-abstract] + + with mock.patch.dict( + os.environ, {"ZARR_CODEC_PIPELINE__PATH": fully_qualified_name(MockEnvCodecPipeline)} + ): + assert get_pipeline_class(reload_config=True) == MockEnvCodecPipeline + + +@pytest.mark.filterwarnings("error") +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_config_codec_implementation(store: Store) -> None: + # has default value + assert fully_qualified_name(get_codec_class("blosc")) == config.defaults[0]["codecs"]["blosc"] + + _mock = Mock() + + class MockBloscCodec(BloscCodec): + async def _encode_single(self, chunk_bytes: Buffer, chunk_spec: ArraySpec) -> Buffer | None: + _mock.call() + return None + + register_codec("blosc", MockBloscCodec) + with config.set({"codecs.blosc": fully_qualified_name(MockBloscCodec)}): + assert get_codec_class("blosc") == MockBloscCodec + + # test if codec is used + arr = zarr.create_array( + store=store, + shape=(100,), + chunks=(10,), + zarr_format=3, + dtype="i4", + compressors=[{"name": "blosc", "configuration": {}}], + ) + arr[:] = range(100) + _mock.call.assert_called() + + # test set codec with environment variable + class NewBloscCodec(BloscCodec): + pass + + register_codec("blosc", NewBloscCodec) + with mock.patch.dict(os.environ, {"ZARR_CODECS__BLOSC": fully_qualified_name(NewBloscCodec)}): + assert get_codec_class("blosc", reload_config=True) == NewBloscCodec + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_config_ndbuffer_implementation(store: Store) -> None: + # set custom ndbuffer with TestNDArrayLike implementation + register_ndbuffer(NDBufferUsingTestNDArrayLike) + with config.set({"ndbuffer": fully_qualified_name(NDBufferUsingTestNDArrayLike)}): + assert get_ndbuffer_class() == NDBufferUsingTestNDArrayLike + arr = zarr.create_array( + store=store, + shape=(100,), + chunks=(10,), + zarr_format=3, + dtype="i4", + ) + got = arr[:] + assert isinstance(got, TestNDArrayLike) + + +def test_config_buffer_implementation() -> None: + # has default value + assert config.defaults[0]["buffer"] == "zarr.buffer.cpu.Buffer" + + arr = zeros(shape=(100,), store=StoreExpectingTestBuffer()) + + # AssertionError of StoreExpectingTestBuffer when not using my buffer + with pytest.raises(AssertionError): + arr[:] = np.arange(100) + + register_buffer(TestBuffer) + with config.set({"buffer": fully_qualified_name(TestBuffer)}): + assert get_buffer_class() == TestBuffer + + # no error using TestBuffer + data = np.arange(100) + arr[:] = np.arange(100) + assert np.array_equal(arr[:], data) + + data2d = np.arange(1000).reshape(100, 10) + arr_sharding = zeros( + shape=(100, 10), + store=StoreExpectingTestBuffer(), + codecs=[ShardingCodec(chunk_shape=(10, 10))], + ) + arr_sharding[:] = data2d + assert np.array_equal(arr_sharding[:], data2d) + + arr_Crc32c = zeros( + shape=(100, 10), + store=StoreExpectingTestBuffer(), + codecs=[BytesCodec(), Crc32cCodec()], + ) + arr_Crc32c[:] = data2d + assert np.array_equal(arr_Crc32c[:], data2d) + + +def test_config_buffer_backwards_compatibility() -> None: + # This should warn once zarr.core is private + # https://github.com/zarr-developers/zarr-python/issues/2621 + with zarr.config.set( + {"buffer": "zarr.core.buffer.cpu.Buffer", "ndbuffer": "zarr.core.buffer.cpu.NDBuffer"} + ): + get_buffer_class() + get_ndbuffer_class() + + +@pytest.mark.gpu +def test_config_buffer_backwards_compatibility_gpu() -> None: + # This should warn once zarr.core is private + # https://github.com/zarr-developers/zarr-python/issues/2621 + with zarr.config.set( + {"buffer": "zarr.core.buffer.gpu.Buffer", "ndbuffer": "zarr.core.buffer.gpu.NDBuffer"} + ): + get_buffer_class() + get_ndbuffer_class() + + +@pytest.mark.filterwarnings("error") +def test_warning_on_missing_codec_config() -> None: + class NewCodec(BytesCodec): + pass + + class NewCodec2(BytesCodec): + pass + + # error if codec is not registered + with pytest.raises(KeyError): + get_codec_class("missing_codec") + + # no warning if only one implementation is available + register_codec("new_codec", NewCodec) + get_codec_class("new_codec") + + # warning because multiple implementations are available but none is selected in the config + register_codec("new_codec", NewCodec2) + with pytest.warns(UserWarning): + get_codec_class("new_codec") + + # no warning if multiple implementations are available and one is selected in the config + with config.set({"codecs.new_codec": fully_qualified_name(NewCodec)}): + get_codec_class("new_codec") + + +@pytest.mark.parametrize("dtype_category", ["variable-length-string", "default"]) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +async def test_default_codecs(dtype_category: str) -> None: + """ + Test that the default compressors are sensitive to the current setting of the config. + """ + zdtype: ZDType[Any, Any] + if dtype_category == "variable-length-string": + zdtype = VariableLengthUTF8() + else: + zdtype = Int8() + expected_compressors = (GzipCodec(),) + new_conf = { + f"array.v3_default_compressors.{dtype_category}": [ + c.to_dict() for c in expected_compressors + ] + } + with config.set(new_conf): + arr = await create_array( + shape=(100,), + chunks=(100,), + dtype=zdtype, + zarr_format=3, + store=MemoryStore(), + ) + assert arr.compressors == expected_compressors diff --git a/tests/test_dtype/__init__.py b/tests/test_dtype/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_dtype/conftest.py b/tests/test_dtype/conftest.py new file mode 100644 index 0000000000..0be1c60088 --- /dev/null +++ b/tests/test_dtype/conftest.py @@ -0,0 +1,71 @@ +# Generate a collection of zdtype instances for use in testing. +import warnings +from typing import Any + +import numpy as np + +from zarr.core.dtype import data_type_registry +from zarr.core.dtype.common import HasLength +from zarr.core.dtype.npy.structured import Structured +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64 +from zarr.core.dtype.wrapper import ZDType + +zdtype_examples: tuple[ZDType[Any, Any], ...] = () +for wrapper_cls in data_type_registry.contents.values(): + # The Structured dtype has to be constructed with some actual fields + if wrapper_cls is Structured: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + zdtype_examples += ( + wrapper_cls.from_native_dtype(np.dtype([("a", np.float64), ("b", np.int8)])), + ) + elif issubclass(wrapper_cls, HasLength): + zdtype_examples += (wrapper_cls(length=1),) + elif issubclass(wrapper_cls, DateTime64 | TimeDelta64): + zdtype_examples += (wrapper_cls(unit="s", scale_factor=10),) + else: + zdtype_examples += (wrapper_cls(),) + + +def pytest_generate_tests(metafunc: Any) -> None: + """ + This is a pytest hook to parametrize class-scoped fixtures. + + This hook allows us to define class-scoped fixtures as class attributes and then + generate the parametrize calls for pytest. This allows the fixtures to be + reused across multiple tests within the same class. + + For example, if you had a regular pytest class like this: + + class TestClass: + @pytest.mark.parametrize("param_a", [1, 2, 3]) + def test_method(self, param_a): + ... + + Child classes inheriting from ``TestClass`` would not be able to override the ``param_a`` fixture + + this implementation of ``pytest_generate_tests`` allows you to define class-scoped fixtures as + class attributes, which allows the following to work: + + class TestExample: + param_a = [1, 2, 3] + + def test_example(self, param_a): + ... + + # this class will have its test_example method parametrized with the values of TestB.param_a + class TestB(TestExample): + param_a = [1, 2, 100, 10] + + """ + # Iterate over all the fixtures defined in the class + # and parametrize them with the values defined in the class + # This allows us to define class-scoped fixtures as class attributes + # and then generate the parametrize calls for pytest + for fixture_name in metafunc.fixturenames: + if hasattr(metafunc.cls, fixture_name): + params = getattr(metafunc.cls, fixture_name) + if len(params) == 0: + msg = f"{metafunc.cls}.{fixture_name} is empty. Please provide a non-empty sequence of values." + raise ValueError(msg) + metafunc.parametrize(fixture_name, params, scope="class") diff --git a/tests/test_dtype/test_npy/test_bool.py b/tests/test_dtype/test_npy/test_bool.py new file mode 100644 index 0000000000..010dec2e47 --- /dev/null +++ b/tests/test_dtype/test_npy/test_bool.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import numpy as np + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.npy.bool import Bool + + +class TestBool(BaseTestZDType): + test_cls = Bool + + valid_dtype = (np.dtype(np.bool_),) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype(np.uint16), + ) + valid_json_v2 = ({"name": "|b1", "object_codec_id": None},) + valid_json_v3 = ("bool",) + invalid_json_v2 = ( + "|b1", + "bool", + "|f8", + ) + invalid_json_v3 = ( + "|b1", + "|f8", + {"name": "bool", "configuration": {"endianness": "little"}}, + ) + + scalar_v2_params = ((Bool(), True), (Bool(), False)) + scalar_v3_params = ((Bool(), True), (Bool(), False)) + + cast_value_params = ( + (Bool(), "true", np.True_), + (Bool(), True, np.True_), + (Bool(), False, np.False_), + (Bool(), np.True_, np.True_), + (Bool(), np.False_, np.False_), + ) + item_size_params = (Bool(),) diff --git a/tests/test_dtype/test_npy/test_bytes.py b/tests/test_dtype/test_npy/test_bytes.py new file mode 100644 index 0000000000..b7c16f573e --- /dev/null +++ b/tests/test_dtype/test_npy/test_bytes.py @@ -0,0 +1,154 @@ +import numpy as np +import pytest + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.common import UnstableSpecificationWarning +from zarr.core.dtype.npy.bytes import NullTerminatedBytes, RawBytes, VariableLengthBytes + + +class TestNullTerminatedBytes(BaseTestZDType): + test_cls = NullTerminatedBytes + valid_dtype = (np.dtype("|S10"), np.dtype("|S4")) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|U10"), + ) + valid_json_v2 = ( + {"name": "|S0", "object_codec_id": None}, + {"name": "|S2", "object_codec_id": None}, + {"name": "|S4", "object_codec_id": None}, + ) + valid_json_v3 = ({"name": "null_terminated_bytes", "configuration": {"length_bytes": 10}},) + invalid_json_v2 = ( + "|S", + "|U10", + "|f8", + ) + invalid_json_v3 = ( + {"name": "fixed_length_ascii", "configuration": {"length_bits": 0}}, + {"name": "numpy.fixed_length_ascii", "configuration": {"length_bits": "invalid"}}, + ) + + scalar_v2_params = ( + (NullTerminatedBytes(length=0), ""), + (NullTerminatedBytes(length=2), "YWI="), + (NullTerminatedBytes(length=4), "YWJjZA=="), + ) + scalar_v3_params = ( + (NullTerminatedBytes(length=0), ""), + (NullTerminatedBytes(length=2), "YWI="), + (NullTerminatedBytes(length=4), "YWJjZA=="), + ) + cast_value_params = ( + (NullTerminatedBytes(length=0), "", np.bytes_("")), + (NullTerminatedBytes(length=2), "ab", np.bytes_("ab")), + (NullTerminatedBytes(length=4), "abcdefg", np.bytes_("abcd")), + ) + item_size_params = ( + NullTerminatedBytes(length=0), + NullTerminatedBytes(length=4), + NullTerminatedBytes(length=10), + ) + + +class TestRawBytes(BaseTestZDType): + test_cls = RawBytes + valid_dtype = (np.dtype("|V10"),) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|S10"), + ) + valid_json_v2 = ({"name": "|V10", "object_codec_id": None},) + valid_json_v3 = ( + {"name": "raw_bytes", "configuration": {"length_bytes": 0}}, + {"name": "raw_bytes", "configuration": {"length_bytes": 8}}, + ) + + invalid_json_v2 = ( + "|V", + "|S10", + "|f8", + ) + invalid_json_v3 = ( + {"name": "r10"}, + {"name": "r-80"}, + ) + + scalar_v2_params = ( + (RawBytes(length=0), ""), + (RawBytes(length=2), "YWI="), + (RawBytes(length=4), "YWJjZA=="), + ) + scalar_v3_params = ( + (RawBytes(length=0), ""), + (RawBytes(length=2), "YWI="), + (RawBytes(length=4), "YWJjZA=="), + ) + cast_value_params = ( + (RawBytes(length=0), b"", np.void(b"")), + (RawBytes(length=2), b"ab", np.void(b"ab")), + (RawBytes(length=4), b"abcd", np.void(b"abcd")), + ) + item_size_params = ( + RawBytes(length=0), + RawBytes(length=4), + RawBytes(length=10), + ) + + +class TestVariableLengthBytes(BaseTestZDType): + test_cls = VariableLengthBytes + valid_dtype = (np.dtype("|O"),) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|U10"), + ) + valid_json_v2 = ({"name": "|O", "object_codec_id": "vlen-bytes"},) + valid_json_v3 = ("variable_length_bytes",) + invalid_json_v2 = ( + "|S", + "|U10", + "|f8", + ) + invalid_json_v3 = ( + {"name": "fixed_length_ascii", "configuration": {"length_bits": 0}}, + {"name": "numpy.fixed_length_ascii", "configuration": {"length_bits": "invalid"}}, + ) + + scalar_v2_params = ( + (VariableLengthBytes(), ""), + (VariableLengthBytes(), "YWI="), + (VariableLengthBytes(), "YWJjZA=="), + ) + scalar_v3_params = ( + (VariableLengthBytes(), ""), + (VariableLengthBytes(), "YWI="), + (VariableLengthBytes(), "YWJjZA=="), + ) + cast_value_params = ( + (VariableLengthBytes(), "", b""), + (VariableLengthBytes(), "ab", b"ab"), + (VariableLengthBytes(), "abcdefg", b"abcdefg"), + ) + item_size_params = ( + VariableLengthBytes(), + VariableLengthBytes(), + VariableLengthBytes(), + ) + + +@pytest.mark.parametrize( + "zdtype", [NullTerminatedBytes(length=10), RawBytes(length=10), VariableLengthBytes()] +) +def test_unstable_dtype_warning( + zdtype: NullTerminatedBytes | RawBytes | VariableLengthBytes, +) -> None: + """ + Test that we get a warning when serializing a dtype without a zarr v3 spec to json + when zarr_format is 3 + """ + with pytest.raises(UnstableSpecificationWarning): + zdtype.to_json(zarr_format=3) diff --git a/tests/test_dtype/test_npy/test_common.py b/tests/test_dtype/test_npy/test_common.py new file mode 100644 index 0000000000..d39d308112 --- /dev/null +++ b/tests/test_dtype/test_npy/test_common.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import base64 +import math +import re +import sys +from typing import TYPE_CHECKING, Any, get_args + +import numpy as np +import pytest + +from zarr.core.dtype.common import ENDIANNESS_STR, JSONFloatV2, SpecialFloatStrings +from zarr.core.dtype.npy.common import ( + NumpyEndiannessStr, + bytes_from_json, + bytes_to_json, + check_json_bool, + check_json_complex_float_v2, + check_json_complex_float_v3, + check_json_float_v2, + check_json_float_v3, + check_json_int, + check_json_str, + complex_float_to_json_v2, + complex_float_to_json_v3, + endianness_from_numpy_str, + endianness_to_numpy_str, + float_from_json_v2, + float_from_json_v3, + float_to_json_v2, + float_to_json_v3, +) + +if TYPE_CHECKING: + from zarr.core.common import JSON, ZarrFormat + + +def nan_equal(a: object, b: object) -> bool: + """ + Convenience function for equality comparison between two values ``a`` and ``b``, that might both + be NaN. Returns True if both ``a`` and ``b`` are NaN, otherwise returns a == b + """ + if math.isnan(a) and math.isnan(b): # type: ignore[arg-type] + return True + return a == b + + +json_float_v2_roundtrip_cases: tuple[tuple[JSONFloatV2, float | np.floating[Any]], ...] = ( + ("Infinity", float("inf")), + ("Infinity", np.inf), + ("-Infinity", float("-inf")), + ("-Infinity", -np.inf), + ("NaN", float("nan")), + ("NaN", np.nan), + (1.0, 1.0), +) + +json_float_v3_cases = json_float_v2_roundtrip_cases + + +@pytest.mark.parametrize( + ("data", "expected"), + [(">", "big"), ("<", "little"), ("=", sys.byteorder), ("|", None), ("err", "")], +) +def test_endianness_from_numpy_str(data: str, expected: str | None) -> None: + """ + Test that endianness_from_numpy_str correctly converts a numpy str literal to a human-readable literal value. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data in get_args(NumpyEndiannessStr): + assert endianness_from_numpy_str(data) == expected # type: ignore[arg-type] + else: + msg = f"Invalid endianness: {data!r}. Expected one of {get_args(NumpyEndiannessStr)}" + with pytest.raises(ValueError, match=re.escape(msg)): + endianness_from_numpy_str(data) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("data", "expected"), + [("big", ">"), ("little", "<"), (None, "|"), ("err", "")], +) +def test_endianness_to_numpy_str(data: str | None, expected: str) -> None: + """ + Test that endianness_to_numpy_str correctly converts a human-readable literal value to a numpy str literal. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data in ENDIANNESS_STR: + assert endianness_to_numpy_str(data) == expected # type: ignore[arg-type] + else: + msg = f"Invalid endianness: {data!r}. Expected one of {ENDIANNESS_STR}" + with pytest.raises(ValueError, match=re.escape(msg)): + endianness_to_numpy_str(data) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("data", "expected"), json_float_v2_roundtrip_cases + (("SHOULD_ERR", ""),) +) +def test_float_from_json_v2(data: JSONFloatV2 | str, expected: float | str) -> None: + """ + Test that float_from_json_v2 correctly converts a JSON string representation of a float to a float. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data != "SHOULD_ERR": + assert nan_equal(float_from_json_v2(data), expected) # type: ignore[arg-type] + else: + msg = f"could not convert string to float: {data!r}" + with pytest.raises(ValueError, match=msg): + float_from_json_v2(data) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("data", "expected"), json_float_v3_cases + (("SHOULD_ERR", ""), ("0x", "")) +) +def test_float_from_json_v3(data: JSONFloatV2 | str, expected: float | str) -> None: + """ + Test that float_from_json_v3 correctly converts a JSON string representation of a float to a float. + This test also checks that an invalid string input raises a ``ValueError`` + """ + if data == "SHOULD_ERR": + msg = ( + f"Invalid float value: {data!r}. Expected a string starting with the hex prefix" + " '0x', or one of 'NaN', 'Infinity', or '-Infinity'." + ) + with pytest.raises(ValueError, match=msg): + float_from_json_v3(data) + elif data == "0x": + msg = ( + f"Invalid hexadecimal float value: {data!r}. " + "Expected the '0x' prefix to be followed by 4, 8, or 16 numeral characters" + ) + + with pytest.raises(ValueError, match=msg): + float_from_json_v3(data) + else: + assert nan_equal(float_from_json_v3(data), expected) + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("expected", "data"), json_float_v2_roundtrip_cases) +def test_float_to_json_v2(data: float | np.floating[Any], expected: JSONFloatV2) -> None: + """ + Test that floats are JSON-encoded properly for zarr v2 + """ + observed = float_to_json_v2(data) + assert observed == expected + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("expected", "data"), json_float_v3_cases) +def test_float_to_json_v3(data: float | np.floating[Any], expected: JSONFloatV2) -> None: + """ + Test that floats are JSON-encoded properly for zarr v3 + """ + observed = float_to_json_v3(data) + assert observed == expected + + +def test_bytes_from_json(zarr_format: ZarrFormat) -> None: + """ + Test that a string is interpreted as base64-encoded bytes using the ascii alphabet. + This test takes zarr_format as a parameter but doesn't actually do anything with it, because at + present there is no zarr-format-specific logic in the code being tested, but such logic may + exist in the future. + """ + data = "\00" + assert bytes_from_json(data, zarr_format=zarr_format) == base64.b64decode(data.encode("ascii")) + + +def test_bytes_to_json(zarr_format: ZarrFormat) -> None: + """ + Test that bytes are encoded with base64 using the ascii alphabet. + + This test takes zarr_format as a parameter but doesn't actually do anything with it, because at + present there is no zarr-format-specific logic in the code being tested, but such logic may + exist in the future. + """ + + data = b"asdas" + assert bytes_to_json(data, zarr_format=zarr_format) == base64.b64encode(data).decode("ascii") + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("json_expected", "float_data"), json_float_v2_roundtrip_cases) +def test_complex_to_json_v2( + float_data: float | np.floating[Any], json_expected: JSONFloatV2 +) -> None: + """ + Test that complex numbers are correctly converted to JSON in v2 format. + + This use the same test input as the float tests, but the conversion is tested + for complex numbers with real and imaginary parts equal to the float + values provided in the test cases. + """ + cplx = complex(float_data, float_data) + cplx_npy = np.complex128(cplx) + assert complex_float_to_json_v2(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v2(cplx_npy) == (json_expected, json_expected) + + +# note the order of parameters relative to the order of the parametrized variable. +@pytest.mark.parametrize(("json_expected", "float_data"), json_float_v3_cases) +def test_complex_to_json_v3( + float_data: float | np.floating[Any], json_expected: JSONFloatV2 +) -> None: + """ + Test that complex numbers are correctly converted to JSON in v3 format. + + This use the same test input as the float tests, but the conversion is tested + for complex numbers with real and imaginary parts equal to the float + values provided in the test cases. + """ + cplx = complex(float_data, float_data) + cplx_npy = np.complex128(cplx) + assert complex_float_to_json_v3(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v3(cplx_npy) == (json_expected, json_expected) + + +@pytest.mark.parametrize(("json_expected", "float_data"), json_float_v3_cases) +def test_complex_float_to_json( + float_data: float | np.floating[Any], json_expected: JSONFloatV2, zarr_format: ZarrFormat +) -> None: + """ + Test that complex numbers are correctly converted to JSON in v2 or v3 formats, depending + on the ``zarr_format`` keyword argument. + + This use the same test input as the float tests, but the conversion is tested + for complex numbers with real and imaginary parts equal to the float + values provided in the test cases. + """ + + cplx = complex(float_data, float_data) + cplx_npy = np.complex128(cplx) + if zarr_format == 2: + assert complex_float_to_json_v2(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v2(cplx_npy) == ( + json_expected, + json_expected, + ) + elif zarr_format == 3: + assert complex_float_to_json_v3(cplx) == (json_expected, json_expected) + assert complex_float_to_json_v3(cplx_npy) == ( + json_expected, + json_expected, + ) + else: + raise ValueError("zarr_format must be 2 or 3") # pragma: no cover + + +check_json_float_cases = get_args(SpecialFloatStrings) + (1.0, 2) + + +@pytest.mark.parametrize("data", check_json_float_cases) +def test_check_json_float_v2_valid(data: JSONFloatV2 | int) -> None: + assert check_json_float_v2(data) + + +def test_check_json_float_v2_invalid() -> None: + assert not check_json_float_v2("invalid") + + +@pytest.mark.parametrize("data", check_json_float_cases) +def test_check_json_float_v3_valid(data: JSONFloatV2 | int) -> None: + assert check_json_float_v3(data) + + +def test_check_json_float_v3_invalid() -> None: + assert not check_json_float_v3("invalid") + + +check_json_complex_float_true_cases: tuple[list[JSONFloatV2], ...] = ( + [0.0, 1.0], + [0.0, 1.0], + [-1.0, "NaN"], + ["Infinity", 1.0], + ["Infinity", "NaN"], +) + +check_json_complex_float_false_cases: tuple[object, ...] = ( + 0.0, + "foo", + [0.0], + [1.0, 2.0, 3.0], + [1.0, "_infinity_"], + {"hello": 1.0}, +) + + +@pytest.mark.parametrize("data", check_json_complex_float_true_cases) +def test_check_json_complex_float_v2_true(data: JSON) -> None: + assert check_json_complex_float_v2(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_false_cases) +def test_check_json_complex_float_v2_false(data: JSON) -> None: + assert not check_json_complex_float_v2(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_true_cases) +def test_check_json_complex_float_v3_true(data: JSON) -> None: + assert check_json_complex_float_v3(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_false_cases) +def test_check_json_complex_float_v3_false(data: JSON) -> None: + assert not check_json_complex_float_v3(data) + + +@pytest.mark.parametrize("data", check_json_complex_float_true_cases) +def test_check_json_complex_float_true(data: JSON, zarr_format: ZarrFormat) -> None: + if zarr_format == 2: + assert check_json_complex_float_v2(data) + elif zarr_format == 3: + assert check_json_complex_float_v3(data) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +@pytest.mark.parametrize("data", check_json_complex_float_false_cases) +def test_check_json_complex_float_false(data: JSON, zarr_format: ZarrFormat) -> None: + if zarr_format == 2: + assert not check_json_complex_float_v2(data) + elif zarr_format == 3: + assert not check_json_complex_float_v3(data) + else: + raise ValueError(f"zarr_format must be 2 or 3, got {zarr_format}") # pragma: no cover + + +def test_check_json_int() -> None: + assert check_json_int(0) + assert not check_json_int(1.0) + + +def test_check_json_str() -> None: + assert check_json_str("0") + assert not check_json_str(1.0) + + +def test_check_json_bool() -> None: + assert check_json_bool(True) + assert check_json_bool(False) + assert not check_json_bool(1.0) + assert not check_json_bool("True") diff --git a/tests/test_dtype/test_npy/test_complex.py b/tests/test_dtype/test_npy/test_complex.py new file mode 100644 index 0000000000..b6a1e799eb --- /dev/null +++ b/tests/test_dtype/test_npy/test_complex.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import math + +import numpy as np + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.npy.complex import Complex64, Complex128 + + +class _BaseTestFloat(BaseTestZDType): + def scalar_equals(self, scalar1: object, scalar2: object) -> bool: + if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] + return True + return super().scalar_equals(scalar1, scalar2) + + +class TestComplex64(_BaseTestFloat): + test_cls = Complex64 + valid_dtype = (np.dtype(">c8"), np.dtype("c8", "object_codec_id": None}, + {"name": "c16"), np.dtype("c16", "object_codec_id": None}, + {"name": " bool: + if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] + return True + return super().scalar_equals(scalar1, scalar2) + + hex_string_params: tuple[tuple[str, float], ...] = () + + def test_hex_encoding(self, hex_string_params: tuple[str, float]) -> None: + """ + Test that hexadecimal strings can be read as NaN values + """ + hex_string, expected = hex_string_params + zdtype = self.test_cls() + observed = zdtype.from_json_scalar(hex_string, zarr_format=3) + assert self.scalar_equals(observed, expected) + + +class TestFloat16(_BaseTestFloat): + test_cls = Float16 + valid_dtype = (np.dtype(">f2"), np.dtype("f2", "object_codec_id": None}, + {"name": "f4"), np.dtype("f4", "object_codec_id": None}, + {"name": "f8"), np.dtype("f8", "object_codec_id": None}, + {"name": "i1", + "int8", + "|f8", + ) + invalid_json_v3 = ( + "|i1", + "|f8", + {"name": "int8", "configuration": {"endianness": "little"}}, + ) + + scalar_v2_params = ((Int8(), 1), (Int8(), -1)) + scalar_v3_params = ((Int8(), 1), (Int8(), -1)) + cast_value_params = ( + (Int8(), 1, np.int8(1)), + (Int8(), -1, np.int8(-1)), + ) + item_size_params = (Int8(),) + + +class TestInt16(BaseTestZDType): + test_cls = Int16 + scalar_type = np.int16 + valid_dtype = (np.dtype(">i2"), np.dtype("i2", "object_codec_id": None}, + {"name": "i4"), np.dtype("i4", "object_codec_id": None}, + {"name": "i8"), np.dtype("i8", "object_codec_id": None}, + {"name": "u2"), np.dtype("u2", "object_codec_id": None}, + {"name": "u4"), np.dtype("u4", "object_codec_id": None}, + {"name": "u8"), np.dtype("u8", "object_codec_id": None}, + {"name": "U10"), np.dtype("U10", "object_codec_id": None}, + {"name": " None: + """ + Test that we get a warning when serializing a dtype without a zarr v3 spec to json + when zarr_format is 3 + """ + with pytest.raises(UnstableSpecificationWarning): + zdtype.to_json(zarr_format=3) diff --git a/tests/test_dtype/test_npy/test_structured.py b/tests/test_dtype/test_npy/test_structured.py new file mode 100644 index 0000000000..e9c9ab11d0 --- /dev/null +++ b/tests/test_dtype/test_npy/test_structured.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype import ( + Float16, + Float64, + Int32, + Int64, + Structured, +) + + +class TestStructured(BaseTestZDType): + test_cls = Structured + valid_dtype = ( + np.dtype([("field1", np.int32), ("field2", np.float64)]), + np.dtype([("field1", np.int64), ("field2", np.int32)]), + ) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("|S10"), + ) + valid_json_v2 = ( + {"name": [["field1", ">i4"], ["field2", ">f8"]], "object_codec_id": None}, + {"name": [["field1", ">i8"], ["field2", ">i4"]], "object_codec_id": None}, + ) + valid_json_v3 = ( + { + "name": "structured", + "configuration": { + "fields": [ + ["field1", "int32"], + ["field2", "float64"], + ] + }, + }, + { + "name": "structured", + "configuration": { + "fields": [ + [ + "field1", + { + "name": "numpy.datetime64", + "configuration": {"unit": "s", "scale_factor": 1}, + }, + ], + [ + "field2", + {"name": "fixed_length_utf32", "configuration": {"length_bytes": 32}}, + ], + ] + }, + }, + ) + invalid_json_v2 = ( + [("field1", "|i1"), ("field2", "|f8")], + [("field1", "|S10"), ("field2", "|f8")], + ) + invalid_json_v3 = ( + { + "name": "structured", + "configuration": { + "fields": [ + ("field1", {"name": "int32", "configuration": {"endianness": "invalid"}}), + ("field2", {"name": "float64", "configuration": {"endianness": "big"}}), + ] + }, + }, + {"name": "invalid_name"}, + ) + + scalar_v2_params = ( + (Structured(fields=(("field1", Int32()), ("field2", Float64()))), "AQAAAAAAAAAAAPA/"), + (Structured(fields=(("field1", Float16()), ("field2", Int32()))), "AQAAAAAA"), + ) + scalar_v3_params = ( + (Structured(fields=(("field1", Int32()), ("field2", Float64()))), "AQAAAAAAAAAAAPA/"), + (Structured(fields=(("field1", Int64()), ("field2", Int32()))), "AQAAAAAAAAAAAPA/"), + ) + + cast_value_params = ( + ( + Structured(fields=(("field1", Int32()), ("field2", Float64()))), + (1, 2.0), + np.array((1, 2.0), dtype=[("field1", np.int32), ("field2", np.float64)]), + ), + ( + Structured(fields=(("field1", Int64()), ("field2", Int32()))), + (3, 4.5), + np.array((3, 4.5), dtype=[("field1", np.int64), ("field2", np.int32)]), + ), + ) + + def scalar_equals(self, scalar1: Any, scalar2: Any) -> bool: + if hasattr(scalar1, "shape") and hasattr(scalar2, "shape"): + return np.array_equal(scalar1, scalar2) + return super().scalar_equals(scalar1, scalar2) + + item_size_params = ( + Structured(fields=(("field1", Int32()), ("field2", Float64()))), + Structured(fields=(("field1", Int64()), ("field2", Int32()))), + ) diff --git a/tests/test_dtype/test_npy/test_time.py b/tests/test_dtype/test_npy/test_time.py new file mode 100644 index 0000000000..e201be5cf6 --- /dev/null +++ b/tests/test_dtype/test_npy/test_time.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import re +from typing import get_args + +import numpy as np +import pytest + +from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.dtype.npy.common import DateTimeUnit +from zarr.core.dtype.npy.time import DateTime64, TimeDelta64, datetime_from_int + + +class _TestTimeBase(BaseTestZDType): + def json_scalar_equals(self, scalar1: object, scalar2: object) -> bool: + # This method gets overridden here to support the equivalency between NaT and + # -9223372036854775808 fill values + nat_scalars = (-9223372036854775808, "NaT") + if scalar1 in nat_scalars and scalar2 in nat_scalars: + return True + return scalar1 == scalar2 + + def scalar_equals(self, scalar1: object, scalar2: object) -> bool: + if np.isnan(scalar1) and np.isnan(scalar2): # type: ignore[call-overload] + return True + return super().scalar_equals(scalar1, scalar2) + + +class TestDateTime64(_TestTimeBase): + test_cls = DateTime64 + valid_dtype = (np.dtype("datetime64[10ns]"), np.dtype("datetime64[us]"), np.dtype("datetime64")) + invalid_dtype = ( + np.dtype(np.int8), + np.dtype(np.float64), + np.dtype("timedelta64[ns]"), + ) + valid_json_v2 = ( + {"name": ">M8", "object_codec_id": None}, + {"name": ">M8[s]", "object_codec_id": None}, + {"name": "m8", "object_codec_id": None}, + {"name": ">m8[s]", "object_codec_id": None}, + {"name": " None: + """ + Test that an invalid unit raises a ValueError. + """ + unit = "invalid" + msg = f"unit must be one of ('Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', 'us', 'μs', 'ns', 'ps', 'fs', 'as', 'generic'), got {unit!r}." + with pytest.raises(ValueError, match=re.escape(msg)): + DateTime64(unit=unit) # type: ignore[arg-type] + with pytest.raises(ValueError, match=re.escape(msg)): + TimeDelta64(unit=unit) # type: ignore[arg-type] + + +def test_time_scale_factor_too_low() -> None: + """ + Test that an invalid unit raises a ValueError. + """ + scale_factor = 0 + msg = f"scale_factor must be > 0, got {scale_factor}." + with pytest.raises(ValueError, match=msg): + DateTime64(scale_factor=scale_factor) + with pytest.raises(ValueError, match=msg): + TimeDelta64(scale_factor=scale_factor) + + +def test_time_scale_factor_too_high() -> None: + """ + Test that an invalid unit raises a ValueError. + """ + scale_factor = 2**31 + msg = f"scale_factor must be < 2147483648, got {scale_factor}." + with pytest.raises(ValueError, match=msg): + DateTime64(scale_factor=scale_factor) + with pytest.raises(ValueError, match=msg): + TimeDelta64(scale_factor=scale_factor) + + +@pytest.mark.parametrize("unit", get_args(DateTimeUnit)) +@pytest.mark.parametrize("scale_factor", [1, 10]) +@pytest.mark.parametrize("value", [0, 1, 10]) +def test_datetime_from_int(unit: DateTimeUnit, scale_factor: int, value: int) -> None: + """ + Test datetime_from_int. + """ + expected = np.int64(value).view(f"datetime64[{scale_factor}{unit}]") + assert datetime_from_int(value, unit=unit, scale_factor=scale_factor) == expected diff --git a/tests/test_dtype/test_wrapper.py b/tests/test_dtype/test_wrapper.py new file mode 100644 index 0000000000..8f461f1a77 --- /dev/null +++ b/tests/test_dtype/test_wrapper.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar + +import pytest + +from zarr.core.dtype.common import DTypeSpec_V2, DTypeSpec_V3, HasItemSize + +if TYPE_CHECKING: + from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType + + +""" +class _TestZDTypeSchema: + # subclasses define the URL for the schema, if available + schema_url: ClassVar[str] = "" + + @pytest.fixture(scope="class") + def get_schema(self) -> object: + response = requests.get(self.schema_url) + response.raise_for_status() + return json_schema.loads(response.text) + + def test_schema(self, schema: json_schema.Schema) -> None: + assert schema.is_valid(self.test_cls.to_json(zarr_format=2)) +""" + + +class BaseTestZDType: + """ + A base class for testing ZDType subclasses. This class works in conjunction with the custom + pytest collection function ``pytest_generate_tests`` defined in conftest.py, which applies the + following procedure when generating tests: + + At test generation time, for each test fixture referenced by a method on this class + pytest will look for an attribute with the same name as that fixture. Pytest will assume that + this class attribute is a tuple of values to be used for generating a parametrized test fixture. + + This means that child classes can, by using different values for these class attributes, have + customized test parametrization. + + Attributes + ---------- + test_cls : type[ZDType[TBaseDType, TBaseScalar]] + The ZDType subclass being tested. + scalar_type : ClassVar[type[TBaseScalar]] + The expected scalar type for the ZDType. + valid_dtype : ClassVar[tuple[TBaseDType, ...]] + A tuple of valid numpy dtypes for the ZDType. + invalid_dtype : ClassVar[tuple[TBaseDType, ...]] + A tuple of invalid numpy dtypes for the ZDType. + valid_json_v2 : ClassVar[tuple[str | dict[str, object] | list[object], ...]] + A tuple of valid JSON representations for Zarr format version 2. + invalid_json_v2 : ClassVar[tuple[str | dict[str, object] | list[object], ...]] + A tuple of invalid JSON representations for Zarr format version 2. + valid_json_v3 : ClassVar[tuple[str | dict[str, object], ...]] + A tuple of valid JSON representations for Zarr format version 3. + invalid_json_v3 : ClassVar[tuple[str | dict[str, object], ...]] + A tuple of invalid JSON representations for Zarr format version 3. + cast_value_params : ClassVar[tuple[tuple[Any, Any, Any], ...]] + A tuple of (dtype, value, expected) tuples for testing ZDType.cast_value. + """ + + test_cls: type[ZDType[TBaseDType, TBaseScalar]] + scalar_type: ClassVar[type[TBaseScalar]] + valid_dtype: ClassVar[tuple[TBaseDType, ...]] = () + invalid_dtype: ClassVar[tuple[TBaseDType, ...]] = () + + valid_json_v2: ClassVar[tuple[DTypeSpec_V2, ...]] = () + invalid_json_v2: ClassVar[tuple[str | dict[str, object] | list[object], ...]] = () + + valid_json_v3: ClassVar[tuple[DTypeSpec_V3, ...]] = () + invalid_json_v3: ClassVar[tuple[str | dict[str, object], ...]] = () + + # for testing scalar round-trip serialization, we need a tuple of (data type json, scalar json) + # pairs. the first element of the pair is used to create a dtype instance, and the second + # element is the json serialization of the scalar that we want to round-trip. + + scalar_v2_params: ClassVar[tuple[tuple[Any, Any], ...]] = () + scalar_v3_params: ClassVar[tuple[tuple[Any, Any], ...]] = () + cast_value_params: ClassVar[tuple[tuple[Any, Any, Any], ...]] + item_size_params: ClassVar[tuple[ZDType[Any, Any], ...]] + + def json_scalar_equals(self, scalar1: object, scalar2: object) -> bool: + # An equality check for json-encoded scalars. This defaults to regular equality, + # but some classes may need to override this for special cases + return scalar1 == scalar2 + + def scalar_equals(self, scalar1: object, scalar2: object) -> bool: + # An equality check for scalars. This defaults to regular equality, + # but some classes may need to override this for special cases + return scalar1 == scalar2 + + def test_check_dtype_valid(self, valid_dtype: TBaseDType) -> None: + assert self.test_cls._check_native_dtype(valid_dtype) + + def test_check_dtype_invalid(self, invalid_dtype: object) -> None: + assert not self.test_cls._check_native_dtype(invalid_dtype) # type: ignore[arg-type] + + def test_from_dtype_roundtrip(self, valid_dtype: Any) -> None: + zdtype = self.test_cls.from_native_dtype(valid_dtype) + assert zdtype.to_native_dtype() == valid_dtype + + def test_from_json_roundtrip_v2(self, valid_json_v2: DTypeSpec_V2) -> None: + zdtype = self.test_cls.from_json(valid_json_v2, zarr_format=2) + assert zdtype.to_json(zarr_format=2) == valid_json_v2 + + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + def test_from_json_roundtrip_v3(self, valid_json_v3: DTypeSpec_V3) -> None: + zdtype = self.test_cls.from_json(valid_json_v3, zarr_format=3) + assert zdtype.to_json(zarr_format=3) == valid_json_v3 + + def test_scalar_roundtrip_v2(self, scalar_v2_params: tuple[ZDType[Any, Any], Any]) -> None: + zdtype, scalar_json = scalar_v2_params + scalar = zdtype.from_json_scalar(scalar_json, zarr_format=2) + assert self.json_scalar_equals(scalar_json, zdtype.to_json_scalar(scalar, zarr_format=2)) + + def test_scalar_roundtrip_v3(self, scalar_v3_params: tuple[ZDType[Any, Any], Any]) -> None: + zdtype, scalar_json = scalar_v3_params + scalar = zdtype.from_json_scalar(scalar_json, zarr_format=3) + assert self.json_scalar_equals(scalar_json, zdtype.to_json_scalar(scalar, zarr_format=3)) + + def test_cast_value(self, cast_value_params: tuple[ZDType[Any, Any], Any, Any]) -> None: + zdtype, value, expected = cast_value_params + observed = zdtype.cast_scalar(value) + assert self.scalar_equals(expected, observed) + + def test_item_size(self, item_size_params: ZDType[Any, Any]) -> None: + """ + Test that the item_size attribute matches the numpy dtype itemsize attribute, for dtypes + with a fixed scalar size. + """ + if isinstance(item_size_params, HasItemSize): + assert item_size_params.item_size == item_size_params.to_native_dtype().itemsize + else: + pytest.skip(f"Dtype {item_size_params} does not implement HasItemSize") diff --git a/tests/test_dtype_registry.py b/tests/test_dtype_registry.py new file mode 100644 index 0000000000..c7d5f90065 --- /dev/null +++ b/tests/test_dtype_registry.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any, get_args + +import numpy as np +import pytest + +import zarr +from tests.conftest import skip_object_dtype +from zarr.core.config import config +from zarr.core.dtype import ( + AnyDType, + Bool, + DataTypeRegistry, + DateTime64, + FixedLengthUTF32, + Int8, + Int16, + TBaseDType, + TBaseScalar, + ZDType, + data_type_registry, + get_data_type_from_json, + parse_data_type, +) + +if TYPE_CHECKING: + from collections.abc import Generator + + from zarr.core.common import ZarrFormat + +from .test_dtype.conftest import zdtype_examples + + +@pytest.fixture +def data_type_registry_fixture() -> DataTypeRegistry: + return DataTypeRegistry() + + +class TestRegistry: + @staticmethod + def test_register(data_type_registry_fixture: DataTypeRegistry) -> None: + """ + Test that registering a dtype in a data type registry works. + """ + data_type_registry_fixture.register(Bool._zarr_v3_name, Bool) + assert data_type_registry_fixture.get(Bool._zarr_v3_name) == Bool + assert isinstance(data_type_registry_fixture.match_dtype(np.dtype("bool")), Bool) + + @staticmethod + def test_override(data_type_registry_fixture: DataTypeRegistry) -> None: + """ + Test that registering a new dtype with the same name works (overriding the previous one). + """ + data_type_registry_fixture.register(Bool._zarr_v3_name, Bool) + + class NewBool(Bool): + def default_scalar(self) -> np.bool_: + return np.True_ + + data_type_registry_fixture.register(NewBool._zarr_v3_name, NewBool) + assert isinstance(data_type_registry_fixture.match_dtype(np.dtype("bool")), NewBool) + + @staticmethod + @pytest.mark.parametrize( + ("wrapper_cls", "dtype_str"), [(Bool, "bool"), (FixedLengthUTF32, "|U4")] + ) + def test_match_dtype( + data_type_registry_fixture: DataTypeRegistry, + wrapper_cls: type[ZDType[TBaseDType, TBaseScalar]], + dtype_str: str, + ) -> None: + """ + Test that match_dtype resolves a numpy dtype into an instance of the correspond wrapper for that dtype. + """ + data_type_registry_fixture.register(wrapper_cls._zarr_v3_name, wrapper_cls) + assert isinstance(data_type_registry_fixture.match_dtype(np.dtype(dtype_str)), wrapper_cls) + + @staticmethod + def test_unregistered_dtype(data_type_registry_fixture: DataTypeRegistry) -> None: + """ + Test that match_dtype raises an error if the dtype is not registered. + """ + outside_dtype_name = "int8" + outside_dtype = np.dtype(outside_dtype_name) + msg = f"No Zarr data type found that matches dtype '{outside_dtype!r}'" + with pytest.raises(ValueError, match=re.escape(msg)): + data_type_registry_fixture.match_dtype(outside_dtype) + + with pytest.raises(KeyError): + data_type_registry_fixture.get(outside_dtype_name) + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("zdtype", zdtype_examples) + def test_registered_dtypes_match_dtype(zdtype: ZDType[TBaseDType, TBaseScalar]) -> None: + """ + Test that the registered dtypes can be retrieved from the registry. + """ + skip_object_dtype(zdtype) + assert data_type_registry.match_dtype(zdtype.to_native_dtype()) == zdtype + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("zdtype", zdtype_examples) + def test_registered_dtypes_match_json( + zdtype: ZDType[TBaseDType, TBaseScalar], zarr_format: ZarrFormat + ) -> None: + assert ( + data_type_registry.match_json( + zdtype.to_json(zarr_format=zarr_format), zarr_format=zarr_format + ) + == zdtype + ) + + @staticmethod + @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") + @pytest.mark.parametrize("zdtype", zdtype_examples) + def test_match_dtype_unique( + zdtype: ZDType[Any, Any], + data_type_registry_fixture: DataTypeRegistry, + zarr_format: ZarrFormat, + ) -> None: + """ + Test that the match_dtype method uniquely specifies a registered data type. We create a local registry + that excludes the data type class being tested, and ensure that an instance of the wrapped data type + fails to match anything in the registry + """ + skip_object_dtype(zdtype) + for _cls in get_args(AnyDType): + if _cls is not type(zdtype): + data_type_registry_fixture.register(_cls._zarr_v3_name, _cls) + + dtype_instance = zdtype.to_native_dtype() + + msg = f"No Zarr data type found that matches dtype '{dtype_instance!r}'" + with pytest.raises(ValueError, match=re.escape(msg)): + data_type_registry_fixture.match_dtype(dtype_instance) + + instance_dict = zdtype.to_json(zarr_format=zarr_format) + msg = f"No Zarr data type found that matches {instance_dict!r}" + with pytest.raises(ValueError, match=re.escape(msg)): + data_type_registry_fixture.match_json(instance_dict, zarr_format=zarr_format) + + +# this is copied from the registry tests -- we should deduplicate +here = str(Path(__file__).parent.absolute()) + + +@pytest.fixture +def set_path() -> Generator[None, None, None]: + sys.path.append(here) + zarr.registry._collect_entrypoints() + yield + sys.path.remove(here) + registries = zarr.registry._collect_entrypoints() + for registry in registries: + registry.lazy_load_list.clear() + config.reset() + + +@pytest.mark.usefixtures("set_path") +def test_entrypoint_dtype(zarr_format: ZarrFormat) -> None: + from package_with_entrypoint import TestDataType + + data_type_registry.lazy_load() + instance = TestDataType() + dtype_json = instance.to_json(zarr_format=zarr_format) + assert get_data_type_from_json(dtype_json, zarr_format=zarr_format) == instance + data_type_registry.unregister(TestDataType._zarr_v3_name) + + +@pytest.mark.parametrize( + ("dtype_params", "expected", "zarr_format"), + [ + ("int8", Int8(), 3), + (Int8(), Int8(), 3), + (">i2", Int16(endianness="big"), 2), + ("datetime64[10s]", DateTime64(unit="s", scale_factor=10), 2), + ( + {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}}, + DateTime64(unit="s", scale_factor=10), + 3, + ), + ], +) +def test_parse_data_type( + dtype_params: Any, expected: ZDType[Any, Any], zarr_format: ZarrFormat +) -> None: + """ + Test that parse_data_type accepts alternative representations of ZDType instances, and resolves + those inputs to the expected ZDType instance. + """ + observed = parse_data_type(dtype_params, zarr_format=zarr_format) + assert observed == expected diff --git a/tests/test_group.py b/tests/test_group.py new file mode 100644 index 0000000000..60a1fcb9bf --- /dev/null +++ b/tests/test_group.py @@ -0,0 +1,2065 @@ +from __future__ import annotations + +import contextlib +import inspect +import operator +import pickle +import re +import time +import warnings +from typing import TYPE_CHECKING, Any, Literal + +import numpy as np +import pytest +from numcodecs import Blosc + +import zarr +import zarr.api.asynchronous +import zarr.api.synchronous +import zarr.storage +from zarr import Array, AsyncArray, AsyncGroup, Group +from zarr.abc.store import Store +from zarr.core import sync_group +from zarr.core._info import GroupInfo +from zarr.core.buffer import default_buffer_prototype +from zarr.core.config import config as zarr_config +from zarr.core.dtype.common import unpack_dtype_json +from zarr.core.dtype.npy.int import UInt8 +from zarr.core.group import ( + ConsolidatedMetadata, + GroupMetadata, + ImplicitGroupMarker, + _build_metadata_v3, + _get_roots, + _parse_hierarchy_dict, + create_hierarchy, + create_nodes, + create_rooted_hierarchy, + get_node, +) +from zarr.core.metadata.v3 import ArrayV3Metadata +from zarr.core.sync import _collect_aiterator, sync +from zarr.errors import ContainsArrayError, ContainsGroupError, MetadataValidationError +from zarr.storage import LocalStore, MemoryStore, StorePath, ZipStore +from zarr.storage._common import make_store_path +from zarr.storage._utils import _join_paths, normalize_path +from zarr.testing.store import LatencyStore + +from .conftest import meta_from_array, parse_store + +if TYPE_CHECKING: + from collections.abc import Callable + + from _pytest.compat import LEGACY_PATH + + from zarr.core.common import JSON, ZarrFormat + + +@pytest.fixture(params=["local", "memory", "zip"]) +async def store(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> Store: + result = await parse_store(request.param, str(tmpdir)) + if not isinstance(result, Store): + raise TypeError("Wrong store class returned by test fixture! got " + result + " instead") + return result + + +@pytest.fixture(params=[True, False]) +def overwrite(request: pytest.FixtureRequest) -> bool: + result = request.param + if not isinstance(result, bool): + raise TypeError("Wrong type returned by test fixture.") + return result + + +def test_group_init(store: Store, zarr_format: ZarrFormat) -> None: + """ + Test that initializing a group from an asyncgroup works. + """ + agroup = sync(AsyncGroup.from_store(store=store, zarr_format=zarr_format)) + group = Group(agroup) + assert group._async_group == agroup + + +async def test_create_creates_parents(store: Store, zarr_format: ZarrFormat) -> None: + # prepare a root node, with some data set + await zarr.api.asynchronous.open_group( + store=store, path="a", zarr_format=zarr_format, attributes={"key": "value"} + ) + objs = {x async for x in store.list()} + if zarr_format == 2: + assert objs == {".zgroup", ".zattrs", "a/.zgroup", "a/.zattrs"} + else: + assert objs == {"zarr.json", "a/zarr.json"} + + # test that root group node was created + root = await zarr.api.asynchronous.open_group( + store=store, + ) + agroup = await root.getitem("a") + assert agroup.attrs == {"key": "value"} + + # create a child node with a couple intermediates + await zarr.api.asynchronous.open_group(store=store, path="a/b/c/d", zarr_format=zarr_format) + parts = ["a", "a/b", "a/b/c"] + + if zarr_format == 2: + files = [".zattrs", ".zgroup"] + else: + files = ["zarr.json"] + + expected = [f"{part}/{file}" for file in files for part in parts] + + if zarr_format == 2: + expected.extend([".zgroup", ".zattrs", "a/b/c/d/.zgroup", "a/b/c/d/.zattrs"]) + else: + expected.extend(["zarr.json", "a/b/c/d/zarr.json"]) + + expected = sorted(expected) + + result = sorted([x async for x in store.list_prefix("")]) + + assert result == expected + + paths = ["a", "a/b", "a/b/c"] + for path in paths: + g = await zarr.api.asynchronous.open_group(store=store, path=path) + assert isinstance(g, AsyncGroup) + + if path == "a": + # ensure we didn't overwrite the root attributes + assert g.attrs == {"key": "value"} + else: + assert g.attrs == {} + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("root_name", ["", "/", "a", "/a"]) +@pytest.mark.parametrize("branch_name", ["foo", "/foo", "foo/bar", "/foo/bar"]) +def test_group_name_properties( + store: Store, zarr_format: ZarrFormat, root_name: str, branch_name: str +) -> None: + """ + Test that the path, name, and basename attributes of a group and its subgroups are consistent + """ + root = Group.from_store(store=StorePath(store=store, path=root_name), zarr_format=zarr_format) + assert root.path == normalize_path(root_name) + assert root.name == "/" + root.path + assert root.basename == root.path + + branch = root.create_group(branch_name) + if root.path == "": + assert branch.path == normalize_path(branch_name) + else: + assert branch.path == "/".join([root.path, normalize_path(branch_name)]) + assert branch.name == "/" + branch.path + assert branch.basename == branch_name.split("/")[-1] + + +@pytest.mark.parametrize("consolidated_metadata", [True, False]) +def test_group_members(store: Store, zarr_format: ZarrFormat, consolidated_metadata: bool) -> None: + """ + Test that `Group.members` returns correct values, i.e. the arrays and groups + (explicit and implicit) contained in that group. + """ + # group/ + # subgroup/ + # subsubgroup/ + # subsubsubgroup + # subarray + + path = "group" + group = Group.from_store( + store=store, + zarr_format=zarr_format, + ) + members_expected: dict[str, Array | Group] = {} + + members_expected["subgroup"] = group.create_group("subgroup") + # make a sub-sub-subgroup, to ensure that the children calculation doesn't go + # too deep in the hierarchy + subsubgroup = members_expected["subgroup"].create_group("subsubgroup") + subsubsubgroup = subsubgroup.create_group("subsubsubgroup") + + members_expected["subarray"] = group.create_array( + "subarray", shape=(100,), dtype="uint8", chunks=(10,), overwrite=True + ) + # add an extra object to the domain of the group. + # the list of children should ignore this object. + sync( + store.set( + f"{path}/extra_object-1", + default_buffer_prototype().buffer.from_bytes(b"000000"), + ) + ) + # add an extra object under a directory-like prefix in the domain of the group. + # this creates a directory with a random key in it + # this should not show up as a member + sync( + store.set( + f"{path}/extra_directory/extra_object-2", + default_buffer_prototype().buffer.from_bytes(b"000000"), + ) + ) + + # this warning shows up when extra objects show up in the hierarchy + warn_context = pytest.warns( + UserWarning, match=r"Object at .* is not recognized as a component of a Zarr hierarchy." + ) + if consolidated_metadata: + with warn_context: + zarr.consolidate_metadata(store=store, zarr_format=zarr_format) + # now that we've consolidated the store, we shouldn't get the warnings from the unrecognized objects anymore + # we use a nullcontext to handle these cases + warn_context = contextlib.nullcontext() + group = zarr.open_consolidated(store=store, zarr_format=zarr_format) + + with warn_context: + members_observed = group.members() + # members are not guaranteed to be ordered, so sort before comparing + assert sorted(dict(members_observed)) == sorted(members_expected) + + # partial + with warn_context: + members_observed = group.members(max_depth=1) + members_expected["subgroup/subsubgroup"] = subsubgroup + # members are not guaranteed to be ordered, so sort before comparing + assert sorted(dict(members_observed)) == sorted(members_expected) + + # total + with warn_context: + members_observed = group.members(max_depth=None) + members_expected["subgroup/subsubgroup/subsubsubgroup"] = subsubsubgroup + # members are not guaranteed to be ordered, so sort before comparing + assert sorted(dict(members_observed)) == sorted(members_expected) + + with pytest.raises(ValueError, match="max_depth"): + members_observed = group.members(max_depth=-1) + + +def test_group(store: Store, zarr_format: ZarrFormat) -> None: + """ + Test basic Group routines. + """ + store_path = StorePath(store) + agroup = AsyncGroup(metadata=GroupMetadata(zarr_format=zarr_format), store_path=store_path) + group = Group(agroup) + assert agroup.metadata is group.metadata + assert agroup.store_path == group.store_path == store_path + + # create two groups + foo = group.create_group("foo") + bar = foo.create_group("bar", attributes={"baz": "qux"}) + + # create an array from the "bar" group + data = np.arange(0, 4 * 4, dtype="uint16").reshape((4, 4)) + arr = bar.create_array("baz", shape=data.shape, dtype=data.dtype, chunks=(2, 2), overwrite=True) + arr[:] = data + + # check the array + assert arr == bar["baz"] + assert arr.shape == data.shape + assert arr.dtype == data.dtype + + # TODO: update this once the array api settles down + assert arr.chunks == (2, 2) + + bar2 = foo["bar"] + assert dict(bar2.attrs) == {"baz": "qux"} + + # update a group's attributes + bar2.attrs.update({"name": "bar"}) + # bar.attrs was modified in-place + assert dict(bar2.attrs) == {"baz": "qux", "name": "bar"} + + # and the attrs were modified in the store + bar3 = foo["bar"] + assert dict(bar3.attrs) == {"baz": "qux", "name": "bar"} + + +def test_group_create(store: Store, overwrite: bool, zarr_format: ZarrFormat) -> None: + """ + Test that `Group.from_store` works as expected. + """ + attributes = {"foo": 100} + group = Group.from_store( + store, attributes=attributes, zarr_format=zarr_format, overwrite=overwrite + ) + + assert group.attrs == attributes + + if not overwrite: + with pytest.raises(ContainsGroupError): + _ = Group.from_store(store, overwrite=overwrite, zarr_format=zarr_format) + + +def test_group_open(store: Store, zarr_format: ZarrFormat, overwrite: bool) -> None: + """ + Test the `Group.open` method. + """ + spath = StorePath(store) + # attempt to open a group that does not exist + with pytest.raises(FileNotFoundError): + Group.open(store) + + # create the group + attrs = {"path": "foo"} + group_created = Group.from_store( + store, attributes=attrs, zarr_format=zarr_format, overwrite=overwrite + ) + assert group_created.attrs == attrs + assert group_created.metadata.zarr_format == zarr_format + assert group_created.store_path == spath + + # attempt to create a new group in place, to test overwrite + new_attrs = {"path": "bar"} + if not overwrite: + with pytest.raises(ContainsGroupError): + Group.from_store(store, attributes=attrs, zarr_format=zarr_format, overwrite=overwrite) + else: + if not store.supports_deletes: + pytest.skip( + "Store does not support deletes but `overwrite` is True, requiring deletes to override a group" + ) + group_created_again = Group.from_store( + store, attributes=new_attrs, zarr_format=zarr_format, overwrite=overwrite + ) + assert group_created_again.attrs == new_attrs + assert group_created_again.metadata.zarr_format == zarr_format + assert group_created_again.store_path == spath + + +@pytest.mark.parametrize("consolidated", [True, False]) +def test_group_getitem(store: Store, zarr_format: ZarrFormat, consolidated: bool) -> None: + """ + Test the `Group.__getitem__` method. + """ + + group = Group.from_store(store, zarr_format=zarr_format) + subgroup = group.create_group(name="subgroup") + subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + subsubarray = subgroup.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + + if consolidated: + group = zarr.api.synchronous.consolidate_metadata(store=store, zarr_format=zarr_format) + # we're going to assume that `group.metadata` is correct, and reuse that to focus + # on indexing in this test. Other tests verify the correctness of group.metadata + object.__setattr__( + subgroup.metadata, + "consolidated_metadata", + ConsolidatedMetadata( + metadata={"subarray": group.metadata.consolidated_metadata.metadata["subarray"]} + ), + ) + + assert group["subgroup"] == subgroup + assert group["subarray"] == subarray + assert group["subgroup"]["subarray"] == subsubarray + assert group["subgroup/subarray"] == subsubarray + + with pytest.raises(KeyError): + group["nope"] + + with pytest.raises(KeyError, match="subarray/subsubarray"): + group["subarray/subsubarray"] + + # Now test the mixed case + if consolidated: + object.__setattr__( + group.metadata.consolidated_metadata.metadata["subgroup"], + "consolidated_metadata", + None, + ) + + # test the implementation directly + with pytest.raises(KeyError): + group._async_group._getitem_consolidated( + group.store_path, "subgroup/subarray", prefix="/" + ) + + with pytest.raises(KeyError): + # We've chosen to trust the consolidated metadata, which doesn't + # contain this array + group["subgroup/subarray"] + + with pytest.raises(KeyError, match="subarray/subsubarray"): + group["subarray/subsubarray"] + + +def test_group_get_with_default(store: Store, zarr_format: ZarrFormat) -> None: + group = Group.from_store(store, zarr_format=zarr_format) + + # default behavior + result = group.get("subgroup") + assert result is None + + # custom default + result = group.get("subgroup", 8) + assert result == 8 + + # now with a group + subgroup = group.require_group("subgroup") + subgroup.attrs["foo"] = "bar" + + result = group.get("subgroup", 8) + assert result.attrs["foo"] == "bar" + + +@pytest.mark.parametrize("consolidated", [True, False]) +def test_group_delitem(store: Store, zarr_format: ZarrFormat, consolidated: bool) -> None: + """ + Test the `Group.__delitem__` method. + """ + if not store.supports_deletes: + pytest.skip("store does not support deletes") + + group = Group.from_store(store, zarr_format=zarr_format) + subgroup = group.create_group(name="subgroup") + subarray = group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="uint8") + + if consolidated: + group = zarr.api.synchronous.consolidate_metadata(store=store, zarr_format=zarr_format) + object.__setattr__( + subgroup.metadata, "consolidated_metadata", ConsolidatedMetadata(metadata={}) + ) + + assert group["subgroup"] == subgroup + assert group["subarray"] == subarray + + del group["subgroup"] + with pytest.raises(KeyError): + group["subgroup"] + + del group["subarray"] + with pytest.raises(KeyError): + group["subarray"] + + +def test_group_iter(store: Store, zarr_format: ZarrFormat) -> None: + """ + Test the `Group.__iter__` method. + """ + + group = Group.from_store(store, zarr_format=zarr_format) + assert list(group) == [] + + +def test_group_len(store: Store, zarr_format: ZarrFormat) -> None: + """ + Test the `Group.__len__` method. + """ + + group = Group.from_store(store, zarr_format=zarr_format) + assert len(group) == 0 + + +def test_group_setitem(store: Store, zarr_format: ZarrFormat) -> None: + """ + Test the `Group.__setitem__` method. + """ + group = Group.from_store(store, zarr_format=zarr_format) + arr = np.ones((2, 4)) + group["key"] = arr + assert list(group.array_keys()) == ["key"] + assert group["key"].shape == (2, 4) + np.testing.assert_array_equal(group["key"][:], arr) + + if store.supports_deletes: + key = "key" + else: + # overwriting with another array requires deletes + # for stores that don't support this, we just use a new key + key = "key2" + + # overwrite with another array + arr = np.zeros((3, 5)) + group[key] = arr + assert key in list(group.array_keys()) + assert group[key].shape == (3, 5) + np.testing.assert_array_equal(group[key], arr) + + +def test_group_contains(store: Store, zarr_format: ZarrFormat) -> None: + """ + Test the `Group.__contains__` method + """ + group = Group.from_store(store, zarr_format=zarr_format) + assert "foo" not in group + _ = group.create_group(name="foo") + assert "foo" in group + + +@pytest.mark.parametrize("consolidate", [True, False]) +def test_group_child_iterators(store: Store, zarr_format: ZarrFormat, consolidate: bool): + group = Group.from_store(store, zarr_format=zarr_format) + expected_group_keys = ["g0", "g1"] + expected_group_values = [group.create_group(name=name) for name in expected_group_keys] + expected_groups = list(zip(expected_group_keys, expected_group_values, strict=False)) + + fill_value = 3 + dtype = UInt8() + + expected_group_values[0].create_group("subgroup") + expected_group_values[0].create_array( + "subarray", shape=(1,), dtype=dtype, fill_value=fill_value + ) + + expected_array_keys = ["a0", "a1"] + + expected_array_values = [ + group.create_array(name=name, shape=(1,), dtype=dtype, fill_value=fill_value) + for name in expected_array_keys + ] + expected_arrays = list(zip(expected_array_keys, expected_array_values, strict=False)) + + if consolidate: + group = zarr.consolidate_metadata(store) + if zarr_format == 2: + metadata = { + "subarray": { + "attributes": {}, + "dtype": unpack_dtype_json(dtype.to_json(zarr_format=zarr_format)), + "fill_value": fill_value, + "shape": (1,), + "chunks": (1,), + "order": "C", + "filters": None, + "compressor": Blosc(), + "zarr_format": zarr_format, + }, + "subgroup": { + "attributes": {}, + "consolidated_metadata": { + "metadata": {}, + "kind": "inline", + "must_understand": False, + }, + "node_type": "group", + "zarr_format": zarr_format, + }, + } + else: + metadata = { + "subarray": { + "attributes": {}, + "chunk_grid": { + "configuration": {"chunk_shape": (1,)}, + "name": "regular", + }, + "chunk_key_encoding": { + "configuration": {"separator": "/"}, + "name": "default", + }, + "codecs": ( + {"configuration": {"endian": "little"}, "name": "bytes"}, + {"configuration": {}, "name": "zstd"}, + ), + "data_type": unpack_dtype_json(dtype.to_json(zarr_format=zarr_format)), + "fill_value": fill_value, + "node_type": "array", + "shape": (1,), + "zarr_format": zarr_format, + }, + "subgroup": { + "attributes": {}, + "consolidated_metadata": { + "metadata": {}, + "kind": "inline", + "must_understand": False, + }, + "node_type": "group", + "zarr_format": zarr_format, + }, + } + + object.__setattr__( + expected_group_values[0].metadata, + "consolidated_metadata", + ConsolidatedMetadata.from_dict( + { + "kind": "inline", + "metadata": metadata, + "must_understand": False, + } + ), + ) + object.__setattr__( + expected_group_values[1].metadata, + "consolidated_metadata", + ConsolidatedMetadata(metadata={}), + ) + + result = sorted(group.groups(), key=operator.itemgetter(0)) + assert result == expected_groups + + assert sorted(group.groups(), key=operator.itemgetter(0)) == expected_groups + assert sorted(group.group_keys()) == expected_group_keys + assert sorted(group.group_values(), key=lambda x: x.name) == expected_group_values + + assert sorted(group.arrays(), key=operator.itemgetter(0)) == expected_arrays + assert sorted(group.array_keys()) == expected_array_keys + assert sorted(group.array_values(), key=lambda x: x.name) == expected_array_values + + +def test_group_update_attributes(store: Store, zarr_format: ZarrFormat) -> None: + """ + Test the behavior of `Group.update_attributes` + """ + attrs = {"foo": 100} + group = Group.from_store(store, zarr_format=zarr_format, attributes=attrs) + assert group.attrs == attrs + new_attrs = {"bar": 100} + new_group = group.update_attributes(new_attrs) + + updated_attrs = attrs.copy() + updated_attrs.update(new_attrs) + assert new_group.attrs == updated_attrs + + +async def test_group_update_attributes_async(store: Store, zarr_format: ZarrFormat) -> None: + """ + Test the behavior of `Group.update_attributes_async` + """ + attrs = {"foo": 100} + group = Group.from_store(store, zarr_format=zarr_format, attributes=attrs) + assert group.attrs == attrs + new_attrs = {"bar": 100} + new_group = await group.update_attributes_async(new_attrs) + assert new_group.attrs == new_attrs + + +@pytest.mark.parametrize("method", ["create_array", "array"]) +@pytest.mark.parametrize("name", ["a", "/a"]) +def test_group_create_array( + store: Store, + zarr_format: ZarrFormat, + overwrite: bool, + method: Literal["create_array", "array"], + name: str, +) -> None: + """ + Test `Group.from_store` + """ + group = Group.from_store(store, zarr_format=zarr_format) + shape = (10, 10) + dtype = "uint8" + data = np.arange(np.prod(shape)).reshape(shape).astype(dtype) + + if method == "create_array": + array = group.create_array(name=name, shape=shape, dtype=dtype) + array[:] = data + elif method == "array": + with pytest.warns(DeprecationWarning): + array = group.array(name=name, data=data, shape=shape, dtype=dtype) + else: + raise AssertionError + + if not overwrite: + if method == "create_array": + with pytest.raises(ContainsArrayError): + a = group.create_array(name=name, shape=shape, dtype=dtype) + a[:] = data + elif method == "array": + with pytest.raises(ContainsArrayError), pytest.warns(DeprecationWarning): + a = group.array(name=name, shape=shape, dtype=dtype) + a[:] = data + + assert array.path == normalize_path(name) + assert array.name == "/" + array.path + assert array.shape == shape + assert array.dtype == np.dtype(dtype) + assert np.array_equal(array[:], data) + + +def test_group_array_creation( + store: Store, + zarr_format: ZarrFormat, +): + group = Group.from_store(store, zarr_format=zarr_format) + shape = (10, 10) + empty_array = group.empty(name="empty", shape=shape) + assert isinstance(empty_array, Array) + assert empty_array.fill_value == 0 + assert empty_array.shape == shape + assert empty_array.store_path.store == store + assert empty_array.store_path.path == "empty" + + empty_like_array = group.empty_like(name="empty_like", data=empty_array) + assert isinstance(empty_like_array, Array) + assert empty_like_array.fill_value == 0 + assert empty_like_array.shape == shape + assert empty_like_array.store_path.store == store + + empty_array_bool = group.empty(name="empty_bool", shape=shape, dtype=np.dtype("bool")) + assert isinstance(empty_array_bool, Array) + assert not empty_array_bool.fill_value + assert empty_array_bool.shape == shape + assert empty_array_bool.store_path.store == store + + empty_like_array_bool = group.empty_like(name="empty_like_bool", data=empty_array_bool) + assert isinstance(empty_like_array_bool, Array) + assert not empty_like_array_bool.fill_value + assert empty_like_array_bool.shape == shape + assert empty_like_array_bool.store_path.store == store + + zeros_array = group.zeros(name="zeros", shape=shape) + assert isinstance(zeros_array, Array) + assert zeros_array.fill_value == 0 + assert zeros_array.shape == shape + assert zeros_array.store_path.store == store + + zeros_like_array = group.zeros_like(name="zeros_like", data=zeros_array) + assert isinstance(zeros_like_array, Array) + assert zeros_like_array.fill_value == 0 + assert zeros_like_array.shape == shape + assert zeros_like_array.store_path.store == store + + ones_array = group.ones(name="ones", shape=shape) + assert isinstance(ones_array, Array) + assert ones_array.fill_value == 1 + assert ones_array.shape == shape + assert ones_array.store_path.store == store + + ones_like_array = group.ones_like(name="ones_like", data=ones_array) + assert isinstance(ones_like_array, Array) + assert ones_like_array.fill_value == 1 + assert ones_like_array.shape == shape + assert ones_like_array.store_path.store == store + + full_array = group.full(name="full", shape=shape, fill_value=42) + assert isinstance(full_array, Array) + assert full_array.fill_value == 42 + assert full_array.shape == shape + assert full_array.store_path.store == store + + full_like_array = group.full_like(name="full_like", data=full_array, fill_value=43) + assert isinstance(full_like_array, Array) + assert full_like_array.fill_value == 43 + assert full_like_array.shape == shape + assert full_like_array.store_path.store == store + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.parametrize("overwrite", [True, False]) +@pytest.mark.parametrize("extant_node", ["array", "group"]) +def test_group_creation_existing_node( + store: Store, + zarr_format: ZarrFormat, + overwrite: bool, + extant_node: Literal["array", "group"], +) -> None: + """ + Check that an existing array or group is handled as expected during group creation. + """ + spath = StorePath(store) + group = Group.from_store(spath, zarr_format=zarr_format) + expected_exception: type[ContainsArrayError | ContainsGroupError] + attributes: dict[str, JSON] = {"old": True} + + if extant_node == "array": + expected_exception = ContainsArrayError + _ = group.create_array("extant", shape=(10,), dtype="uint8", attributes=attributes) + elif extant_node == "group": + expected_exception = ContainsGroupError + _ = group.create_group("extant", attributes=attributes) + else: + raise AssertionError + + new_attributes = {"new": True} + + if overwrite: + if not store.supports_deletes: + pytest.skip("store does not support deletes but overwrite is True") + node_new = Group.from_store( + spath / "extant", + attributes=new_attributes, + zarr_format=zarr_format, + overwrite=overwrite, + ) + assert node_new.attrs == new_attributes + else: + with pytest.raises(expected_exception): + node_new = Group.from_store( + spath / "extant", + attributes=new_attributes, + zarr_format=zarr_format, + overwrite=overwrite, + ) + + +async def test_asyncgroup_create( + store: Store, + overwrite: bool, + zarr_format: ZarrFormat, +) -> None: + """ + Test that `AsyncGroup.from_store` works as expected. + """ + spath = StorePath(store=store) + attributes = {"foo": 100} + agroup = await AsyncGroup.from_store( + store, + attributes=attributes, + overwrite=overwrite, + zarr_format=zarr_format, + ) + + assert agroup.metadata == GroupMetadata(zarr_format=zarr_format, attributes=attributes) + assert agroup.store_path == await make_store_path(store) + + if not overwrite: + with pytest.raises(ContainsGroupError): + agroup = await AsyncGroup.from_store( + spath, + attributes=attributes, + overwrite=overwrite, + zarr_format=zarr_format, + ) + # create an array at our target path + collision_name = "foo" + _ = await zarr.api.asynchronous.create_array( + spath / collision_name, shape=(10,), dtype="uint8", zarr_format=zarr_format + ) + with pytest.raises(ContainsArrayError): + _ = await AsyncGroup.from_store( + StorePath(store=store) / collision_name, + attributes=attributes, + overwrite=overwrite, + zarr_format=zarr_format, + ) + + +async def test_asyncgroup_attrs(store: Store, zarr_format: ZarrFormat) -> None: + attributes = {"foo": 100} + agroup = await AsyncGroup.from_store(store, zarr_format=zarr_format, attributes=attributes) + + assert agroup.attrs == agroup.metadata.attributes == attributes + + +async def test_asyncgroup_open( + store: Store, + zarr_format: ZarrFormat, +) -> None: + """ + Create an `AsyncGroup`, then ensure that we can open it using `AsyncGroup.open` + """ + attributes = {"foo": 100} + group_w = await AsyncGroup.from_store( + store=store, + attributes=attributes, + overwrite=False, + zarr_format=zarr_format, + ) + + group_r = await AsyncGroup.open(store=store, zarr_format=zarr_format) + + assert group_w.attrs == group_w.attrs == attributes + assert group_w == group_r + + +async def test_asyncgroup_open_wrong_format( + store: Store, + zarr_format: ZarrFormat, +) -> None: + _ = await AsyncGroup.from_store(store=store, overwrite=False, zarr_format=zarr_format) + zarr_format_wrong: ZarrFormat + # try opening with the wrong zarr format + if zarr_format == 3: + zarr_format_wrong = 2 + elif zarr_format == 2: + zarr_format_wrong = 3 + else: + raise AssertionError + + with pytest.raises(FileNotFoundError): + await AsyncGroup.open(store=store, zarr_format=zarr_format_wrong) + + +# todo: replace the dict[str, Any] type with something a bit more specific +# should this be async? +@pytest.mark.parametrize( + "data", + [ + {"zarr_format": 3, "node_type": "group", "attributes": {"foo": 100}}, + {"zarr_format": 2, "attributes": {"foo": 100}}, + ], +) +def test_asyncgroup_from_dict(store: Store, data: dict[str, Any]) -> None: + """ + Test that we can create an AsyncGroup from a dict + """ + path = "test" + store_path = StorePath(store=store, path=path) + group = AsyncGroup.from_dict(store_path, data=data) + + assert group.metadata.zarr_format == data["zarr_format"] + assert group.metadata.attributes == data["attributes"] + + +# todo: replace this with a declarative API where we model a full hierarchy + + +async def test_asyncgroup_getitem(store: Store, zarr_format: ZarrFormat) -> None: + """ + Create an `AsyncGroup`, then create members of that group, and ensure that we can access those + members via the `AsyncGroup.getitem` method. + """ + agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) + + array_name = "sub_array" + sub_array = await agroup.create_array(name=array_name, shape=(10,), dtype="uint8", chunks=(2,)) + assert await agroup.getitem(array_name) == sub_array + + sub_group_path = "sub_group" + sub_group = await agroup.create_group(sub_group_path, attributes={"foo": 100}) + assert await agroup.getitem(sub_group_path) == sub_group + + # check that asking for a nonexistent key raises KeyError + with pytest.raises(KeyError): + await agroup.getitem("foo") + + +async def test_asyncgroup_delitem(store: Store, zarr_format: ZarrFormat) -> None: + if not store.supports_deletes: + pytest.skip("store does not support deletes") + + agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) + array_name = "sub_array" + _ = await agroup.create_array( + name=array_name, + shape=(10,), + dtype="uint8", + chunks=(2,), + attributes={"foo": 100}, + ) + await agroup.delitem(array_name) + + # todo: clean up the code duplication here + if zarr_format == 2: + assert not await agroup.store_path.store.exists(array_name + "/" + ".zarray") + assert not await agroup.store_path.store.exists(array_name + "/" + ".zattrs") + elif zarr_format == 3: + assert not await agroup.store_path.store.exists(array_name + "/" + "zarr.json") + else: + raise AssertionError + + sub_group_path = "sub_group" + _ = await agroup.create_group(sub_group_path, attributes={"foo": 100}) + await agroup.delitem(sub_group_path) + if zarr_format == 2: + assert not await agroup.store_path.store.exists(array_name + "/" + ".zgroup") + assert not await agroup.store_path.store.exists(array_name + "/" + ".zattrs") + elif zarr_format == 3: + assert not await agroup.store_path.store.exists(array_name + "/" + "zarr.json") + else: + raise AssertionError + + +@pytest.mark.parametrize("name", ["a", "/a"]) +async def test_asyncgroup_create_group( + store: Store, + name: str, + zarr_format: ZarrFormat, +) -> None: + agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) + attributes = {"foo": 999} + subgroup = await agroup.create_group(name=name, attributes=attributes) + + assert isinstance(subgroup, AsyncGroup) + assert subgroup.path == normalize_path(name) + assert subgroup.name == "/" + subgroup.path + assert subgroup.attrs == attributes + assert subgroup.store_path.path == subgroup.path + assert subgroup.store_path.store == store + assert subgroup.metadata.zarr_format == zarr_format + + +async def test_asyncgroup_create_array( + store: Store, zarr_format: ZarrFormat, overwrite: bool +) -> None: + """ + Test that the AsyncGroup.create_array method works correctly. We ensure that array properties + specified in create_array are present on the resulting array. + """ + + agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) + + if not overwrite: + with pytest.raises(ContainsGroupError): + agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) + + shape = (10,) + dtype = "uint8" + chunk_shape = (4,) + attributes: dict[str, JSON] = {"foo": 100} + + sub_node_path = "sub_array" + subnode = await agroup.create_array( + name=sub_node_path, + shape=shape, + dtype=dtype, + chunks=chunk_shape, + attributes=attributes, + ) + assert isinstance(subnode, AsyncArray) + assert subnode.attrs == attributes + assert subnode.store_path.path == sub_node_path + assert subnode.store_path.store == store + assert subnode.shape == shape + assert subnode.dtype == dtype + # todo: fix the type annotation of array.metadata.chunk_grid so that we get some autocomplete + # here. + assert subnode.metadata.chunk_grid.chunk_shape == chunk_shape + assert subnode.metadata.zarr_format == zarr_format + + +async def test_asyncgroup_update_attributes(store: Store, zarr_format: ZarrFormat) -> None: + """ + Test that the AsyncGroup.update_attributes method works correctly. + """ + attributes_old = {"foo": 10} + attributes_new = {"baz": "new"} + agroup = await AsyncGroup.from_store( + store=store, zarr_format=zarr_format, attributes=attributes_old + ) + + agroup_new_attributes = await agroup.update_attributes(attributes_new) + attributes_updated = attributes_old.copy() + attributes_updated.update(attributes_new) + assert agroup_new_attributes.attrs == attributes_updated + + +@pytest.mark.parametrize("store", ["local"], indirect=["store"]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +async def test_serializable_async_group(store: LocalStore, zarr_format: ZarrFormat) -> None: + expected = await AsyncGroup.from_store( + store=store, attributes={"foo": 999}, zarr_format=zarr_format + ) + p = pickle.dumps(expected) + actual = pickle.loads(p) + assert actual == expected + + +@pytest.mark.parametrize("store", ["local"], indirect=["store"]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +def test_serializable_sync_group(store: LocalStore, zarr_format: ZarrFormat) -> None: + expected = Group.from_store(store=store, attributes={"foo": 999}, zarr_format=zarr_format) + p = pickle.dumps(expected) + actual = pickle.loads(p) + assert actual == expected + + +@pytest.mark.parametrize("consolidated_metadata", [True, False]) +async def test_group_members_async(store: Store, consolidated_metadata: bool) -> None: + group = await AsyncGroup.from_store( + store=store, + ) + a0 = await group.create_array("a0", shape=(1,), dtype="uint8") + g0 = await group.create_group("g0") + a1 = await g0.create_array("a1", shape=(1,), dtype="uint8") + g1 = await g0.create_group("g1") + a2 = await g1.create_array("a2", shape=(1,), dtype="uint8") + g2 = await g1.create_group("g2") + + # immediate children + children = sorted([x async for x in group.members()], key=operator.itemgetter(0)) + assert children == [ + ("a0", a0), + ("g0", g0), + ] + + nmembers = await group.nmembers() + assert nmembers == 2 + + # partial + children = sorted([x async for x in group.members(max_depth=1)], key=operator.itemgetter(0)) + expected = [ + ("a0", a0), + ("g0", g0), + ("g0/a1", a1), + ("g0/g1", g1), + ] + assert children == expected + nmembers = await group.nmembers(max_depth=1) + assert nmembers == 4 + + # all children + all_children = sorted( + [x async for x in group.members(max_depth=None)], key=operator.itemgetter(0) + ) + expected = [ + ("a0", a0), + ("g0", g0), + ("g0/a1", a1), + ("g0/g1", g1), + ("g0/g1/a2", a2), + ("g0/g1/g2", g2), + ] + assert all_children == expected + + if consolidated_metadata: + await zarr.api.asynchronous.consolidate_metadata(store=store) + group = await zarr.api.asynchronous.open_group(store=store) + + nmembers = await group.nmembers(max_depth=None) + assert nmembers == 6 + + with pytest.raises(ValueError, match="max_depth"): + [x async for x in group.members(max_depth=-1)] + + if consolidated_metadata: + # test for mixed known and unknown metadata. + # For now, we trust the consolidated metadata. + object.__setattr__( + group.metadata.consolidated_metadata.metadata["g0"].consolidated_metadata.metadata[ + "g1" + ], + "consolidated_metadata", + None, + ) + all_children = sorted( + [x async for x in group.members(max_depth=None)], key=operator.itemgetter(0) + ) + assert len(all_children) == 4 + nmembers = await group.nmembers(max_depth=None) + assert nmembers == 4 + + +async def test_require_group(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) + + # create foo group + _ = await root.create_group("foo", attributes={"foo": 100}) + + # test that we can get the group using require_group + foo_group = await root.require_group("foo") + assert foo_group.attrs == {"foo": 100} + + # test that we can get the group using require_group and overwrite=True + if store.supports_deletes: + foo_group = await root.require_group("foo", overwrite=True) + assert foo_group.attrs == {} + + _ = await foo_group.create_array( + "bar", shape=(10,), dtype="uint8", chunks=(2,), attributes={"foo": 100} + ) + + # test that overwriting a group w/ children fails + # TODO: figure out why ensure_no_existing_node is not catching the foo.bar array + # + # with pytest.raises(ContainsArrayError): + # await root.require_group("foo", overwrite=True) + + # test that requiring a group where an array is fails + with pytest.raises(TypeError): + await foo_group.require_group("bar") + + +async def test_require_groups(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) + # create foo group + _ = await root.create_group("foo", attributes={"foo": 100}) + # create bar group + _ = await root.create_group("bar", attributes={"bar": 200}) + + foo_group, bar_group = await root.require_groups("foo", "bar") + assert foo_group.attrs == {"foo": 100} + assert bar_group.attrs == {"bar": 200} + + # get a mix of existing and new groups + foo_group, spam_group = await root.require_groups("foo", "spam") + assert foo_group.attrs == {"foo": 100} + assert spam_group.attrs == {} + + # no names + no_group = await root.require_groups() + assert no_group == () + + +def test_create_dataset_with_data(store: Store, zarr_format: ZarrFormat) -> None: + """Check that deprecated create_dataset method allows input data. + + See https://github.com/zarr-developers/zarr-python/issues/2631. + """ + root = Group.from_store(store=store, zarr_format=zarr_format) + arr = np.random.random((5, 5)) + with pytest.warns(DeprecationWarning): + data = root.create_dataset("random", data=arr, shape=arr.shape) + np.testing.assert_array_equal(np.asarray(data), arr) + + +async def test_create_dataset(store: Store, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) + with pytest.warns(DeprecationWarning): + foo = await root.create_dataset("foo", shape=(10,), dtype="uint8") + assert foo.shape == (10,) + + with pytest.raises(ContainsArrayError), pytest.warns(DeprecationWarning): + await root.create_dataset("foo", shape=(100,), dtype="int8") + + _ = await root.create_group("bar") + with pytest.raises(ContainsGroupError), pytest.warns(DeprecationWarning): + await root.create_dataset("bar", shape=(100,), dtype="int8") + + +async def test_require_array(store: Store, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) + foo1 = await root.require_array("foo", shape=(10,), dtype="i8", attributes={"foo": 101}) + assert foo1.attrs == {"foo": 101} + foo2 = await root.require_array("foo", shape=(10,), dtype="i8") + assert foo2.attrs == {"foo": 101} + + # exact = False + _ = await root.require_array("foo", shape=10, dtype="f8") + + # errors w/ exact True + with pytest.raises(TypeError, match="Incompatible dtype"): + await root.require_array("foo", shape=(10,), dtype="f8", exact=True) + + with pytest.raises(TypeError, match="Incompatible shape"): + await root.require_array("foo", shape=(100, 100), dtype="i8") + + with pytest.raises(TypeError, match="Incompatible dtype"): + await root.require_array("foo", shape=(10,), dtype="f4") + + _ = await root.create_group("bar") + with pytest.raises(TypeError, match="Incompatible object"): + await root.require_array("bar", shape=(10,), dtype="int8") + + +@pytest.mark.parametrize("consolidate", [True, False]) +async def test_members_name(store: Store, consolidate: bool, zarr_format: ZarrFormat): + group = Group.from_store(store=store, zarr_format=zarr_format) + a = group.create_group(name="a") + a.create_array("array", shape=(1,), dtype="uint8") + b = a.create_group(name="b") + b.create_array("array", shape=(1,), dtype="uint8") + + if consolidate: + group = zarr.api.synchronous.consolidate_metadata(store) + + result = group["a"]["b"] + assert result.name == "/a/b" + + paths = sorted(x.name for _, x in group.members(max_depth=None)) + expected = ["/a", "/a/array", "/a/b", "/a/b/array"] + assert paths == expected + + # regression test for https://github.com/zarr-developers/zarr-python/pull/2356 + g = zarr.open_group(store, use_consolidated=False) + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert list(g) + + +async def test_open_mutable_mapping(): + group = await zarr.api.asynchronous.open_group( + store={}, + ) + assert isinstance(group.store_path.store, MemoryStore) + + +def test_open_mutable_mapping_sync(): + group = zarr.open_group( + store={}, + ) + assert isinstance(group.store_path.store, MemoryStore) + + +class TestConsolidated: + async def test_group_getitem_consolidated(self, store: Store) -> None: + root = await AsyncGroup.from_store(store=store) + # Set up the test structure with + # / + # g0/ # group /g0 + # g1/ # group /g0/g1 + # g2/ # group /g0/g1/g2 + # x1/ # group /x0 + # x2/ # group /x0/x1 + # x3/ # group /x0/x1/x2 + + g0 = await root.create_group("g0") + g1 = await g0.create_group("g1") + await g1.create_group("g2") + + x0 = await root.create_group("x0") + x1 = await x0.create_group("x1") + await x1.create_group("x2") + + await zarr.api.asynchronous.consolidate_metadata(store) + + # On disk, we've consolidated all the metadata in the root zarr.json + group = await zarr.api.asynchronous.open(store=store) + rg0 = await group.getitem("g0") + + expected = ConsolidatedMetadata( + metadata={ + "g1": GroupMetadata( + attributes={}, + zarr_format=3, + consolidated_metadata=ConsolidatedMetadata( + metadata={ + "g2": GroupMetadata( + attributes={}, + zarr_format=3, + consolidated_metadata=ConsolidatedMetadata(metadata={}), + ) + } + ), + ), + } + ) + assert rg0.metadata.consolidated_metadata == expected + + rg1 = await rg0.getitem("g1") + assert rg1.metadata.consolidated_metadata == expected.metadata["g1"].consolidated_metadata + + rg2 = await rg1.getitem("g2") + assert rg2.metadata.consolidated_metadata == ConsolidatedMetadata(metadata={}) + + async def test_group_delitem_consolidated(self, store: Store) -> None: + if isinstance(store, ZipStore): + raise pytest.skip("Not implemented") + + root = await AsyncGroup.from_store(store=store) + # Set up the test structure with + # / + # g0/ # group /g0 + # g1/ # group /g0/g1 + # g2/ # group /g0/g1/g2 + # data # array + # x1/ # group /x0 + # x2/ # group /x0/x1 + # x3/ # group /x0/x1/x2 + # data # array + + g0 = await root.create_group("g0") + g1 = await g0.create_group("g1") + g2 = await g1.create_group("g2") + await g2.create_array("data", shape=(1,), dtype="uint8") + + x0 = await root.create_group("x0") + x1 = await x0.create_group("x1") + x2 = await x1.create_group("x2") + await x2.create_array("data", shape=(1,), dtype="uint8") + + await zarr.api.asynchronous.consolidate_metadata(store) + + group = await zarr.api.asynchronous.open_consolidated(store=store) + assert len(group.metadata.consolidated_metadata.metadata) == 2 + assert "g0" in group.metadata.consolidated_metadata.metadata + + await group.delitem("g0") + assert len(group.metadata.consolidated_metadata.metadata) == 1 + assert "g0" not in group.metadata.consolidated_metadata.metadata + + def test_open_consolidated_raises(self, store: Store) -> None: + if isinstance(store, ZipStore): + raise pytest.skip("Not implemented") + + root = Group.from_store(store=store) + + # fine to be missing by default + zarr.open_group(store=store) + + with pytest.raises(ValueError, match="Consolidated metadata requested."): + zarr.open_group(store=store, use_consolidated=True) + + # Now create consolidated metadata... + root.create_group("g0") + zarr.consolidate_metadata(store) + + # and explicitly ignore it. + group = zarr.open_group(store=store, use_consolidated=False) + assert group.metadata.consolidated_metadata is None + + async def test_open_consolidated_raises_async(self, store: Store) -> None: + if isinstance(store, ZipStore): + raise pytest.skip("Not implemented") + + root = await AsyncGroup.from_store(store=store) + + # fine to be missing by default + await zarr.api.asynchronous.open_group(store=store) + + with pytest.raises(ValueError, match="Consolidated metadata requested."): + await zarr.api.asynchronous.open_group(store=store, use_consolidated=True) + + # Now create consolidated metadata... + await root.create_group("g0") + await zarr.api.asynchronous.consolidate_metadata(store) + + # and explicitly ignore it. + group = await zarr.api.asynchronous.open_group(store=store, use_consolidated=False) + assert group.metadata.consolidated_metadata is None + + +class TestGroupMetadata: + def test_from_dict_extra_fields(self): + data = { + "attributes": {"key": "value"}, + "_nczarr_superblock": {"version": "2.0.0"}, + "zarr_format": 2, + } + result = GroupMetadata.from_dict(data) + expected = GroupMetadata(attributes={"key": "value"}, zarr_format=2) + assert result == expected + + +class TestInfo: + def test_info(self): + store = zarr.storage.MemoryStore() + A = zarr.group(store=store, path="A") + B = A.create_group(name="B") + + B.create_array(name="x", shape=(1,), dtype="uint8") + B.create_array(name="y", shape=(2,), dtype="uint8") + + result = A.info + expected = GroupInfo( + _name="A", + _read_only=False, + _store_type="MemoryStore", + _zarr_format=3, + ) + assert result == expected + + result = A.info_complete() + expected = GroupInfo( + _name="A", + _read_only=False, + _store_type="MemoryStore", + _zarr_format=3, + _count_members=3, + _count_arrays=2, + _count_groups=1, + ) + assert result == expected + + +def test_update_attrs() -> None: + # regression test for https://github.com/zarr-developers/zarr-python/issues/2328 + root = Group.from_store( + MemoryStore(), + ) + root.attrs["foo"] = "bar" + assert root.attrs["foo"] == "bar" + + +@pytest.mark.parametrize("method", ["empty", "zeros", "ones", "full"]) +def test_group_deprecated_positional_args(method: str) -> None: + if method == "full": + kwargs = {"fill_value": 0} + else: + kwargs = {} + + root = zarr.group() + with pytest.warns(FutureWarning, match=r"Pass name=.* as keyword args."): + arr = getattr(root, method)("foo", shape=1, **kwargs) + assert arr.shape == (1,) + + method += "_like" + data = np.ones(1) + + with pytest.warns(FutureWarning, match=r"Pass name=.*, data=.* as keyword args."): + arr = getattr(root, method)("foo_like", data, **kwargs) + assert arr.shape == data.shape + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_delitem_removes_children(store: Store, zarr_format: ZarrFormat) -> None: + # https://github.com/zarr-developers/zarr-python/issues/2191 + g1 = zarr.group(store=store, zarr_format=zarr_format) + g1.create_group("0") + g1.create_group("0/0") + arr = g1.create_array("0/0/0", shape=(1,), dtype="uint8") + arr[:] = 1 + del g1["0"] + with pytest.raises(KeyError): + g1["0/0"] + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_nodes( + impl: Literal["async", "sync"], store: Store, zarr_format: ZarrFormat +) -> None: + """ + Ensure that ``create_nodes`` can create a zarr hierarchy from a model of that + hierarchy in dict form. Note that this creates an incomplete Zarr hierarchy. + """ + node_spec = { + "group": GroupMetadata(attributes={"foo": 10}), + "group/array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), + "group/array_1": meta_from_array(np.arange(4), zarr_format=zarr_format), + "group/subgroup/array_0": meta_from_array(np.arange(4), zarr_format=zarr_format), + "group/subgroup/array_1": meta_from_array(np.arange(5), zarr_format=zarr_format), + } + if impl == "sync": + observed_nodes = dict(sync_group.create_nodes(store=store, nodes=node_spec)) + elif impl == "async": + observed_nodes = dict(await _collect_aiterator(create_nodes(store=store, nodes=node_spec))) + else: + raise ValueError(f"Invalid impl: {impl}") + + assert node_spec == {k: v.metadata for k, v in observed_nodes.items()} + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_create_nodes_concurrency_limit(store: MemoryStore) -> None: + """ + Test that the execution time of create_nodes can be constrained by the async concurrency + configuration setting. + """ + set_latency = 0.02 + num_groups = 10 + groups = {str(idx): GroupMetadata() for idx in range(num_groups)} + + latency_store = LatencyStore(store, set_latency=set_latency) + + # check how long it takes to iterate over the groups + # if create_nodes is sensitive to IO latency, + # this should take (num_groups * get_latency) seconds + # otherwise, it should take only marginally more than get_latency seconds + with zarr_config.set({"async.concurrency": 1}): + start = time.time() + _ = tuple(sync_group.create_nodes(store=latency_store, nodes=groups)) + elapsed = time.time() - start + assert elapsed > num_groups * set_latency + + +@pytest.mark.parametrize( + ("a_func", "b_func"), + [ + (zarr.core.group.AsyncGroup.create_hierarchy, zarr.core.group.Group.create_hierarchy), + (zarr.core.group.create_hierarchy, zarr.core.sync_group.create_hierarchy), + (zarr.core.group.create_nodes, zarr.core.sync_group.create_nodes), + (zarr.core.group.create_rooted_hierarchy, zarr.core.sync_group.create_rooted_hierarchy), + (zarr.core.group.get_node, zarr.core.sync_group.get_node), + ], +) +def test_consistent_signatures( + a_func: Callable[[object], object], b_func: Callable[[object], object] +) -> None: + """ + Ensure that pairs of functions have consistent signatures + """ + base_sig = inspect.signature(a_func) + test_sig = inspect.signature(b_func) + assert test_sig.parameters == base_sig.parameters + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("overwrite", [True, False]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy( + impl: Literal["async", "sync"], store: Store, overwrite: bool, zarr_format: ZarrFormat +) -> None: + """ + Test that ``create_hierarchy`` can create a complete Zarr hierarchy, even if the input describes + an incomplete one. + """ + + hierarchy_spec = { + "group": GroupMetadata(attributes={"path": "group"}, zarr_format=zarr_format), + "group/array_0": meta_from_array( + np.arange(3), attributes={"path": "group/array_0"}, zarr_format=zarr_format + ), + "group/subgroup/array_0": meta_from_array( + np.arange(4), attributes={"path": "group/subgroup/array_0"}, zarr_format=zarr_format + ), + } + pre_existing_nodes = { + "group/extra": GroupMetadata(zarr_format=zarr_format, attributes={"path": "group/extra"}), + "": GroupMetadata(zarr_format=zarr_format, attributes={"name": "root"}), + } + # we expect create_hierarchy to insert a group that was missing from the hierarchy spec + expected_meta = hierarchy_spec | {"group/subgroup": GroupMetadata(zarr_format=zarr_format)} + + # initialize the group with some nodes + _ = dict(sync_group.create_nodes(store=store, nodes=pre_existing_nodes)) + + if impl == "sync": + created = dict( + sync_group.create_hierarchy(store=store, nodes=hierarchy_spec, overwrite=overwrite) + ) + elif impl == "async": + created = { + k: v + async for k, v in create_hierarchy( + store=store, nodes=hierarchy_spec, overwrite=overwrite + ) + } + else: + raise ValueError(f"Invalid impl: {impl}") + if not overwrite: + extra_group = sync_group.get_node(store=store, path="group/extra", zarr_format=zarr_format) + assert extra_group.metadata.attributes == {"path": "group/extra"} + else: + with pytest.raises(FileNotFoundError): + await get_node(store=store, path="group/extra", zarr_format=zarr_format) + assert expected_meta == {k: v.metadata for k, v in created.items()} + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("extant_node", ["array", "group"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy_existing_nodes( + impl: Literal["async", "sync"], + store: Store, + extant_node: Literal["array", "group"], + zarr_format: ZarrFormat, +) -> None: + """ + Test that create_hierarchy with overwrite = False will not overwrite an existing array or group, + and raises an exception instead. + """ + extant_node_path = "node" + + if extant_node == "array": + extant_metadata = meta_from_array( + np.zeros(4), zarr_format=zarr_format, attributes={"extant": True} + ) + new_metadata = meta_from_array(np.zeros(4), zarr_format=zarr_format) + err_cls = ContainsArrayError + else: + extant_metadata = GroupMetadata(zarr_format=zarr_format, attributes={"extant": True}) + new_metadata = GroupMetadata(zarr_format=zarr_format) + err_cls = ContainsGroupError + + # write the extant metadata + tuple(sync_group.create_nodes(store=store, nodes={extant_node_path: extant_metadata})) + + msg = f"{extant_node} exists in store {store!r} at path {extant_node_path!r}." + # ensure that we cannot invoke create_hierarchy with overwrite=False here + if impl == "sync": + with pytest.raises(err_cls, match=re.escape(msg)): + tuple( + sync_group.create_hierarchy( + store=store, nodes={"node": new_metadata}, overwrite=False + ) + ) + elif impl == "async": + with pytest.raises(err_cls, match=re.escape(msg)): + tuple( + [ + x + async for x in create_hierarchy( + store=store, nodes={"node": new_metadata}, overwrite=False + ) + ] + ) + else: + raise ValueError(f"Invalid impl: {impl}") + + # ensure that the extant metadata was not overwritten + assert ( + await get_node(store=store, path=extant_node_path, zarr_format=zarr_format) + ).metadata.attributes == {"extant": True} + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("overwrite", [True, False]) +@pytest.mark.parametrize("group_path", ["", "foo"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_group_create_hierarchy( + store: Store, + zarr_format: ZarrFormat, + overwrite: bool, + group_path: str, + impl: Literal["async", "sync"], +) -> None: + """ + Test that the Group.create_hierarchy method creates specified nodes and returns them in a dict. + Also test that off-target nodes are not deleted, and that the root group is not deleted + """ + root_attrs = {"root": True} + g = sync_group.create_rooted_hierarchy( + store=store, + nodes={group_path: GroupMetadata(zarr_format=zarr_format, attributes=root_attrs)}, + ) + node_spec = { + "a": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), + "a/b": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a/b"}), + "a/b/c": meta_from_array( + np.zeros(5), zarr_format=zarr_format, attributes={"name": "a/b/c"} + ), + } + # This node should be kept if overwrite is True + extant_spec = {"b": GroupMetadata(zarr_format=zarr_format, attributes={"name": "b"})} + if impl == "async": + extant_created = dict( + await _collect_aiterator(g._async_group.create_hierarchy(extant_spec, overwrite=False)) + ) + nodes_created = dict( + await _collect_aiterator( + g._async_group.create_hierarchy(node_spec, overwrite=overwrite) + ) + ) + elif impl == "sync": + extant_created = dict(g.create_hierarchy(extant_spec, overwrite=False)) + nodes_created = dict(g.create_hierarchy(node_spec, overwrite=overwrite)) + + all_members = dict(g.members(max_depth=None)) + for k, v in node_spec.items(): + assert all_members[k].metadata == v == nodes_created[k].metadata + + # if overwrite is True, the extant nodes should be erased + for k, v in extant_spec.items(): + if overwrite: + assert k in all_members + else: + assert all_members[k].metadata == v == extant_created[k].metadata + # ensure that we left the root group as-is + assert ( + sync_group.get_node(store=store, path=group_path, zarr_format=zarr_format).attrs.asdict() + == root_attrs + ) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("overwrite", [True, False]) +def test_group_create_hierarchy_no_root( + store: Store, zarr_format: ZarrFormat, overwrite: bool +) -> None: + """ + Test that the Group.create_hierarchy method will error if the dict provided contains a root. + """ + g = Group.from_store(store, zarr_format=zarr_format) + tree = { + "": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), + } + with pytest.raises( + ValueError, match="It is an error to use this method to create a root node. " + ): + _ = dict(g.create_hierarchy(tree, overwrite=overwrite)) + + +class TestParseHierarchyDict: + """ + Tests for the function that parses dicts of str : Metadata pairs, ensuring that the output models a + valid Zarr hierarchy + """ + + @staticmethod + def test_normed_keys() -> None: + """ + Test that keys get normalized properly + """ + + nodes = { + "a": GroupMetadata(), + "/b": GroupMetadata(), + "": GroupMetadata(), + "/a//c////": GroupMetadata(), + } + observed = _parse_hierarchy_dict(data=nodes) + expected = {normalize_path(k): v for k, v in nodes.items()} + assert observed == expected + + @staticmethod + def test_empty() -> None: + """ + Test that an empty dict passes through + """ + assert _parse_hierarchy_dict(data={}) == {} + + @staticmethod + def test_implicit_groups() -> None: + """ + Test that implicit groups were added as needed. + """ + requested = {"a/b/c": GroupMetadata()} + expected = requested | { + "": ImplicitGroupMarker(), + "a": ImplicitGroupMarker(), + "a/b": ImplicitGroupMarker(), + } + observed = _parse_hierarchy_dict(data=requested) + assert observed == expected + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_group_create_hierarchy_invalid_mixed_zarr_format( + store: Store, zarr_format: ZarrFormat +) -> None: + """ + Test that ``Group.create_hierarchy`` will raise an error if the zarr_format of the nodes is + different from the parent group. + """ + other_format = 2 if zarr_format == 3 else 3 + g = Group.from_store(store, zarr_format=other_format) + tree = { + "a": GroupMetadata(zarr_format=zarr_format, attributes={"name": "a"}), + "a/b": meta_from_array(np.zeros(5), zarr_format=zarr_format, attributes={"name": "a/c"}), + } + + msg = "The zarr_format of the nodes must be the same as the parent group." + with pytest.raises(ValueError, match=msg): + _ = tuple(g.create_hierarchy(tree)) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("defect", ["array/array", "array/group"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy_invalid_nested( + impl: Literal["async", "sync"], store: Store, defect: tuple[str, str], zarr_format: ZarrFormat +) -> None: + """ + Test that create_hierarchy will not create a Zarr array that contains a Zarr group + or Zarr array. + """ + if defect == "array/array": + hierarchy_spec = { + "array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), + "array_0/subarray": meta_from_array(np.arange(4), zarr_format=zarr_format), + } + elif defect == "array/group": + hierarchy_spec = { + "array_0": meta_from_array(np.arange(3), zarr_format=zarr_format), + "array_0/subgroup": GroupMetadata(attributes={"foo": 10}, zarr_format=zarr_format), + } + + msg = "Only Zarr groups can contain other nodes." + if impl == "sync": + with pytest.raises(ValueError, match=msg): + tuple(sync_group.create_hierarchy(store=store, nodes=hierarchy_spec)) + elif impl == "async": + with pytest.raises(ValueError, match=msg): + await _collect_aiterator(create_hierarchy(store=store, nodes=hierarchy_spec)) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_hierarchy_invalid_mixed_format( + impl: Literal["async", "sync"], store: Store +) -> None: + """ + Test that create_hierarchy will not create a Zarr group that contains a both Zarr v2 and + Zarr v3 nodes. + """ + msg = ( + "Got data with both Zarr v2 and Zarr v3 nodes, which is invalid. " + "The following keys map to Zarr v2 nodes: ['v2']. " + "The following keys map to Zarr v3 nodes: ['v3']." + "Ensure that all nodes have the same Zarr format." + ) + nodes = { + "v2": GroupMetadata(zarr_format=2), + "v3": GroupMetadata(zarr_format=3), + } + if impl == "sync": + with pytest.raises(ValueError, match=re.escape(msg)): + tuple( + sync_group.create_hierarchy( + store=store, + nodes=nodes, + ) + ) + elif impl == "async": + with pytest.raises(ValueError, match=re.escape(msg)): + await _collect_aiterator( + create_hierarchy( + store=store, + nodes=nodes, + ) + ) + else: + raise ValueError(f"Invalid impl: {impl}") + + +@pytest.mark.parametrize("store", ["memory", "local"], indirect=True) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.parametrize("root_key", ["", "root"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_rooted_hierarchy_group( + impl: Literal["async", "sync"], store: Store, zarr_format, root_key: str +) -> None: + """ + Test that the _create_rooted_hierarchy can create a group. + """ + root_meta = {root_key: GroupMetadata(zarr_format=zarr_format, attributes={"path": root_key})} + group_names = ["a", "a/b"] + array_names = ["a/b/c", "a/b/d"] + + # just to ensure that we don't use the same name twice in tests + assert set(group_names) & set(array_names) == set() + + groups_expected_meta = { + _join_paths([root_key, node_name]): GroupMetadata( + zarr_format=zarr_format, attributes={"path": node_name} + ) + for node_name in group_names + } + + arrays_expected_meta = { + _join_paths([root_key, node_name]): meta_from_array(np.zeros(4), zarr_format=zarr_format) + for node_name in array_names + } + + nodes_create = root_meta | groups_expected_meta | arrays_expected_meta + if impl == "sync": + g = sync_group.create_rooted_hierarchy(store=store, nodes=nodes_create) + assert isinstance(g, Group) + members = g.members(max_depth=None) + elif impl == "async": + g = await create_rooted_hierarchy(store=store, nodes=nodes_create) + assert isinstance(g, AsyncGroup) + members = await _collect_aiterator(g.members(max_depth=None)) + else: + raise ValueError(f"Unknown implementation: {impl}") + + assert g.metadata.attributes == {"path": root_key} + + members_observed_meta = {k: v.metadata for k, v in members} + members_expected_meta_relative = { + k.removeprefix(root_key).lstrip("/"): v + for k, v in (groups_expected_meta | arrays_expected_meta).items() + } + assert members_observed_meta == members_expected_meta_relative + + +@pytest.mark.parametrize("store", ["memory", "local"], indirect=True) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.parametrize("root_key", ["", "root"]) +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_rooted_hierarchy_array( + impl: Literal["async", "sync"], store: Store, zarr_format, root_key: str +) -> None: + """ + Test that _create_rooted_hierarchy can create an array. + """ + + root_meta = { + root_key: meta_from_array( + np.arange(3), zarr_format=zarr_format, attributes={"path": root_key} + ) + } + nodes_create = root_meta + + if impl == "sync": + a = sync_group.create_rooted_hierarchy(store=store, nodes=nodes_create, overwrite=True) + assert isinstance(a, Array) + elif impl == "async": + a = await create_rooted_hierarchy(store=store, nodes=nodes_create, overwrite=True) + assert isinstance(a, AsyncArray) + else: + raise ValueError(f"Invalid impl: {impl}") + assert a.metadata.attributes == {"path": root_key} + + +@pytest.mark.parametrize("impl", ["async", "sync"]) +async def test_create_rooted_hierarchy_invalid(impl: Literal["async", "sync"]) -> None: + """ + Ensure _create_rooted_hierarchy will raise a ValueError if the input does not contain + a root node. + """ + zarr_format = 3 + nodes = { + "a": GroupMetadata(zarr_format=zarr_format), + "b": GroupMetadata(zarr_format=zarr_format), + } + msg = "The input does not specify a root node. " + if impl == "sync": + with pytest.raises(ValueError, match=msg): + sync_group.create_rooted_hierarchy(store=store, nodes=nodes) + elif impl == "async": + with pytest.raises(ValueError, match=msg): + await create_rooted_hierarchy(store=store, nodes=nodes) + else: + raise ValueError(f"Invalid impl: {impl}") + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_group_members_performance(store: Store) -> None: + """ + Test that the execution time of Group.members is less than the number of members times the + latency for accessing each member. + """ + get_latency = 0.1 + + # use the input store to create some groups + group_create = zarr.group(store=store) + num_groups = 10 + + # Create some groups + for i in range(num_groups): + group_create.create_group(f"group{i}") + + latency_store = LatencyStore(store, get_latency=get_latency) + # create a group with some latency on get operations + group_read = zarr.group(store=latency_store) + + # check how long it takes to iterate over the groups + # if .members is sensitive to IO latency, + # this should take (num_groups * get_latency) seconds + # otherwise, it should take only marginally more than get_latency seconds + start = time.time() + _ = group_read.members() + elapsed = time.time() - start + + assert elapsed < (num_groups * get_latency) + + +@pytest.mark.parametrize("store", ["memory"], indirect=True) +def test_group_members_concurrency_limit(store: MemoryStore) -> None: + """ + Test that the execution time of Group.members can be constrained by the async concurrency + configuration setting. + """ + get_latency = 0.02 + + # use the input store to create some groups + group_create = zarr.group(store=store) + num_groups = 10 + + # Create some groups + for i in range(num_groups): + group_create.create_group(f"group{i}") + + latency_store = LatencyStore(store, get_latency=get_latency) + # create a group with some latency on get operations + group_read = zarr.group(store=latency_store) + + # check how long it takes to iterate over the groups + # if .members is sensitive to IO latency, + # this should take (num_groups * get_latency) seconds + # otherwise, it should take only marginally more than get_latency seconds + with zarr_config.set({"async.concurrency": 1}): + start = time.time() + _ = group_read.members() + elapsed = time.time() - start + + assert elapsed > num_groups * get_latency + + +@pytest.mark.parametrize("option", ["array", "group", "invalid"]) +def test_build_metadata_v3(option: Literal["array", "group", "invalid"]) -> None: + """ + Test that _build_metadata_v3 returns the correct metadata for a v3 array or group + """ + match option: + case "array": + metadata_dict = meta_from_array(np.arange(10), zarr_format=3).to_dict() + assert _build_metadata_v3(metadata_dict) == ArrayV3Metadata.from_dict(metadata_dict) + case "group": + metadata_dict = GroupMetadata(attributes={"foo": 10}, zarr_format=3).to_dict() + assert _build_metadata_v3(metadata_dict) == GroupMetadata.from_dict(metadata_dict) + case "invalid": + metadata_dict = GroupMetadata(zarr_format=3).to_dict() + metadata_dict.pop("node_type") + # TODO: fix the error message + msg = "Invalid value for 'node_type'. Expected 'array or group'. Got 'nothing (the key is missing)'." + with pytest.raises(MetadataValidationError, match=re.escape(msg)): + _build_metadata_v3(metadata_dict) + + +@pytest.mark.parametrize("roots", [("",), ("a", "b")]) +def test_get_roots(roots: tuple[str, ...]): + root_nodes = {k: GroupMetadata(attributes={"name": k}) for k in roots} + child_nodes = { + _join_paths([k, "foo"]): GroupMetadata(attributes={"name": _join_paths([k, "foo"])}) + for k in roots + } + data = root_nodes | child_nodes + assert set(_get_roots(data)) == set(roots) diff --git a/tests/test_indexing.py b/tests/test_indexing.py new file mode 100644 index 0000000000..b1707c88a3 --- /dev/null +++ b/tests/test_indexing.py @@ -0,0 +1,1996 @@ +from __future__ import annotations + +import itertools +from collections import Counter +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +import numpy as np +import numpy.typing as npt +import pytest +from numpy.testing import assert_array_equal + +import zarr +from zarr import Array +from zarr.core.buffer import default_buffer_prototype +from zarr.core.indexing import ( + BasicSelection, + CoordinateSelection, + OrthogonalSelection, + Selection, + _iter_grid, + make_slice_selection, + normalize_integer_selection, + oindex, + oindex_set, + replace_ellipsis, +) +from zarr.registry import get_ndbuffer_class +from zarr.storage import MemoryStore, StorePath + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from zarr.core.buffer import BufferPrototype + from zarr.core.buffer.core import Buffer + from zarr.core.common import ChunkCoords + + +@pytest.fixture +async def store() -> AsyncGenerator[StorePath]: + return StorePath(await MemoryStore.open()) + + +def zarr_array_from_numpy_array( + store: StorePath, + a: npt.NDArray[Any], + chunk_shape: ChunkCoords | None = None, +) -> zarr.Array: + z = zarr.create_array( + store=store / str(uuid4()), + shape=a.shape, + dtype=a.dtype, + chunks=chunk_shape or a.shape, + chunk_key_encoding={"name": "v2", "separator": "."}, + ) + z[()] = a + return z + + +class CountingDict(MemoryStore): + counter: Counter[tuple[str, str]] + + @classmethod + async def open(cls) -> CountingDict: + store = await super().open() + store.counter = Counter() + return store + + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: tuple[int | None, int | None] | None = None, + ) -> Buffer | None: + key_suffix = "/".join(key.split("/")[1:]) + self.counter["__getitem__", key_suffix] += 1 + return await super().get(key, prototype, byte_range) + + async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: + key_suffix = "/".join(key.split("/")[1:]) + self.counter["__setitem__", key_suffix] += 1 + return await super().set(key, value, byte_range) + + +def test_normalize_integer_selection() -> None: + assert 1 == normalize_integer_selection(1, 100) + assert 99 == normalize_integer_selection(-1, 100) + with pytest.raises(IndexError): + normalize_integer_selection(100, 100) + with pytest.raises(IndexError): + normalize_integer_selection(1000, 100) + with pytest.raises(IndexError): + normalize_integer_selection(-1000, 100) + + +def test_replace_ellipsis() -> None: + # 1D, single item + assert (0,) == replace_ellipsis(0, (100,)) + + # 1D + assert (slice(None),) == replace_ellipsis(Ellipsis, (100,)) + assert (slice(None),) == replace_ellipsis(slice(None), (100,)) + assert (slice(None, 100),) == replace_ellipsis(slice(None, 100), (100,)) + assert (slice(0, None),) == replace_ellipsis(slice(0, None), (100,)) + assert (slice(None),) == replace_ellipsis((slice(None), Ellipsis), (100,)) + assert (slice(None),) == replace_ellipsis((Ellipsis, slice(None)), (100,)) + + # 2D, single item + assert (0, 0) == replace_ellipsis((0, 0), (100, 100)) + assert (-1, 1) == replace_ellipsis((-1, 1), (100, 100)) + + # 2D, single col/row + assert (0, slice(None)) == replace_ellipsis((0, slice(None)), (100, 100)) + assert (0, slice(None)) == replace_ellipsis((0,), (100, 100)) + assert (slice(None), 0) == replace_ellipsis((slice(None), 0), (100, 100)) + + # 2D slice + assert (slice(None), slice(None)) == replace_ellipsis(Ellipsis, (100, 100)) + assert (slice(None), slice(None)) == replace_ellipsis(slice(None), (100, 100)) + assert (slice(None), slice(None)) == replace_ellipsis((slice(None), slice(None)), (100, 100)) + assert (slice(None), slice(None)) == replace_ellipsis((Ellipsis, slice(None)), (100, 100)) + assert (slice(None), slice(None)) == replace_ellipsis((slice(None), Ellipsis), (100, 100)) + assert (slice(None), slice(None)) == replace_ellipsis( + (slice(None), Ellipsis, slice(None)), (100, 100) + ) + assert (slice(None), slice(None)) == replace_ellipsis( + (Ellipsis, slice(None), slice(None)), (100, 100) + ) + assert (slice(None), slice(None)) == replace_ellipsis( + (slice(None), slice(None), Ellipsis), (100, 100) + ) + + +@pytest.mark.parametrize( + ("value", "dtype"), + [ + (42, "uint8"), + pytest.param( + (b"aaa", 1, 4.2), [("foo", "S3"), ("bar", "i4"), ("baz", "f8")], marks=pytest.mark.xfail + ), + ], +) +@pytest.mark.parametrize("use_out", [True, False]) +def test_get_basic_selection_0d(store: StorePath, use_out: bool, value: Any, dtype: Any) -> None: + # setup + arr_np = np.array(value, dtype=dtype) + arr_z = zarr_array_from_numpy_array(store, arr_np) + + assert_array_equal(arr_np, arr_z.get_basic_selection(Ellipsis)) + assert_array_equal(arr_np, arr_z[...]) + assert value == arr_z.get_basic_selection(()) + assert value == arr_z[()] + + if use_out: + # test out param + b = default_buffer_prototype().nd_buffer.from_numpy_array(np.zeros_like(arr_np)) + arr_z.get_basic_selection(Ellipsis, out=b) + assert_array_equal(arr_np, b.as_ndarray_like()) + + # todo: uncomment the structured array tests when we can make them pass, + # or delete them if we formally decide not to support structured dtypes. + + # test structured array + # value = (b"aaa", 1, 4.2) + # a = np.array(value, dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")]) + # z = zarr_array_from_numpy_array(store, a) + # z[()] = value + # assert_array_equal(a, z.get_basic_selection(Ellipsis)) + # assert_array_equal(a, z[...]) + # assert a[()] == z.get_basic_selection(()) + # assert a[()] == z[()] + # assert b"aaa" == z.get_basic_selection((), fields="foo") + # assert b"aaa" == z["foo"] + # assert a[["foo", "bar"]] == z.get_basic_selection((), fields=["foo", "bar"]) + # assert a[["foo", "bar"]] == z["foo", "bar"] + # # test out param + # b = NDBuffer.from_numpy_array(np.zeros_like(a)) + # z.get_basic_selection(Ellipsis, out=b) + # assert_array_equal(a, b) + # c = NDBuffer.from_numpy_array(np.zeros_like(a[["foo", "bar"]])) + # z.get_basic_selection(Ellipsis, out=c, fields=["foo", "bar"]) + # assert_array_equal(a[["foo", "bar"]], c) + + +basic_selections_1d: list[BasicSelection] = [ + # single value + 42, + -1, + # slices + slice(0, 1050), + slice(50, 150), + slice(0, 2000), + slice(-150, -50), + slice(-2000, 2000), + slice(0, 0), # empty result + slice(-1, 0), # empty result + # total selections + slice(None), + Ellipsis, + (), + (Ellipsis, slice(None)), + # slice with step + slice(None), + slice(None, None), + slice(None, None, 1), + slice(None, None, 10), + slice(None, None, 100), + slice(None, None, 1000), + slice(None, None, 10000), + slice(0, 1050), + slice(0, 1050, 1), + slice(0, 1050, 10), + slice(0, 1050, 100), + slice(0, 1050, 1000), + slice(0, 1050, 10000), + slice(1, 31, 3), + slice(1, 31, 30), + slice(1, 31, 300), + slice(81, 121, 3), + slice(81, 121, 30), + slice(81, 121, 300), + slice(50, 150), + slice(50, 150, 1), + slice(50, 150, 10), +] + +basic_selections_1d_bad = [ + # only positive step supported + slice(None, None, -1), + slice(None, None, -10), + slice(None, None, -100), + slice(None, None, -1000), + slice(None, None, -10000), + slice(1050, -1, -1), + slice(1050, -1, -10), + slice(1050, -1, -100), + slice(1050, -1, -1000), + slice(1050, -1, -10000), + slice(1050, 0, -1), + slice(1050, 0, -10), + slice(1050, 0, -100), + slice(1050, 0, -1000), + slice(1050, 0, -10000), + slice(150, 50, -1), + slice(150, 50, -10), + slice(31, 1, -3), + slice(121, 81, -3), + slice(-1, 0, -1), + # bad stuff + 2.3, + "foo", + b"xxx", + None, + (0, 0), + (slice(None), slice(None)), +] + + +def _test_get_basic_selection( + a: npt.NDArray[Any] | Array, z: Array, selection: BasicSelection +) -> None: + expect = a[selection] + actual = z.get_basic_selection(selection) + assert_array_equal(expect, actual) + actual = z[selection] + assert_array_equal(expect, actual) + + # test out param + b = default_buffer_prototype().nd_buffer.from_numpy_array( + np.empty(shape=expect.shape, dtype=expect.dtype) + ) + z.get_basic_selection(selection, out=b) + assert_array_equal(expect, b.as_numpy_array()) + + +# noinspection PyStatementEffect +def test_get_basic_selection_1d(store: StorePath) -> None: + # setup + a = np.arange(1050, dtype=int) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + + for selection in basic_selections_1d: + _test_get_basic_selection(a, z, selection) + + for selection_bad in basic_selections_1d_bad: + with pytest.raises(IndexError): + z.get_basic_selection(selection_bad) # type: ignore[arg-type] + with pytest.raises(IndexError): + z[selection_bad] # type: ignore[index] + + with pytest.raises(IndexError): + z.get_basic_selection([1, 0]) # type: ignore[arg-type] + + +basic_selections_2d: list[BasicSelection] = [ + # single row + 42, + -1, + (42, slice(None)), + (-1, slice(None)), + # single col + (slice(None), 4), + (slice(None), -1), + # row slices + slice(None), + slice(0, 1000), + slice(250, 350), + slice(0, 2000), + slice(-350, -250), + slice(0, 0), # empty result + slice(-1, 0), # empty result + slice(-2000, 0), + slice(-2000, 2000), + # 2D slices + (slice(None), slice(1, 5)), + (slice(250, 350), slice(None)), + (slice(250, 350), slice(1, 5)), + (slice(250, 350), slice(-5, -1)), + (slice(250, 350), slice(-50, 50)), + (slice(250, 350, 10), slice(1, 5)), + (slice(250, 350), slice(1, 5, 2)), + (slice(250, 350, 33), slice(1, 5, 3)), + # total selections + (slice(None), slice(None)), + Ellipsis, + (), + (Ellipsis, slice(None)), + (Ellipsis, slice(None), slice(None)), +] + +basic_selections_2d_bad = [ + # bad stuff + 2.3, + "foo", + b"xxx", + None, + (2.3, slice(None)), + # only positive step supported + slice(None, None, -1), + (slice(None, None, -1), slice(None)), + (0, 0, 0), + (slice(None), slice(None), slice(None)), +] + + +# noinspection PyStatementEffect +def test_get_basic_selection_2d(store: StorePath) -> None: + # setup + a = np.arange(10000, dtype=int).reshape(1000, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + + for selection in basic_selections_2d: + _test_get_basic_selection(a, z, selection) + + bad_selections = basic_selections_2d_bad + [ + # integer arrays + [0, 1], + (slice(None), [0, 1]), + ] + for selection_bad in bad_selections: + with pytest.raises(IndexError): + z.get_basic_selection(selection_bad) # type: ignore[arg-type] + # check fallback on fancy indexing + fancy_selection = ([0, 1], [0, 1]) + np.testing.assert_array_equal(z[fancy_selection], [0, 11]) + + +def test_fancy_indexing_fallback_on_get_setitem(store: StorePath) -> None: + z = zarr_array_from_numpy_array(store, np.zeros((20, 20))) + z[[1, 2, 3], [1, 2, 3]] = 1 + np.testing.assert_array_equal( + z[:4, :4], + [ + [0, 0, 0, 0], + [0, 1, 0, 0], + [0, 0, 1, 0], + [0, 0, 0, 1], + ], + ) + np.testing.assert_array_equal(z[[1, 2, 3], [1, 2, 3]], 1) + # test broadcasting + np.testing.assert_array_equal(z[1, [1, 2, 3]], [1, 0, 0]) + # test 1D fancy indexing + z2 = zarr_array_from_numpy_array(store, np.zeros(5)) + z2[[1, 2, 3]] = 1 + np.testing.assert_array_equal(z2[:], [0, 1, 1, 1, 0]) + + +@pytest.mark.parametrize( + ("index", "expected_result"), + [ + # Single iterable of integers + ([0, 1], [[0, 1, 2], [3, 4, 5]]), + # List first, then slice + (([0, 1], slice(None)), [[0, 1, 2], [3, 4, 5]]), + # List first, then slice + (([0, 1], slice(1, None)), [[1, 2], [4, 5]]), + # Slice first, then list + ((slice(0, 2), [0, 2]), [[0, 2], [3, 5]]), + # Slices only + ((slice(0, 2), slice(0, 2)), [[0, 1], [3, 4]]), + # List with repeated index + (([1, 0, 1], slice(1, None)), [[4, 5], [1, 2], [4, 5]]), + # 1D indexing + (([1, 0, 1]), [[3, 4, 5], [0, 1, 2], [3, 4, 5]]), + ], +) +def test_orthogonal_indexing_fallback_on_getitem_2d( + store: StorePath, index: Selection, expected_result: npt.ArrayLike +) -> None: + """ + Tests the orthogonal indexing fallback on __getitem__ for a 2D matrix. + + In addition to checking expected behavior, all indexing + is also checked against numpy. + """ + # [0, 1, 2], + # [3, 4, 5], + # [6, 7, 8] + a = np.arange(9).reshape(3, 3) + z = zarr_array_from_numpy_array(store, a) + + np.testing.assert_array_equal(z[index], a[index], err_msg="Indexing disagrees with numpy") + np.testing.assert_array_equal(z[index], expected_result) + + +@pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI") +def test_setitem_repeated_index(): + array = zarr.array(data=np.zeros((4,)), chunks=(1,)) + indexer = np.array([-1, -1, 0, 0]) + array.oindex[(indexer,)] = [0, 1, 2, 3] + np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1])) + + indexer = np.array([-1, 0, 0, -1]) + array.oindex[(indexer,)] = [0, 1, 2, 3] + np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3])) + + +Index = list[int] | tuple[slice | int | list[int], ...] + + +@pytest.mark.parametrize( + ("index", "expected_result"), + [ + # Single iterable of integers + ([0, 1], [[[0, 1, 2], [3, 4, 5], [6, 7, 8]], [[9, 10, 11], [12, 13, 14], [15, 16, 17]]]), + # One slice, two integers + ((slice(0, 2), 1, 1), [4, 13]), + # One integer, two slices + ((slice(0, 2), 1, slice(0, 2)), [[3, 4], [12, 13]]), + # Two slices and a list + ((slice(0, 2), [1, 2], slice(0, 2)), [[[3, 4], [6, 7]], [[12, 13], [15, 16]]]), + ], +) +def test_orthogonal_indexing_fallback_on_getitem_3d( + store: StorePath, index: Selection, expected_result: npt.ArrayLike +) -> None: + """ + Tests the orthogonal indexing fallback on __getitem__ for a 3D matrix. + + In addition to checking expected behavior, all indexing + is also checked against numpy. + """ + # [[[ 0, 1, 2], + # [ 3, 4, 5], + # [ 6, 7, 8]], + + # [[ 9, 10, 11], + # [12, 13, 14], + # [15, 16, 17]], + + # [[18, 19, 20], + # [21, 22, 23], + # [24, 25, 26]]] + a = np.arange(27).reshape(3, 3, 3) + z = zarr_array_from_numpy_array(store, a) + + np.testing.assert_array_equal(z[index], a[index], err_msg="Indexing disagrees with numpy") + np.testing.assert_array_equal(z[index], expected_result) + + +@pytest.mark.parametrize( + ("index", "expected_result"), + [ + # Single iterable of integers + ([0, 1], [[1, 1, 1], [1, 1, 1], [0, 0, 0]]), + # List and slice combined + (([0, 1], slice(1, 3)), [[0, 1, 1], [0, 1, 1], [0, 0, 0]]), + # Index repetition is ignored on setitem + (([0, 1, 1, 1, 1, 1, 1], slice(1, 3)), [[0, 1, 1], [0, 1, 1], [0, 0, 0]]), + # Slice with step + (([0, 2], slice(None, None, 2)), [[1, 0, 1], [0, 0, 0], [1, 0, 1]]), + ], +) +def test_orthogonal_indexing_fallback_on_setitem_2d( + store: StorePath, index: Selection, expected_result: npt.ArrayLike +) -> None: + """ + Tests the orthogonal indexing fallback on __setitem__ for a 3D matrix. + + In addition to checking expected behavior, all indexing + is also checked against numpy. + """ + # Slice + fancy index + a = np.zeros((3, 3)) + z = zarr_array_from_numpy_array(store, a) + z[index] = 1 + a[index] = 1 + np.testing.assert_array_equal(z[:], expected_result) + np.testing.assert_array_equal(z[:], a, err_msg="Indexing disagrees with numpy") + + +def test_fancy_indexing_doesnt_mix_with_implicit_slicing(store: StorePath) -> None: + z2 = zarr_array_from_numpy_array(store, np.zeros((5, 5, 5))) + with pytest.raises(IndexError): + z2[[1, 2, 3], [1, 2, 3]] = 2 + with pytest.raises(IndexError): + np.testing.assert_array_equal(z2[[1, 2, 3], [1, 2, 3]], 0) + with pytest.raises(IndexError): + z2[..., [1, 2, 3]] = 2 # type: ignore[index] + with pytest.raises(IndexError): + np.testing.assert_array_equal(z2[..., [1, 2, 3]], 0) # type: ignore[index] + + +@pytest.mark.parametrize( + ("value", "dtype"), + [ + (42, "uint8"), + pytest.param( + (b"aaa", 1, 4.2), [("foo", "S3"), ("bar", "i4"), ("baz", "f8")], marks=pytest.mark.xfail + ), + ], +) +def test_set_basic_selection_0d( + store: StorePath, value: Any, dtype: str | list[tuple[str, str]] +) -> None: + arr_np = np.array(value, dtype=dtype) + arr_np_zeros = np.zeros_like(arr_np, dtype=dtype) + arr_z = zarr_array_from_numpy_array(store, arr_np_zeros) + assert_array_equal(arr_np_zeros, arr_z) + + arr_z.set_basic_selection(Ellipsis, value) + assert_array_equal(value, arr_z) + arr_z[...] = 0 + assert_array_equal(arr_np_zeros, arr_z) + arr_z[...] = value + assert_array_equal(value, arr_z) + + # todo: uncomment the structured array tests when we can make them pass, + # or delete them if we formally decide not to support structured dtypes. + + # arr_z.set_basic_selection(Ellipsis, v["foo"], fields="foo") + # assert v["foo"] == arr_z["foo"] + # assert arr_np_zeros["bar"] == arr_z["bar"] + # assert arr_np_zeros["baz"] == arr_z["baz"] + # arr_z["bar"] = v["bar"] + # assert v["foo"] == arr_z["foo"] + # assert v["bar"] == arr_z["bar"] + # assert arr_np_zeros["baz"] == arr_z["baz"] + # # multiple field assignment not supported + # with pytest.raises(IndexError): + # arr_z.set_basic_selection(Ellipsis, v[["foo", "bar"]], fields=["foo", "bar"]) + # with pytest.raises(IndexError): + # arr_z[..., "foo", "bar"] = v[["foo", "bar"]] + + +def _test_get_orthogonal_selection( + a: npt.NDArray[Any], z: Array, selection: OrthogonalSelection +) -> None: + expect = oindex(a, selection) + actual = z.get_orthogonal_selection(selection) + assert_array_equal(expect, actual) + actual = z.oindex[selection] + assert_array_equal(expect, actual) + + +# noinspection PyStatementEffect +def test_get_orthogonal_selection_1d_bool(store: StorePath) -> None: + # setup + a = np.arange(1050, dtype=int) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 0.5, 0.1, 0.01: + ix = np.random.binomial(1, p, size=a.shape[0]).astype(bool) + _test_get_orthogonal_selection(a, z, ix) + + # test errors + with pytest.raises(IndexError): + z.oindex[np.zeros(50, dtype=bool)] # too short + with pytest.raises(IndexError): + z.oindex[np.zeros(2000, dtype=bool)] # too long + with pytest.raises(IndexError): + # too many dimensions + z.oindex[[[True, False], [False, True]]] # type: ignore[index] + + +# noinspection PyStatementEffect +def test_get_orthogonal_selection_1d_int(store: StorePath) -> None: + # setup + a = np.arange(1050, dtype=int) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 2, 0.5, 0.1, 0.01: + # unordered + ix = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) + _test_get_orthogonal_selection(a, z, ix) + # increasing + ix.sort() + _test_get_orthogonal_selection(a, z, ix) + # decreasing + ix = ix[::-1] + _test_get_orthogonal_selection(a, z, ix) + + selections = basic_selections_1d + [ + # test wraparound + [0, 3, 10, -23, -12, -1], + # explicit test not sorted + [3, 105, 23, 127], + ] + for selection in selections: + _test_get_orthogonal_selection(a, z, selection) + + bad_selections = basic_selections_1d_bad + [ + [a.shape[0] + 1], # out of bounds + [-(a.shape[0] + 1)], # out of bounds + [[2, 4], [6, 8]], # too many dimensions + ] + for bad_selection in bad_selections: + with pytest.raises(IndexError): + z.get_orthogonal_selection(bad_selection) # type: ignore[arg-type] + with pytest.raises(IndexError): + z.oindex[bad_selection] # type: ignore[index] + + +def _test_get_orthogonal_selection_2d( + a: npt.NDArray[Any], z: Array, ix0: npt.NDArray[np.bool], ix1: npt.NDArray[np.bool] +) -> None: + selections = [ + # index both axes with array + (ix0, ix1), + # mixed indexing with array / slice + (ix0, slice(1, 5)), + (ix0, slice(1, 5, 2)), + (slice(250, 350), ix1), + (slice(250, 350, 10), ix1), + # mixed indexing with array / int + (ix0, 4), + (42, ix1), + ] + for selection in selections: + _test_get_orthogonal_selection(a, z, selection) + + +# noinspection PyStatementEffect +def test_get_orthogonal_selection_2d(store: StorePath) -> None: + # setup + a = np.arange(10000, dtype=int).reshape(1000, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 0.5, 0.1, 0.01: + # boolean arrays + ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) + ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) + _test_get_orthogonal_selection_2d(a, z, ix0, ix1) + + # mixed int array / bool array + selections = ( + (ix0, np.nonzero(ix1)[0]), + (np.nonzero(ix0)[0], ix1), + ) + for selection in selections: + _test_get_orthogonal_selection(a, z, selection) + + # integer arrays + ix0 = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) + ix1 = np.random.choice(a.shape[1], size=int(a.shape[1] * 0.5), replace=True) + _test_get_orthogonal_selection_2d(a, z, ix0, ix1) + ix0.sort() + ix1.sort() + _test_get_orthogonal_selection_2d(a, z, ix0, ix1) + ix0 = ix0[::-1] + ix1 = ix1[::-1] + _test_get_orthogonal_selection_2d(a, z, ix0, ix1) + + for selection_2d in basic_selections_2d: + _test_get_orthogonal_selection(a, z, selection_2d) + + for selection_2d_bad in basic_selections_2d_bad: + with pytest.raises(IndexError): + z.get_orthogonal_selection(selection_2d_bad) # type: ignore[arg-type] + with pytest.raises(IndexError): + z.oindex[selection_2d_bad] # type: ignore[index] + + +def _test_get_orthogonal_selection_3d( + a: npt.NDArray, + z: Array, + ix0: npt.NDArray[np.bool], + ix1: npt.NDArray[np.bool], + ix2: npt.NDArray[np.bool], +) -> None: + selections = [ + # single value + (84, 42, 4), + (-1, -1, -1), + # index all axes with array + (ix0, ix1, ix2), + # mixed indexing with single array / slices + (ix0, slice(15, 25), slice(1, 5)), + (slice(50, 70), ix1, slice(1, 5)), + (slice(50, 70), slice(15, 25), ix2), + (ix0, slice(15, 25, 5), slice(1, 5, 2)), + (slice(50, 70, 3), ix1, slice(1, 5, 2)), + (slice(50, 70, 3), slice(15, 25, 5), ix2), + # mixed indexing with single array / ints + (ix0, 42, 4), + (84, ix1, 4), + (84, 42, ix2), + # mixed indexing with single array / slice / int + (ix0, slice(15, 25), 4), + (42, ix1, slice(1, 5)), + (slice(50, 70), 42, ix2), + # mixed indexing with two array / slice + (ix0, ix1, slice(1, 5)), + (slice(50, 70), ix1, ix2), + (ix0, slice(15, 25), ix2), + # mixed indexing with two array / integer + (ix0, ix1, 4), + (42, ix1, ix2), + (ix0, 42, ix2), + ] + for selection in selections: + _test_get_orthogonal_selection(a, z, selection) + + +def test_get_orthogonal_selection_3d(store: StorePath) -> None: + # setup + a = np.arange(100000, dtype=int).reshape(200, 50, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(60, 20, 3)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 0.5, 0.1, 0.01: + # boolean arrays + ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) + ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) + ix2 = np.random.binomial(1, 0.5, size=a.shape[2]).astype(bool) + _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2) + + # integer arrays + ix0 = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) + ix1 = np.random.choice(a.shape[1], size=int(a.shape[1] * 0.5), replace=True) + ix2 = np.random.choice(a.shape[2], size=int(a.shape[2] * 0.5), replace=True) + _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2) + ix0.sort() + ix1.sort() + ix2.sort() + _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2) + ix0 = ix0[::-1] + ix1 = ix1[::-1] + ix2 = ix2[::-1] + _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2) + + +def test_orthogonal_indexing_edge_cases(store: StorePath) -> None: + a = np.arange(6).reshape(1, 2, 3) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(1, 2, 3)) + + expect = oindex(a, (0, slice(None), [0, 1, 2])) + actual = z.oindex[0, :, [0, 1, 2]] + assert_array_equal(expect, actual) + + expect = oindex(a, (0, slice(None), [True, True, True])) + actual = z.oindex[0, :, [True, True, True]] + assert_array_equal(expect, actual) + + +def _test_set_orthogonal_selection( + v: npt.NDArray[np.int_], a: npt.NDArray[Any], z: Array, selection: OrthogonalSelection +) -> None: + for value in 42, oindex(v, selection), oindex(v, selection).tolist(): + if isinstance(value, list) and value == []: + # skip these cases as cannot preserve all dimensions + continue + # setup expectation + a[:] = 0 + oindex_set(a, selection, value) + # long-form API + z[:] = 0 + z.set_orthogonal_selection(selection, value) + assert_array_equal(a, z[:]) + # short-form API + z[:] = 0 + z.oindex[selection] = value + assert_array_equal(a, z[:]) + + +def test_set_orthogonal_selection_1d(store: StorePath) -> None: + # setup + v = np.arange(1050, dtype=int) + a = np.empty(v.shape, dtype=int) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + + # test with different degrees of sparseness + np.random.seed(42) + for p in 0.5, 0.1, 0.01: + # boolean arrays + ix = np.random.binomial(1, p, size=a.shape[0]).astype(bool) + _test_set_orthogonal_selection(v, a, z, ix) + + # integer arrays + ix = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) + _test_set_orthogonal_selection(v, a, z, ix) + ix.sort() + _test_set_orthogonal_selection(v, a, z, ix) + ix = ix[::-1] + _test_set_orthogonal_selection(v, a, z, ix) + + # basic selections + for selection in basic_selections_1d: + _test_set_orthogonal_selection(v, a, z, selection) + + +def test_set_item_1d_last_two_chunks(store: StorePath): + # regression test for GH2849 + g = zarr.open_group(store=store, zarr_format=3, mode="w") + a = g.create_array("bar", shape=(10,), chunks=(3,), dtype=int) + data = np.array([7, 8, 9]) + a[slice(7, 10)] = data + np.testing.assert_array_equal(a[slice(7, 10)], data) + + z = zarr.open_group(store=store, mode="w") + z.create_array("zoo", dtype=float, shape=()) + z["zoo"][...] = np.array(1) # why doesn't [:] work? + np.testing.assert_equal(z["zoo"][()], np.array(1)) + + z = zarr.open_group(store=store, mode="w") + z.create_array("zoo", dtype=float, shape=()) + z["zoo"][...] = 1 # why doesn't [:] work? + np.testing.assert_equal(z["zoo"][()], np.array(1)) + + +def _test_set_orthogonal_selection_2d( + v: npt.NDArray[np.int_], + a: npt.NDArray[np.int_], + z: Array, + ix0: npt.NDArray[np.bool], + ix1: npt.NDArray[np.bool], +) -> None: + selections = [ + # index both axes with array + (ix0, ix1), + # mixed indexing with array / slice or int + (ix0, slice(1, 5)), + (slice(250, 350), ix1), + (ix0, 4), + (42, ix1), + ] + for selection in selections: + _test_set_orthogonal_selection(v, a, z, selection) + + +def test_set_orthogonal_selection_2d(store: StorePath) -> None: + # setup + v = np.arange(10000, dtype=int).reshape(1000, 10) + a = np.empty_like(v) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 0.5, 0.1, 0.01: + # boolean arrays + ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) + ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) + _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1) + + # integer arrays + ix0 = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) + ix1 = np.random.choice(a.shape[1], size=int(a.shape[1] * 0.5), replace=True) + _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1) + ix0.sort() + ix1.sort() + _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1) + ix0 = ix0[::-1] + ix1 = ix1[::-1] + _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1) + + for selection in basic_selections_2d: + _test_set_orthogonal_selection(v, a, z, selection) + + +def _test_set_orthogonal_selection_3d( + v: npt.NDArray[np.int_], + a: npt.NDArray[np.int_], + z: Array, + ix0: npt.NDArray[np.bool], + ix1: npt.NDArray[np.bool], + ix2: npt.NDArray[np.bool], +) -> None: + selections = ( + # single value + (84, 42, 4), + (-1, -1, -1), + # index all axes with bool array + (ix0, ix1, ix2), + # mixed indexing with single bool array / slice or int + (ix0, slice(15, 25), slice(1, 5)), + (slice(50, 70), ix1, slice(1, 5)), + (slice(50, 70), slice(15, 25), ix2), + (ix0, 42, 4), + (84, ix1, 4), + (84, 42, ix2), + (ix0, slice(15, 25), 4), + (slice(50, 70), ix1, 4), + (slice(50, 70), 42, ix2), + # indexing with two arrays / slice + (ix0, ix1, slice(1, 5)), + # indexing with two arrays / integer + (ix0, ix1, 4), + ) + for selection in selections: + _test_set_orthogonal_selection(v, a, z, selection) + + +def test_set_orthogonal_selection_3d(store: StorePath) -> None: + # setup + v = np.arange(100000, dtype=int).reshape(200, 50, 10) + a = np.empty_like(v) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(60, 20, 3)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 0.5, 0.1, 0.01: + # boolean arrays + ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) + ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) + ix2 = np.random.binomial(1, 0.5, size=a.shape[2]).astype(bool) + _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2) + + # integer arrays + ix0 = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) + ix1 = np.random.choice(a.shape[1], size=int(a.shape[1] * 0.5), replace=True) + ix2 = np.random.choice(a.shape[2], size=int(a.shape[2] * 0.5), replace=True) + _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2) + + # sorted increasing + ix0.sort() + ix1.sort() + ix2.sort() + _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2) + + # sorted decreasing + ix0 = ix0[::-1] + ix1 = ix1[::-1] + ix2 = ix2[::-1] + _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2) + + +def test_orthogonal_indexing_fallback_on_get_setitem(store: StorePath) -> None: + z = zarr_array_from_numpy_array(store, np.zeros((20, 20))) + z[[1, 2, 3], [1, 2, 3]] = 1 + np.testing.assert_array_equal( + z[:4, :4], + [ + [0, 0, 0, 0], + [0, 1, 0, 0], + [0, 0, 1, 0], + [0, 0, 0, 1], + ], + ) + np.testing.assert_array_equal(z[[1, 2, 3], [1, 2, 3]], 1) + # test broadcasting + np.testing.assert_array_equal(z[1, [1, 2, 3]], [1, 0, 0]) + # test 1D fancy indexing + z2 = zarr_array_from_numpy_array(store, np.zeros(5)) + z2[[1, 2, 3]] = 1 + np.testing.assert_array_equal(z2[:], [0, 1, 1, 1, 0]) + + +def _test_get_coordinate_selection( + a: npt.NDArray, z: Array, selection: CoordinateSelection +) -> None: + expect = a[selection] + actual = z.get_coordinate_selection(selection) + assert_array_equal(expect, actual) + actual = z.vindex[selection] + assert_array_equal(expect, actual) + + +coordinate_selections_1d_bad = [ + # slice not supported + slice(5, 15), + slice(None), + Ellipsis, + # bad stuff + 2.3, + "foo", + b"xxx", + None, + (0, 0), + (slice(None), slice(None)), +] + + +# noinspection PyStatementEffect +def test_get_coordinate_selection_1d(store: StorePath) -> None: + # setup + a = np.arange(1050, dtype=int) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 2, 0.5, 0.1, 0.01: + n = int(a.size * p) + ix = np.random.choice(a.shape[0], size=n, replace=True) + _test_get_coordinate_selection(a, z, ix) + ix.sort() + _test_get_coordinate_selection(a, z, ix) + ix = ix[::-1] + _test_get_coordinate_selection(a, z, ix) + + selections = [ + # test single item + 42, + -1, + # test wraparound + [0, 3, 10, -23, -12, -1], + # test out of order + [3, 105, 23, 127], # not monotonically increasing + # test multi-dimensional selection + np.array([[2, 4], [6, 8]]), + ] + for selection in selections: + _test_get_coordinate_selection(a, z, selection) + + # test errors + bad_selections = coordinate_selections_1d_bad + [ + [a.shape[0] + 1], # out of bounds + [-(a.shape[0] + 1)], # out of bounds + ] + for selection in bad_selections: + with pytest.raises(IndexError): + z.get_coordinate_selection(selection) # type: ignore[arg-type] + with pytest.raises(IndexError): + z.vindex[selection] # type: ignore[index] + + +def test_get_coordinate_selection_2d(store: StorePath) -> None: + # setup + a = np.arange(10000, dtype=int).reshape(1000, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + + np.random.seed(42) + ix0: npt.ArrayLike + ix1: npt.ArrayLike + # test with different degrees of sparseness + for p in 2, 0.5, 0.1, 0.01: + n = int(a.size * p) + ix0 = np.random.choice(a.shape[0], size=n, replace=True) + ix1 = np.random.choice(a.shape[1], size=n, replace=True) + selections = [ + # single value + (42, 4), + (-1, -1), + # index both axes with array + (ix0, ix1), + # mixed indexing with array / int + (ix0, 4), + (42, ix1), + (42, 4), + ] + for selection in selections: + _test_get_coordinate_selection(a, z, selection) + + # not monotonically increasing (first dim) + ix0 = [3, 3, 4, 2, 5] + ix1 = [1, 3, 5, 7, 9] + _test_get_coordinate_selection(a, z, (ix0, ix1)) + + # not monotonically increasing (second dim) + ix0 = [1, 1, 2, 2, 5] + ix1 = [1, 3, 2, 1, 0] + _test_get_coordinate_selection(a, z, (ix0, ix1)) + + # multi-dimensional selection + ix0 = np.array([[1, 1, 2], [2, 2, 5]]) + ix1 = np.array([[1, 3, 2], [1, 0, 0]]) + _test_get_coordinate_selection(a, z, (ix0, ix1)) + + with pytest.raises(IndexError): + selection = slice(5, 15), [1, 2, 3] + z.get_coordinate_selection(selection) # type:ignore[arg-type] + with pytest.raises(IndexError): + selection = [1, 2, 3], slice(5, 15) + z.get_coordinate_selection(selection) # type:ignore[arg-type] + with pytest.raises(IndexError): + selection = Ellipsis, [1, 2, 3] + z.get_coordinate_selection(selection) # type:ignore[arg-type] + with pytest.raises(IndexError): + selection = Ellipsis + z.get_coordinate_selection(selection) # type:ignore[arg-type] + + +def _test_set_coordinate_selection( + v: npt.NDArray, a: npt.NDArray, z: Array, selection: CoordinateSelection +) -> None: + for value in 42, v[selection], v[selection].tolist(): + # setup expectation + a[:] = 0 + a[selection] = value + # test long-form API + z[:] = 0 + z.set_coordinate_selection(selection, value) + assert_array_equal(a, z[:]) + # test short-form API + z[:] = 0 + z.vindex[selection] = value + assert_array_equal(a, z[:]) + + +def test_set_coordinate_selection_1d(store: StorePath) -> None: + # setup + v = np.arange(1050, dtype=int) + a = np.empty(v.shape, dtype=v.dtype) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 2, 0.5, 0.1, 0.01: + n = int(a.size * p) + ix = np.random.choice(a.shape[0], size=n, replace=True) + _test_set_coordinate_selection(v, a, z, ix) + + # multi-dimensional selection + ix = np.array([[2, 4], [6, 8]]) + _test_set_coordinate_selection(v, a, z, ix) + + for selection in coordinate_selections_1d_bad: + with pytest.raises(IndexError): + z.set_coordinate_selection(selection, 42) # type:ignore[arg-type] + with pytest.raises(IndexError): + z.vindex[selection] = 42 # type:ignore[index] + + +def test_set_coordinate_selection_2d(store: StorePath) -> None: + # setup + v = np.arange(10000, dtype=int).reshape(1000, 10) + a = np.empty_like(v) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 2, 0.5, 0.1, 0.01: + n = int(a.size * p) + ix0 = np.random.choice(a.shape[0], size=n, replace=True) + ix1 = np.random.choice(a.shape[1], size=n, replace=True) + + selections = ( + (42, 4), + (-1, -1), + # index both axes with array + (ix0, ix1), + # mixed indexing with array / int + (ix0, 4), + (42, ix1), + ) + for selection in selections: + _test_set_coordinate_selection(v, a, z, selection) + + # multi-dimensional selection + ix0 = np.array([[1, 2, 3], [4, 5, 6]]) + ix1 = np.array([[1, 3, 2], [2, 0, 5]]) + _test_set_coordinate_selection(v, a, z, (ix0, ix1)) + + +def _test_get_block_selection( + a: npt.NDArray[Any], + z: Array, + selection: BasicSelection, + expected_idx: slice | tuple[slice, ...], +) -> None: + expect = a[expected_idx] + actual = z.get_block_selection(selection) + assert_array_equal(expect, actual) + actual = z.blocks[selection] + assert_array_equal(expect, actual) + + +block_selections_1d: list[BasicSelection] = [ + # test single item + 0, + 5, + # test wraparound + -1, + -4, + # test slice + slice(5), + slice(None, 3), + slice(5, 6), + slice(-3, -1), + slice(None), # Full slice +] + +block_selections_1d_array_projection: list[slice] = [ + # test single item + slice(100), + slice(500, 600), + # test wraparound + slice(1000, None), + slice(700, 800), + # test slice + slice(500), + slice(None, 300), + slice(500, 600), + slice(800, 1000), + slice(None), +] + +block_selections_1d_bad = [ + # slice not supported + slice(3, 8, 2), + # bad stuff + 2.3, + # "foo", # TODO + b"xxx", + None, + (0, 0), + (slice(None), slice(None)), + [0, 5, 3], +] + + +def test_get_block_selection_1d(store: StorePath) -> None: + # setup + a = np.arange(1050, dtype=int) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + + for selection, expected_idx in zip( + block_selections_1d, block_selections_1d_array_projection, strict=True + ): + _test_get_block_selection(a, z, selection, expected_idx) + + bad_selections = block_selections_1d_bad + [ + z.metadata.chunk_grid.get_nchunks(z.shape) + 1, # out of bounds + -(z.metadata.chunk_grid.get_nchunks(z.shape) + 1), # out of bounds + ] + + for selection_bad in bad_selections: + with pytest.raises(IndexError): + z.get_block_selection(selection_bad) # type:ignore[arg-type] + with pytest.raises(IndexError): + z.blocks[selection_bad] # type:ignore[index] + + +block_selections_2d: list[BasicSelection] = [ + # test single item + (0, 0), + (1, 2), + # test wraparound + (-1, -1), + (-3, -2), + # test slice + (slice(1), slice(2)), + (slice(None, 2), slice(-2, -1)), + (slice(2, 3), slice(-2, None)), + (slice(-3, -1), slice(-3, -2)), + (slice(None), slice(None)), # Full slice +] + +block_selections_2d_array_projection: list[tuple[slice, slice]] = [ + # test single item + (slice(300), slice(3)), + (slice(300, 600), slice(6, 9)), + # test wraparound + (slice(900, None), slice(9, None)), + (slice(300, 600), slice(6, 9)), + # test slice + (slice(300), slice(6)), + (slice(None, 600), slice(6, 9)), + (slice(600, 900), slice(6, None)), + (slice(300, 900), slice(3, 6)), + (slice(None), slice(None)), # Full slice +] + + +def test_get_block_selection_2d(store: StorePath) -> None: + # setup + a = np.arange(10000, dtype=int).reshape(1000, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + + for selection, expected_idx in zip( + block_selections_2d, block_selections_2d_array_projection, strict=True + ): + _test_get_block_selection(a, z, selection, expected_idx) + + with pytest.raises(IndexError): + selection = slice(5, 15), [1, 2, 3] + z.get_block_selection(selection) + with pytest.raises(IndexError): + selection = Ellipsis, [1, 2, 3] + z.get_block_selection(selection) + with pytest.raises(IndexError): # out of bounds + selection = slice(15, 20), slice(None) + z.get_block_selection(selection) + + +def _test_set_block_selection( + v: npt.NDArray[Any], + a: npt.NDArray[Any], + z: zarr.Array, + selection: BasicSelection, + expected_idx: slice, +) -> None: + for value in 42, v[expected_idx], v[expected_idx].tolist(): + # setup expectation + a[:] = 0 + a[expected_idx] = value + # test long-form API + z[:] = 0 + z.set_block_selection(selection, value) + assert_array_equal(a, z[:]) + # test short-form API + z[:] = 0 + z.blocks[selection] = value + assert_array_equal(a, z[:]) + + +def test_set_block_selection_1d(store: StorePath) -> None: + # setup + v = np.arange(1050, dtype=int) + a = np.empty(v.shape, dtype=v.dtype) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + + for selection, expected_idx in zip( + block_selections_1d, block_selections_1d_array_projection, strict=True + ): + _test_set_block_selection(v, a, z, selection, expected_idx) + + for selection_bad in block_selections_1d_bad: + with pytest.raises(IndexError): + z.set_block_selection(selection_bad, 42) # type:ignore[arg-type] + with pytest.raises(IndexError): + z.blocks[selection_bad] = 42 # type:ignore[index] + + +def test_set_block_selection_2d(store: StorePath) -> None: + # setup + v = np.arange(10000, dtype=int).reshape(1000, 10) + a = np.empty(v.shape, dtype=v.dtype) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + + for selection, expected_idx in zip( + block_selections_2d, block_selections_2d_array_projection, strict=True + ): + _test_set_block_selection(v, a, z, selection, expected_idx) + + with pytest.raises(IndexError): + selection = slice(5, 15), [1, 2, 3] + z.set_block_selection(selection, 42) + with pytest.raises(IndexError): + selection = Ellipsis, [1, 2, 3] + z.set_block_selection(selection, 42) + with pytest.raises(IndexError): # out of bounds + selection = slice(15, 20), slice(None) + z.set_block_selection(selection, 42) + + +def _test_get_mask_selection(a: npt.NDArray[Any], z: Array, selection: npt.NDArray) -> None: + expect = a[selection] + actual = z.get_mask_selection(selection) + assert_array_equal(expect, actual) + actual = z.vindex[selection] + assert_array_equal(expect, actual) + actual = z[selection] + assert_array_equal(expect, actual) + + +mask_selections_1d_bad = [ + # slice not supported + slice(5, 15), + slice(None), + Ellipsis, + # bad stuff + 2.3, + "foo", + b"xxx", + None, + (0, 0), + (slice(None), slice(None)), +] + + +# noinspection PyStatementEffect +def test_get_mask_selection_1d(store: StorePath) -> None: + # setup + a = np.arange(1050, dtype=int) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 0.5, 0.1, 0.01: + ix = np.random.binomial(1, p, size=a.shape[0]).astype(bool) + _test_get_mask_selection(a, z, ix) + + # test errors + bad_selections = mask_selections_1d_bad + [ + np.zeros(50, dtype=bool), # too short + np.zeros(2000, dtype=bool), # too long + [[True, False], [False, True]], # too many dimensions + ] + for selection in bad_selections: + with pytest.raises(IndexError): + z.get_mask_selection(selection) # type: ignore[arg-type] + with pytest.raises(IndexError): + z.vindex[selection] # type:ignore[index] + + +# noinspection PyStatementEffect +def test_get_mask_selection_2d(store: StorePath) -> None: + # setup + a = np.arange(10000, dtype=int).reshape(1000, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 0.5, 0.1, 0.01: + ix = np.random.binomial(1, p, size=a.size).astype(bool).reshape(a.shape) + _test_get_mask_selection(a, z, ix) + + # test errors + with pytest.raises(IndexError): + z.vindex[np.zeros((1000, 5), dtype=bool)] # too short + with pytest.raises(IndexError): + z.vindex[np.zeros((2000, 10), dtype=bool)] # too long + with pytest.raises(IndexError): + z.vindex[[True, False]] # wrong no. dimensions + + +def _test_set_mask_selection( + v: npt.NDArray, a: npt.NDArray, z: Array, selection: npt.NDArray +) -> None: + a[:] = 0 + z[:] = 0 + a[selection] = v[selection] + z.set_mask_selection(selection, v[selection]) + assert_array_equal(a, z[:]) + z[:] = 0 + z.vindex[selection] = v[selection] + assert_array_equal(a, z[:]) + z[:] = 0 + z[selection] = v[selection] + assert_array_equal(a, z[:]) + + +def test_set_mask_selection_1d(store: StorePath) -> None: + # setup + v = np.arange(1050, dtype=int) + a = np.empty_like(v) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 0.5, 0.1, 0.01: + ix = np.random.binomial(1, p, size=a.shape[0]).astype(bool) + _test_set_mask_selection(v, a, z, ix) + + for selection in mask_selections_1d_bad: + with pytest.raises(IndexError): + z.set_mask_selection(selection, 42) # type: ignore[arg-type] + with pytest.raises(IndexError): + z.vindex[selection] = 42 # type: ignore[index] + + +def test_set_mask_selection_2d(store: StorePath) -> None: + # setup + v = np.arange(10000, dtype=int).reshape(1000, 10) + a = np.empty_like(v) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + + np.random.seed(42) + # test with different degrees of sparseness + for p in 0.5, 0.1, 0.01: + ix = np.random.binomial(1, p, size=a.size).astype(bool).reshape(a.shape) + _test_set_mask_selection(v, a, z, ix) + + +def test_get_selection_out(store: StorePath) -> None: + # basic selections + a = np.arange(1050) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + + selections = [ + slice(50, 150), + slice(0, 1050), + slice(1, 2), + ] + for selection in selections: + expect = a[selection] + out = get_ndbuffer_class().from_numpy_array(np.empty(expect.shape)) + z.get_basic_selection(selection, out=out) + assert_array_equal(expect, out.as_numpy_array()[:]) + + with pytest.raises(TypeError): + z.get_basic_selection(Ellipsis, out=[]) # type: ignore[arg-type] + + # orthogonal selections + a = np.arange(10000, dtype=int).reshape(1000, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + np.random.seed(42) + # test with different degrees of sparseness + for p in 0.5, 0.1, 0.01: + ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) + ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) + selections = [ + # index both axes with array + (ix0, ix1), + # mixed indexing with array / slice + (ix0, slice(1, 5)), + (slice(250, 350), ix1), + # mixed indexing with array / int + (ix0, 4), + (42, ix1), + # mixed int array / bool array + (ix0, np.nonzero(ix1)[0]), + (np.nonzero(ix0)[0], ix1), + ] + for selection in selections: + expect = oindex(a, selection) + out = get_ndbuffer_class().from_numpy_array(np.zeros(expect.shape, dtype=expect.dtype)) + z.get_orthogonal_selection(selection, out=out) + assert_array_equal(expect, out.as_numpy_array()[:]) + + # coordinate selections + a = np.arange(10000, dtype=int).reshape(1000, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + np.random.seed(42) + # test with different degrees of sparseness + for p in 0.5, 0.1, 0.01: + n = int(a.size * p) + ix0 = np.random.choice(a.shape[0], size=n, replace=True) + ix1 = np.random.choice(a.shape[1], size=n, replace=True) + selections = [ + # index both axes with array + (ix0, ix1), + # mixed indexing with array / int + (ix0, 4), + (42, ix1), + ] + for selection in selections: + expect = a[selection] + out = get_ndbuffer_class().from_numpy_array(np.zeros(expect.shape, dtype=expect.dtype)) + z.get_coordinate_selection(selection, out=out) + assert_array_equal(expect, out.as_numpy_array()[:]) + + +@pytest.mark.xfail(reason="fields are not supported in v3") +def test_get_selections_with_fields(store: StorePath) -> None: + a = np.array( + [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)], + dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")], + ) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(2,)) + + fields_fixture: list[str | list[str]] = [ + "foo", + ["foo"], + ["foo", "bar"], + ["foo", "baz"], + ["bar", "baz"], + ["foo", "bar", "baz"], + ["bar", "foo"], + ["baz", "bar", "foo"], + ] + + for fields in fields_fixture: + # total selection + expect = a[fields] + actual = z.get_basic_selection(Ellipsis, fields=fields) + assert_array_equal(expect, actual) + # alternative API + if isinstance(fields, str): + actual = z[fields] + assert_array_equal(expect, actual) + elif len(fields) == 2: + actual = z[fields[0], fields[1]] + assert_array_equal(expect, actual) + if isinstance(fields, str): + actual = z[..., fields] + assert_array_equal(expect, actual) + elif len(fields) == 2: + actual = z[..., fields[0], fields[1]] + assert_array_equal(expect, actual) + + # basic selection with slice + expect = a[fields][0:2] + actual = z.get_basic_selection(slice(0, 2), fields=fields) + assert_array_equal(expect, actual) + # alternative API + if isinstance(fields, str): + actual = z[0:2, fields] + assert_array_equal(expect, actual) + elif len(fields) == 2: + actual = z[0:2, fields[0], fields[1]] + assert_array_equal(expect, actual) + + # basic selection with single item + expect = a[fields][1] + actual = z.get_basic_selection(1, fields=fields) + assert_array_equal(expect, actual) + # alternative API + if isinstance(fields, str): + actual = z[1, fields] + assert_array_equal(expect, actual) + elif len(fields) == 2: + actual = z[1, fields[0], fields[1]] + assert_array_equal(expect, actual) + + # orthogonal selection + ix = [0, 2] + expect = a[fields][ix] + actual = z.get_orthogonal_selection(ix, fields=fields) + assert_array_equal(expect, actual) + # alternative API + if isinstance(fields, str): + actual = z.oindex[ix, fields] + assert_array_equal(expect, actual) + elif len(fields) == 2: + actual = z.oindex[ix, fields[0], fields[1]] + assert_array_equal(expect, actual) + + # coordinate selection + ix = [0, 2] + expect = a[fields][ix] + actual = z.get_coordinate_selection(ix, fields=fields) + assert_array_equal(expect, actual) + # alternative API + if isinstance(fields, str): + actual = z.vindex[ix, fields] + assert_array_equal(expect, actual) + elif len(fields) == 2: + actual = z.vindex[ix, fields[0], fields[1]] + assert_array_equal(expect, actual) + + # mask selection + ix = [True, False, True] + expect = a[fields][ix] + actual = z.get_mask_selection(ix, fields=fields) + assert_array_equal(expect, actual) + # alternative API + if isinstance(fields, str): + actual = z.vindex[ix, fields] + assert_array_equal(expect, actual) + elif len(fields) == 2: + actual = z.vindex[ix, fields[0], fields[1]] + assert_array_equal(expect, actual) + + # missing/bad fields + with pytest.raises(IndexError): + z.get_basic_selection(Ellipsis, fields=["notafield"]) + with pytest.raises(IndexError): + z.get_basic_selection(Ellipsis, fields=slice(None)) # type: ignore[arg-type] + + +@pytest.mark.xfail(reason="fields are not supported in v3") +def test_set_selections_with_fields(store: StorePath) -> None: + v = np.array( + [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)], + dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")], + ) + a = np.empty_like(v) + z = zarr_array_from_numpy_array(store, v, chunk_shape=(2,)) + + fields_fixture: list[str | list[str]] = [ + "foo", + [], + ["foo"], + ["foo", "bar"], + ["foo", "baz"], + ["bar", "baz"], + ["foo", "bar", "baz"], + ["bar", "foo"], + ["baz", "bar", "foo"], + ] + + for fields in fields_fixture: + # currently multi-field assignment is not supported in numpy, so we won't support + # it either + if isinstance(fields, list) and len(fields) > 1: + with pytest.raises(IndexError): + z.set_basic_selection(Ellipsis, v, fields=fields) + with pytest.raises(IndexError): + z.set_orthogonal_selection([0, 2], v, fields=fields) # type: ignore[arg-type] + with pytest.raises(IndexError): + z.set_coordinate_selection([0, 2], v, fields=fields) + with pytest.raises(IndexError): + z.set_mask_selection([True, False, True], v, fields=fields) # type: ignore[arg-type] + + else: + if isinstance(fields, list) and len(fields) == 1: + # work around numpy does not support multi-field assignment even if there + # is only one field + key = fields[0] + elif isinstance(fields, list) and len(fields) == 0: + # work around numpy ambiguity about what is a field selection + key = Ellipsis + else: + key = fields + + # setup expectation + a[:] = ("", 0, 0) + z[:] = ("", 0, 0) + assert_array_equal(a, z[:]) + a[key] = v[key] + # total selection + z.set_basic_selection(Ellipsis, v[key], fields=fields) + assert_array_equal(a, z[:]) + + # basic selection with slice + a[:] = ("", 0, 0) + z[:] = ("", 0, 0) + a[key][0:2] = v[key][0:2] + z.set_basic_selection(slice(0, 2), v[key][0:2], fields=fields) + assert_array_equal(a, z[:]) + + # orthogonal selection + a[:] = ("", 0, 0) + z[:] = ("", 0, 0) + ix = [0, 2] + a[key][ix] = v[key][ix] + z.set_orthogonal_selection(ix, v[key][ix], fields=fields) + assert_array_equal(a, z[:]) + + # coordinate selection + a[:] = ("", 0, 0) + z[:] = ("", 0, 0) + ix = [0, 2] + a[key][ix] = v[key][ix] + z.set_coordinate_selection(ix, v[key][ix], fields=fields) + assert_array_equal(a, z[:]) + + # mask selection + a[:] = ("", 0, 0) + z[:] = ("", 0, 0) + ix = [True, False, True] + a[key][ix] = v[key][ix] + z.set_mask_selection(ix, v[key][ix], fields=fields) + assert_array_equal(a, z[:]) + + +def test_slice_selection_uints() -> None: + arr = np.arange(24).reshape((4, 6)) + idx = np.uint64(3) + slice_sel = make_slice_selection((idx,)) + assert arr[tuple(slice_sel)].shape == (1, 6) + + +def test_numpy_int_indexing(store: StorePath) -> None: + a = np.arange(1050) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) + assert a[42] == z[42] + assert a[np.int64(42)] == z[np.int64(42)] + + +@pytest.mark.parametrize( + ("shape", "chunks", "ops"), + [ + # 1D test cases + ((1070,), (50,), [("__getitem__", (slice(200, 400),))]), + ((1070,), (50,), [("__getitem__", (slice(200, 400, 100),))]), + ( + (1070,), + (50,), + [ + ("__getitem__", (slice(200, 400),)), + ("__setitem__", (slice(200, 400, 100),)), + ], + ), + # 2D test cases + ( + (40, 50), + (5, 8), + [ + ("__getitem__", (slice(6, 37, 13), (slice(4, 10)))), + ("__setitem__", (slice(None), (slice(None)))), + ], + ), + ], +) +async def test_accessed_chunks( + shape: tuple[int, ...], chunks: tuple[int, ...], ops: list[tuple[str, tuple[slice, ...]]] +) -> None: + # Test that only the required chunks are accessed during basic selection operations + # shape: array shape + # chunks: chunk size + # ops: list of tuples with (optype, tuple of slices) + # optype = "__getitem__" or "__setitem__", tuple length must match number of dims + + # Use a counting dict as the backing store so we can track the items access + store = await CountingDict.open() + z = zarr_array_from_numpy_array(StorePath(store), np.zeros(shape), chunk_shape=chunks) + + for ii, (optype, slices) in enumerate(ops): + # Resolve the slices into the accessed chunks for each dimension + chunks_per_dim = [] + for N, C, sl in zip(shape, chunks, slices, strict=True): + chunk_ind = np.arange(N, dtype=int)[sl] // C + chunks_per_dim.append(np.unique(chunk_ind)) + + # Combine and generate the cartesian product to determine the chunks keys that + # will be accessed + chunks_accessed = [".".join(map(str, comb)) for comb in itertools.product(*chunks_per_dim)] + + counts_before = store.counter.copy() + + # Perform the operation + if optype == "__getitem__": + z[slices] + else: + z[slices] = ii + + # Get the change in counts + delta_counts = store.counter - counts_before + + # Check that the access counts for the operation have increased by one for all + # the chunks we expect to be included + for ci in chunks_accessed: + assert delta_counts.pop((optype, ci)) == 1 + + # If the chunk was partially written to it will also have been read once. We + # don't determine if the chunk was actually partial here, just that the + # counts are consistent that this might have happened + if optype == "__setitem__": + assert ("__getitem__", ci) not in delta_counts or delta_counts.pop( + ("__getitem__", ci) + ) == 1 + # Check that no other chunks were accessed + assert len(delta_counts) == 0 + + +@pytest.mark.parametrize( + "selection", + [ + # basic selection + [...], + [1, ...], + [slice(None)], + [1, 3], + [[1, 2, 3], 9], + [np.arange(1000)], + [slice(5, 15)], + [slice(2, 4), 4], + [[1, 3]], + # mask selection + [np.tile([True, False], (1000, 5))], + [np.full((1000, 10), False)], + # coordinate selection + [[1, 2, 3, 4], [5, 6, 7, 8]], + [[100, 200, 300], [4, 5, 6]], + ], +) +def test_indexing_equals_numpy(store: StorePath, selection: Selection) -> None: + a = np.arange(10000, dtype=int).reshape(1000, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + # note: in python 3.10 a[*selection] is not valid unpacking syntax + expected = a[*selection,] + actual = z[*selection,] + assert_array_equal(expected, actual, err_msg=f"selection: {selection}") + + +@pytest.mark.parametrize( + "selection", + [ + [np.tile([True, False], 500), np.tile([True, False], 5)], + [np.full(1000, False), np.tile([True, False], 5)], + [np.full(1000, True), np.full(10, True)], + [np.full(1000, True), [True, False] * 5], + ], +) +def test_orthogonal_bool_indexing_like_numpy_ix( + store: StorePath, selection: list[npt.ArrayLike] +) -> None: + a = np.arange(10000, dtype=int).reshape(1000, 10) + z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) + expected = a[np.ix_(*selection)] + # note: in python 3.10 z[*selection] is not valid unpacking syntax + actual = z[*selection,] + assert_array_equal(expected, actual, err_msg=f"{selection=}") + + +@pytest.mark.parametrize("ndim", [1, 2, 3]) +@pytest.mark.parametrize("origin_0d", [None, (0,), (1,)]) +@pytest.mark.parametrize("selection_shape_0d", [None, (2,), (3,)]) +def test_iter_grid( + ndim: int, origin_0d: tuple[int] | None, selection_shape_0d: tuple[int] | None +) -> None: + """ + Test that iter_grid works as expected for 1, 2, and 3 dimensions. + """ + grid_shape = (10, 5, 7)[:ndim] + + if origin_0d is not None: + origin_kwarg = origin_0d * ndim + origin = origin_kwarg + else: + origin_kwarg = None + origin = (0,) * ndim + + if selection_shape_0d is not None: + selection_shape_kwarg = selection_shape_0d * ndim + selection_shape = selection_shape_kwarg + else: + selection_shape_kwarg = None + selection_shape = tuple(gs - o for gs, o in zip(grid_shape, origin, strict=False)) + + observed = tuple( + _iter_grid(grid_shape, origin=origin_kwarg, selection_shape=selection_shape_kwarg) + ) + + # generate a numpy array of indices, and index it + coord_array = np.array(list(itertools.product(*[range(s) for s in grid_shape]))).reshape( + (*grid_shape, ndim) + ) + coord_array_indexed = coord_array[ + tuple(slice(o, o + s, 1) for o, s in zip(origin, selection_shape, strict=False)) + + (range(ndim),) + ] + + expected = tuple(map(tuple, coord_array_indexed.reshape(-1, ndim).tolist())) + assert observed == expected + + +def test_iter_grid_invalid() -> None: + """ + Ensure that a selection_shape that exceeds the grid_shape + origin produces an indexing error. + """ + with pytest.raises(IndexError): + list(_iter_grid((5,), origin=(0,), selection_shape=(10,))) + + +def test_indexing_with_zarr_array(store: StorePath) -> None: + # regression test for https://github.com/zarr-developers/zarr-python/issues/2133 + a = np.arange(10) + za = zarr.array(a, chunks=2, store=store, path="a") + ix = [False, True, False, True, False, True, False, True, False, True] + ii = [0, 2, 4, 5] + + zix = zarr.array(ix, chunks=2, store=store, dtype="bool", path="ix") + zii = zarr.array(ii, chunks=2, store=store, dtype="i4", path="ii") + assert_array_equal(a[ix], za[zix]) + assert_array_equal(a[ix], za.oindex[zix]) + assert_array_equal(a[ix], za.vindex[zix]) + + assert_array_equal(a[ii], za[zii]) + assert_array_equal(a[ii], za.oindex[zii]) + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +@pytest.mark.parametrize("shape", [(0, 2, 3), (0), (3, 0)]) +def test_zero_sized_chunks(store: StorePath, shape: list[int]) -> None: + z = zarr.create_array(store=store, shape=shape, chunks=shape, zarr_format=3, dtype="f8") + z[...] = 42 + assert_array_equal(z[...], np.zeros(shape, dtype="f8")) + + +@pytest.mark.parametrize("store", ["memory"], indirect=["store"]) +def test_vectorized_indexing_incompatible_shape(store) -> None: + # GH2469 + shape = (4, 4) + chunks = (2, 2) + fill_value = 32767 + arr = zarr.create( + shape, + store=store, + chunks=chunks, + dtype=np.int16, + fill_value=fill_value, + codecs=[zarr.codecs.BytesCodec(), zarr.codecs.BloscCodec()], + ) + with pytest.raises(ValueError, match="Attempting to set"): + arr[np.array([1, 2]), np.array([1, 2])] = np.array([[-1, -2], [-3, -4]]) + + +def test_iter_chunk_regions(): + chunks = (2, 3) + a = zarr.create((10, 10), chunks=chunks) + a[:] = 1 + for region in a._iter_chunk_regions(): + assert_array_equal(a[region], np.ones_like(a[region])) + a[region] = 0 + assert_array_equal(a[region], np.zeros_like(a[region])) diff --git a/tests/test_info.py b/tests/test_info.py new file mode 100644 index 0000000000..0abaff9ae7 --- /dev/null +++ b/tests/test_info.py @@ -0,0 +1,141 @@ +import textwrap + +import pytest + +from zarr.codecs.bytes import BytesCodec +from zarr.core._info import ArrayInfo, GroupInfo, human_readable_size +from zarr.core.common import ZarrFormat +from zarr.core.dtype.npy.int import Int32 + +ZARR_FORMATS = [2, 3] + + +@pytest.mark.parametrize("zarr_format", ZARR_FORMATS) +def test_group_info_repr(zarr_format: ZarrFormat) -> None: + info = GroupInfo( + _name="a", _store_type="MemoryStore", _read_only=False, _zarr_format=zarr_format + ) + result = repr(info) + expected = textwrap.dedent(f"""\ + Name : a + Type : Group + Zarr format : {zarr_format} + Read-only : False + Store type : MemoryStore""") + assert result == expected + + +@pytest.mark.parametrize("zarr_format", ZARR_FORMATS) +def test_group_info_complete(zarr_format: ZarrFormat) -> None: + info = GroupInfo( + _name="a", + _store_type="MemoryStore", + _zarr_format=zarr_format, + _read_only=False, + _count_arrays=10, + _count_groups=4, + _count_members=14, + ) + result = repr(info) + expected = textwrap.dedent(f"""\ + Name : a + Type : Group + Zarr format : {zarr_format} + Read-only : False + Store type : MemoryStore + No. members : 14 + No. arrays : 10 + No. groups : 4""") + assert result == expected + + +@pytest.mark.parametrize("zarr_format", ZARR_FORMATS) +def test_array_info(zarr_format: ZarrFormat) -> None: + info = ArrayInfo( + _zarr_format=zarr_format, + _data_type=Int32(), + _fill_value=0, + _shape=(100, 100), + _chunk_shape=(10, 100), + _order="C", + _read_only=True, + _store_type="MemoryStore", + _serializer=BytesCodec(), + ) + result = repr(info) + assert result == textwrap.dedent(f"""\ + Type : Array + Zarr format : {zarr_format} + Data type : Int32(endianness='little') + Fill value : 0 + Shape : (100, 100) + Chunk shape : (10, 100) + Order : C + Read-only : True + Store type : MemoryStore + Filters : () + Serializer : BytesCodec(endian=) + Compressors : ()""") + + +@pytest.mark.parametrize("zarr_format", ZARR_FORMATS) +@pytest.mark.parametrize("bytes_things", [(1_000_000, "976.6K", 500_000, "500000", "2.0", 5)]) +def test_array_info_complete( + zarr_format: ZarrFormat, bytes_things: tuple[int, str, int, str, str, int] +) -> None: + ( + count_bytes, + count_bytes_formatted, + count_bytes_stored, + count_bytes_stored_formatted, + storage_ratio_formatted, + count_chunks_initialized, + ) = bytes_things + info = ArrayInfo( + _zarr_format=zarr_format, + _data_type=Int32(), + _fill_value=0, + _shape=(100, 100), + _chunk_shape=(10, 100), + _order="C", + _read_only=True, + _store_type="MemoryStore", + _serializer=BytesCodec(), + _count_bytes=count_bytes, + _count_bytes_stored=count_bytes_stored, + _count_chunks_initialized=count_chunks_initialized, + ) + result = repr(info) + assert result == textwrap.dedent(f"""\ + Type : Array + Zarr format : {zarr_format} + Data type : Int32(endianness='little') + Fill value : 0 + Shape : (100, 100) + Chunk shape : (10, 100) + Order : C + Read-only : True + Store type : MemoryStore + Filters : () + Serializer : BytesCodec(endian=) + Compressors : () + No. bytes : {count_bytes} ({count_bytes_formatted}) + No. bytes stored : {count_bytes_stored_formatted} + Storage ratio : {storage_ratio_formatted} + Chunks Initialized : 5""") + + +@pytest.mark.parametrize( + ("size", "expected"), + [ + (1, "1"), + (2**10, "1.0K"), + (2**20, "1.0M"), + (2**30, "1.0G"), + (2**40, "1.0T"), + (2**50, "1.0P"), + ], +) +def test_human_readable_size(size: int, expected: str) -> None: + result = human_readable_size(size) + assert result == expected diff --git a/tests/test_metadata/__init__.py b/tests/test_metadata/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py new file mode 100644 index 0000000000..395e036db2 --- /dev/null +++ b/tests/test_metadata/test_consolidated.py @@ -0,0 +1,681 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import numpy as np +import pytest +from numcodecs import Blosc + +import zarr.api.asynchronous +import zarr.api.synchronous +import zarr.storage +from zarr.api.asynchronous import ( + AsyncGroup, + consolidate_metadata, + group, + open, + open_consolidated, +) +from zarr.core.buffer import cpu, default_buffer_prototype +from zarr.core.dtype import parse_data_type +from zarr.core.group import ConsolidatedMetadata, GroupMetadata +from zarr.core.metadata import ArrayV3Metadata +from zarr.core.metadata.v2 import ArrayV2Metadata +from zarr.storage import StorePath + +if TYPE_CHECKING: + from zarr.abc.store import Store + from zarr.core.common import ZarrFormat + + +@pytest.fixture +async def memory_store_with_hierarchy(memory_store: Store) -> None: + g = await group(store=memory_store, attributes={"foo": "bar"}) + dtype = "uint8" + await g.create_array(name="air", shape=(1, 2, 3), dtype=dtype) + await g.create_array(name="lat", shape=(1,), dtype=dtype) + await g.create_array(name="lon", shape=(2,), dtype=dtype) + await g.create_array(name="time", shape=(3,), dtype=dtype) + + child = await g.create_group("child", attributes={"key": "child"}) + await child.create_array("array", shape=(4, 4), attributes={"key": "child"}, dtype=dtype) + + grandchild = await child.create_group("grandchild", attributes={"key": "grandchild"}) + await grandchild.create_array( + "array", shape=(4, 4), attributes={"key": "grandchild"}, dtype=dtype + ) + await grandchild.create_group("empty_group", attributes={"key": "empty"}) + return memory_store + + +class TestConsolidated: + async def test_open_consolidated_false_raises(self): + store = zarr.storage.MemoryStore() + with pytest.raises(TypeError, match="use_consolidated"): + await zarr.api.asynchronous.open_consolidated(store, use_consolidated=False) + + def test_open_consolidated_false_raises_sync(self): + store = zarr.storage.MemoryStore() + with pytest.raises(TypeError, match="use_consolidated"): + zarr.open_consolidated(store, use_consolidated=False) + + async def test_consolidated(self, memory_store_with_hierarchy: Store) -> None: + # TODO: Figure out desired keys in + # TODO: variety in the hierarchies + # More nesting + # arrays under arrays + # single array + # etc. + await consolidate_metadata(memory_store_with_hierarchy) + group2 = await AsyncGroup.open(memory_store_with_hierarchy) + + array_metadata = { + "attributes": {}, + "chunk_key_encoding": { + "configuration": {"separator": "/"}, + "name": "default", + }, + "codecs": ( + {"configuration": {"endian": "little"}, "name": "bytes"}, + {"configuration": {"level": 0, "checksum": False}, "name": "zstd"}, + ), + "data_type": "uint8", + "fill_value": 0, + "node_type": "array", + # "shape": (1, 2, 3), + "zarr_format": 3, + } + + expected = GroupMetadata( + attributes={"foo": "bar"}, + consolidated_metadata=ConsolidatedMetadata( + kind="inline", + must_understand=False, + metadata={ + "air": ArrayV3Metadata.from_dict( + { + "shape": (1, 2, 3), + "chunk_grid": { + "configuration": {"chunk_shape": (1, 2, 3)}, + "name": "regular", + }, + **array_metadata, + } + ), + "lat": ArrayV3Metadata.from_dict( + { + "shape": (1,), + "chunk_grid": { + "configuration": {"chunk_shape": (1,)}, + "name": "regular", + }, + **array_metadata, + } + ), + "lon": ArrayV3Metadata.from_dict( + { + "shape": (2,), + "chunk_grid": { + "configuration": {"chunk_shape": (2,)}, + "name": "regular", + }, + **array_metadata, + } + ), + "time": ArrayV3Metadata.from_dict( + { + "shape": (3,), + "chunk_grid": { + "configuration": {"chunk_shape": (3,)}, + "name": "regular", + }, + **array_metadata, + } + ), + "child": GroupMetadata( + attributes={"key": "child"}, + consolidated_metadata=ConsolidatedMetadata( + metadata={ + "array": ArrayV3Metadata.from_dict( + { + **array_metadata, + "attributes": {"key": "child"}, + "shape": (4, 4), + "chunk_grid": { + "configuration": {"chunk_shape": (4, 4)}, + "name": "regular", + }, + } + ), + "grandchild": GroupMetadata( + attributes={"key": "grandchild"}, + consolidated_metadata=ConsolidatedMetadata( + metadata={ + # known to be empty child group + "empty_group": GroupMetadata( + consolidated_metadata=ConsolidatedMetadata( + metadata={} + ), + attributes={"key": "empty"}, + ), + "array": ArrayV3Metadata.from_dict( + { + **array_metadata, + "attributes": {"key": "grandchild"}, + "shape": (4, 4), + "chunk_grid": { + "configuration": {"chunk_shape": (4, 4)}, + "name": "regular", + }, + } + ), + } + ), + ), + }, + ), + ), + }, + ), + ) + + assert group2.metadata == expected + group3 = await open(store=memory_store_with_hierarchy) + assert group3.metadata == expected + + group4 = await open_consolidated(store=memory_store_with_hierarchy) + assert group4.metadata == expected + + result_raw = json.loads( + ( + await memory_store_with_hierarchy.get( + "zarr.json", prototype=default_buffer_prototype() + ) + ).to_bytes() + )["consolidated_metadata"] + assert result_raw["kind"] == "inline" + assert sorted(result_raw["metadata"]) == [ + "air", + "child", + "child/array", + "child/grandchild", + "child/grandchild/array", + "child/grandchild/empty_group", + "lat", + "lon", + "time", + ] + + def test_consolidated_sync(self, memory_store): + g = zarr.api.synchronous.group(store=memory_store, attributes={"foo": "bar"}) + dtype = "uint8" + g.create_array(name="air", shape=(1, 2, 3), dtype=dtype) + g.create_array(name="lat", shape=(1,), dtype=dtype) + g.create_array(name="lon", shape=(2,), dtype=dtype) + g.create_array(name="time", shape=(3,), dtype=dtype) + + zarr.api.synchronous.consolidate_metadata(memory_store) + group2 = zarr.api.synchronous.Group.open(memory_store) + + array_metadata = { + "attributes": {}, + "chunk_key_encoding": { + "configuration": {"separator": "/"}, + "name": "default", + }, + "codecs": ( + {"configuration": {"endian": "little"}, "name": "bytes"}, + {"configuration": {"level": 0, "checksum": False}, "name": "zstd"}, + ), + "data_type": dtype, + "fill_value": 0, + "node_type": "array", + # "shape": (1, 2, 3), + "zarr_format": 3, + } + + expected = GroupMetadata( + attributes={"foo": "bar"}, + consolidated_metadata=ConsolidatedMetadata( + kind="inline", + must_understand=False, + metadata={ + "air": ArrayV3Metadata.from_dict( + { + "shape": (1, 2, 3), + "chunk_grid": { + "configuration": {"chunk_shape": (1, 2, 3)}, + "name": "regular", + }, + **array_metadata, + } + ), + "lat": ArrayV3Metadata.from_dict( + { + "shape": (1,), + "chunk_grid": { + "configuration": {"chunk_shape": (1,)}, + "name": "regular", + }, + **array_metadata, + } + ), + "lon": ArrayV3Metadata.from_dict( + { + "shape": (2,), + "chunk_grid": { + "configuration": {"chunk_shape": (2,)}, + "name": "regular", + }, + **array_metadata, + } + ), + "time": ArrayV3Metadata.from_dict( + { + "shape": (3,), + "chunk_grid": { + "configuration": {"chunk_shape": (3,)}, + "name": "regular", + }, + **array_metadata, + } + ), + }, + ), + ) + assert group2.metadata == expected + group3 = zarr.api.synchronous.open(store=memory_store) + assert group3.metadata == expected + + group4 = zarr.api.synchronous.open_consolidated(store=memory_store) + assert group4.metadata == expected + + async def test_not_writable_raises(self, memory_store: zarr.storage.MemoryStore) -> None: + await group(store=memory_store, attributes={"foo": "bar"}) + read_store = zarr.storage.MemoryStore(store_dict=memory_store._store_dict, read_only=True) + with pytest.raises(ValueError, match="does not support writing"): + await consolidate_metadata(read_store) + + async def test_non_root_node(self, memory_store_with_hierarchy: Store) -> None: + await consolidate_metadata(memory_store_with_hierarchy, path="child") + root = await AsyncGroup.open(memory_store_with_hierarchy) + child = await AsyncGroup.open(StorePath(memory_store_with_hierarchy) / "child") + + assert root.metadata.consolidated_metadata is None + assert child.metadata.consolidated_metadata is not None + assert "air" not in child.metadata.consolidated_metadata.metadata + assert "grandchild" in child.metadata.consolidated_metadata.metadata + + def test_consolidated_metadata_from_dict(self): + data = {"must_understand": False} + + # missing kind + with pytest.raises(ValueError, match="kind='None'"): + ConsolidatedMetadata.from_dict(data) + + # invalid kind + data["kind"] = "invalid" + with pytest.raises(ValueError, match="kind='invalid'"): + ConsolidatedMetadata.from_dict(data) + + # missing metadata + data["kind"] = "inline" + + with pytest.raises(TypeError, match="Unexpected type for 'metadata'"): + ConsolidatedMetadata.from_dict(data) + + data["kind"] = "inline" + # empty is fine + data["metadata"] = {} + ConsolidatedMetadata.from_dict(data) + + def test_flatten(self): + array_metadata = { + "attributes": {}, + "chunk_key_encoding": { + "configuration": {"separator": "/"}, + "name": "default", + }, + "codecs": ({"configuration": {"endian": "little"}, "name": "bytes"},), + "data_type": "float64", + "fill_value": np.float64(0.0), + "node_type": "array", + # "shape": (1, 2, 3), + "zarr_format": 3, + } + + metadata = ConsolidatedMetadata( + kind="inline", + must_understand=False, + metadata={ + "air": ArrayV3Metadata.from_dict( + { + "shape": (1, 2, 3), + "chunk_grid": { + "configuration": {"chunk_shape": (1, 2, 3)}, + "name": "regular", + }, + **array_metadata, + } + ), + "lat": ArrayV3Metadata.from_dict( + { + "shape": (1,), + "chunk_grid": { + "configuration": {"chunk_shape": (1,)}, + "name": "regular", + }, + **array_metadata, + } + ), + "child": GroupMetadata( + attributes={"key": "child"}, + consolidated_metadata=ConsolidatedMetadata( + metadata={ + "array": ArrayV3Metadata.from_dict( + { + **array_metadata, + "attributes": {"key": "child"}, + "shape": (4, 4), + "chunk_grid": { + "configuration": {"chunk_shape": (4, 4)}, + "name": "regular", + }, + } + ), + "grandchild": GroupMetadata( + attributes={"key": "grandchild"}, + consolidated_metadata=ConsolidatedMetadata( + metadata={ + "array": ArrayV3Metadata.from_dict( + { + **array_metadata, + "attributes": {"key": "grandchild"}, + "shape": (4, 4), + "chunk_grid": { + "configuration": {"chunk_shape": (4, 4)}, + "name": "regular", + }, + } + ) + } + ), + ), + }, + ), + ), + }, + ) + result = metadata.flattened_metadata + expected = { + "air": metadata.metadata["air"], + "lat": metadata.metadata["lat"], + "child": GroupMetadata( + attributes={"key": "child"}, consolidated_metadata=ConsolidatedMetadata(metadata={}) + ), + "child/array": metadata.metadata["child"].consolidated_metadata.metadata["array"], + "child/grandchild": GroupMetadata( + attributes={"key": "grandchild"}, + consolidated_metadata=ConsolidatedMetadata(metadata={}), + ), + "child/grandchild/array": ( + metadata.metadata["child"] + .consolidated_metadata.metadata["grandchild"] + .consolidated_metadata.metadata["array"] + ), + } + assert result == expected + + def test_invalid_metadata_raises(self): + payload = { + "kind": "inline", + "must_understand": False, + "metadata": { + "foo": [1, 2, 3] # invalid + }, + } + + with pytest.raises(TypeError, match="key='foo', type='list'"): + ConsolidatedMetadata.from_dict(payload) + + def test_to_dict_empty(self): + meta = ConsolidatedMetadata( + metadata={ + "empty": GroupMetadata( + attributes={"key": "empty"}, + consolidated_metadata=ConsolidatedMetadata(metadata={}), + ) + } + ) + result = meta.to_dict() + expected = { + "kind": "inline", + "must_understand": False, + "metadata": { + "empty": { + "attributes": {"key": "empty"}, + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {}, + }, + "node_type": "group", + "zarr_format": 3, + } + }, + } + assert result == expected + + @pytest.mark.parametrize("zarr_format", [2, 3]) + async def test_open_consolidated_raises_async(self, zarr_format: ZarrFormat): + store = zarr.storage.MemoryStore() + await AsyncGroup.from_store(store, zarr_format=zarr_format) + with pytest.raises(ValueError): + await zarr.api.asynchronous.open_consolidated(store, zarr_format=zarr_format) + + with pytest.raises(ValueError): + await zarr.api.asynchronous.open_consolidated(store, zarr_format=None) + + @pytest.fixture + async def v2_consolidated_metadata_empty_dataset( + self, memory_store: zarr.storage.MemoryStore + ) -> AsyncGroup: + zgroup_bytes = cpu.Buffer.from_bytes(json.dumps({"zarr_format": 2}).encode()) + zmetadata_bytes = cpu.Buffer.from_bytes( + b'{"metadata":{".zgroup":{"zarr_format":2}},"zarr_consolidated_format":1}' + ) + return AsyncGroup._from_bytes_v2( + None, zgroup_bytes, zattrs_bytes=None, consolidated_metadata_bytes=zmetadata_bytes + ) + + async def test_consolidated_metadata_backwards_compatibility( + self, v2_consolidated_metadata_empty_dataset + ): + """ + Test that consolidated metadata handles a missing .zattrs key. This is necessary for backwards compatibility with zarr-python 2.x. See https://github.com/zarr-developers/zarr-python/issues/2694 + """ + store = zarr.storage.MemoryStore() + await zarr.api.asynchronous.open(store=store, zarr_format=2) + await zarr.api.asynchronous.consolidate_metadata(store) + result = await zarr.api.asynchronous.open_consolidated(store, zarr_format=2) + assert result.metadata == v2_consolidated_metadata_empty_dataset.metadata + + async def test_consolidated_metadata_v2(self): + store = zarr.storage.MemoryStore() + g = await AsyncGroup.from_store(store, attributes={"key": "root"}, zarr_format=2) + dtype = parse_data_type("uint8", zarr_format=2) + await g.create_array(name="a", shape=(1,), attributes={"key": "a"}, dtype=dtype) + g1 = await g.create_group(name="g1", attributes={"key": "g1"}) + await g1.create_group(name="g2", attributes={"key": "g2"}) + + await zarr.api.asynchronous.consolidate_metadata(store) + result = await zarr.api.asynchronous.open_consolidated(store, zarr_format=2) + + expected = GroupMetadata( + attributes={"key": "root"}, + zarr_format=2, + consolidated_metadata=ConsolidatedMetadata( + metadata={ + "a": ArrayV2Metadata( + shape=(1,), + dtype=dtype, + attributes={"key": "a"}, + chunks=(1,), + fill_value=0, + compressor=Blosc(), + order="C", + ), + "g1": GroupMetadata( + attributes={"key": "g1"}, + zarr_format=2, + consolidated_metadata=ConsolidatedMetadata( + metadata={ + "g2": GroupMetadata( + attributes={"key": "g2"}, + zarr_format=2, + consolidated_metadata=ConsolidatedMetadata(metadata={}), + ) + } + ), + ), + } + ), + ) + assert result.metadata == expected + + @pytest.mark.parametrize("zarr_format", [2, 3]) + async def test_use_consolidated_false( + self, memory_store: zarr.storage.MemoryStore, zarr_format: ZarrFormat + ) -> None: + with zarr.config.set(default_zarr_format=zarr_format): + g = await group(store=memory_store, attributes={"foo": "bar"}) + await g.create_group(name="a") + + # test a stale read + await zarr.api.asynchronous.consolidate_metadata(memory_store) + await g.create_group(name="b") + + stale = await zarr.api.asynchronous.open_group(store=memory_store) + assert len([x async for x in stale.members()]) == 1 + assert stale.metadata.consolidated_metadata + assert list(stale.metadata.consolidated_metadata.metadata) == ["a"] + + # bypass stale data + good = await zarr.api.asynchronous.open_group( + store=memory_store, use_consolidated=False + ) + assert len([x async for x in good.members()]) == 2 + + # reconsolidate + await zarr.api.asynchronous.consolidate_metadata(memory_store) + + good = await zarr.api.asynchronous.open_group(store=memory_store) + assert len([x async for x in good.members()]) == 2 + assert good.metadata.consolidated_metadata + assert sorted(good.metadata.consolidated_metadata.metadata) == ["a", "b"] + + async def test_stale_child_metadata_ignored(self, memory_store: zarr.storage.MemoryStore): + # https://github.com/zarr-developers/zarr-python/issues/2921 + # When consolidating metadata, we should ignore any (possibly stale) metadata + # from previous consolidations, *including at child nodes*. + root = await zarr.api.asynchronous.group(store=memory_store, zarr_format=3) + await root.create_group("foo") + await zarr.api.asynchronous.consolidate_metadata(memory_store, path="foo") + await root.create_group("foo/bar/spam") + + await zarr.api.asynchronous.consolidate_metadata(memory_store) + + reopened = await zarr.api.asynchronous.open_consolidated(store=memory_store, zarr_format=3) + result = [x[0] async for x in reopened.members(max_depth=None)] + expected = ["foo", "foo/bar", "foo/bar/spam"] + assert result == expected + + async def test_use_consolidated_for_children_members( + self, memory_store: zarr.storage.MemoryStore + ): + # A test that has *unconsolidated* metadata at the root group, but discovers + # a child group with consolidated metadata. + + root = await zarr.api.asynchronous.create_group(store=memory_store) + await root.create_group("a/b") + # Consolidate metadata at "a/b" + await zarr.api.asynchronous.consolidate_metadata(memory_store, path="a/b") + + # Add a new group a/b/c, that's not present in the CM at "a/b" + await root.create_group("a/b/c") + + # Now according to the consolidated metadata, "a" has children ["b"] + # but according to the unconsolidated metadata, "a" has children ["b", "c"] + group = await zarr.api.asynchronous.open_group(store=memory_store, path="a") + with pytest.warns(UserWarning, match="Object at 'c' not found"): + result = sorted([x[0] async for x in group.members(max_depth=None)]) + expected = ["b"] + assert result == expected + + result = sorted( + [x[0] async for x in group.members(max_depth=None, use_consolidated_for_children=False)] + ) + expected = ["b", "b/c"] + assert result == expected + + +@pytest.mark.parametrize("fill_value", [np.nan, np.inf, -np.inf]) +async def test_consolidated_metadata_encodes_special_chars( + memory_store: Store, zarr_format: ZarrFormat, fill_value: float +): + root = await group(store=memory_store, zarr_format=zarr_format) + _time = await root.create_array("time", shape=(12,), dtype=np.float64, fill_value=fill_value) + await zarr.api.asynchronous.consolidate_metadata(memory_store) + + root = await group(store=memory_store, zarr_format=zarr_format) + root_buffer = root.metadata.to_buffer_dict(default_buffer_prototype()) + + if zarr_format == 2: + root_metadata = json.loads(root_buffer[".zmetadata"].to_bytes().decode("utf-8"))["metadata"] + elif zarr_format == 3: + root_metadata = json.loads(root_buffer["zarr.json"].to_bytes().decode("utf-8"))[ + "consolidated_metadata" + ]["metadata"] + + expected_fill_value = _time._zdtype.to_json_scalar(fill_value, zarr_format=2) + + if zarr_format == 2: + assert root_metadata["time/.zarray"]["fill_value"] == expected_fill_value + elif zarr_format == 3: + assert root_metadata["time"]["fill_value"] == expected_fill_value + + +class NonConsolidatedStore(zarr.storage.MemoryStore): + """A store that doesn't support consolidated metadata""" + + @property + def supports_consolidated_metadata(self) -> bool: + return False + + +async def test_consolidate_metadata_raises_for_self_consolidating_stores(): + """Verify calling consolidate_metadata on a non supporting stores raises an error.""" + + memory_store = NonConsolidatedStore() + root = await zarr.api.asynchronous.create_group(store=memory_store) + await root.create_group("a/b") + + with pytest.raises(TypeError, match="doesn't support consolidated metadata"): + await zarr.api.asynchronous.consolidate_metadata(memory_store) + + +async def test_open_group_in_non_consolidating_stores(): + memory_store = NonConsolidatedStore() + root = await zarr.api.asynchronous.create_group(store=memory_store) + await root.create_group("a/b") + + # Opening a group without consolidatedion works as expected + await AsyncGroup.open(memory_store, use_consolidated=False) + + # let the Store opt out of consolidation + await AsyncGroup.open(memory_store, use_consolidated=None) + + # Opening a group with use_consolidated=True should fail + with pytest.raises(ValueError, match="doesn't support consolidated metadata"): + await AsyncGroup.open(memory_store, use_consolidated=True) diff --git a/tests/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py new file mode 100644 index 0000000000..a2894529aa --- /dev/null +++ b/tests/test_metadata/test_v2.py @@ -0,0 +1,338 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Literal + +import numpy as np +import pytest + +import zarr.api.asynchronous +import zarr.storage +from zarr.core.buffer import cpu +from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.dtype.npy.float import Float32, Float64 +from zarr.core.dtype.npy.int import Int16 +from zarr.core.group import ConsolidatedMetadata, GroupMetadata +from zarr.core.metadata import ArrayV2Metadata +from zarr.core.metadata.v2 import parse_zarr_format + +if TYPE_CHECKING: + from typing import Any + + from zarr.abc.codec import Codec + + +def test_parse_zarr_format_valid() -> None: + assert parse_zarr_format(2) == 2 + + +@pytest.mark.parametrize("data", [None, 1, 3, 4, 5, "3"]) +def test_parse_zarr_format_invalid(data: Any) -> None: + with pytest.raises(ValueError, match=f"Invalid value. Expected 2. Got {data}"): + parse_zarr_format(data) + + +@pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) +@pytest.mark.parametrize("filters", [None, [{"id": "gzip", "level": 1}]]) +@pytest.mark.parametrize("compressor", [None, {"id": "gzip", "level": 1}]) +@pytest.mark.parametrize("fill_value", [None, 0, 1]) +@pytest.mark.parametrize("order", ["C", "F"]) +@pytest.mark.parametrize("dimension_separator", [".", "/", None]) +def test_metadata_to_dict( + compressor: Codec | None, + filters: tuple[Codec] | None, + fill_value: Any, + order: Literal["C", "F"], + dimension_separator: Literal[".", "/"] | None, + attributes: dict[str, Any] | None, +) -> None: + shape = (1, 2, 3) + chunks = (1,) * len(shape) + data_type = "|u1" + metadata_dict = { + "zarr_format": 2, + "shape": shape, + "chunks": chunks, + "dtype": data_type, + "order": order, + "compressor": compressor, + "filters": filters, + "fill_value": fill_value, + } + + if attributes is not None: + metadata_dict["attributes"] = attributes + if dimension_separator is not None: + metadata_dict["dimension_separator"] = dimension_separator + + metadata = ArrayV2Metadata.from_dict(metadata_dict) + observed = metadata.to_dict() + expected = metadata_dict.copy() + + if attributes is None: + assert observed["attributes"] == {} + observed.pop("attributes") + + if dimension_separator is None: + expected_dimension_sep = "." + assert observed["dimension_separator"] == expected_dimension_sep + observed.pop("dimension_separator") + + assert observed == expected + + +def test_filters_empty_tuple_warns() -> None: + metadata_dict = { + "zarr_format": 2, + "shape": (1,), + "chunks": (1,), + "dtype": "|u1", + "order": "C", + "compressor": None, + "filters": (), + "fill_value": 0, + } + with pytest.warns( + UserWarning, match="Found an empty list of filters in the array metadata document." + ): + meta = ArrayV2Metadata.from_dict(metadata_dict) + assert meta.filters is None + + +class TestConsolidated: + @pytest.fixture + async def v2_consolidated_metadata( + self, memory_store: zarr.storage.MemoryStore + ) -> zarr.storage.MemoryStore: + zmetadata = { + "metadata": { + ".zattrs": { + "Conventions": "COARDS", + }, + ".zgroup": {"zarr_format": 2}, + "air/.zarray": { + "chunks": [730], + "compressor": None, + "dtype": " None: + data = { + "_nczarr_array": {"dimrefs": ["/dim1", "/dim2"], "storage": "chunked"}, + "attributes": {"key": "value"}, + "chunks": [8], + "compressor": None, + "dtype": " None: + arr = zarr.create_array( + {}, + shape=(10,), + chunks=(10,), + dtype="int32", + compressors={"id": "zstd", "level": 5, "checksum": False}, + zarr_format=2, + ) + metadata = json.loads( + arr.metadata.to_buffer_dict(default_buffer_prototype())[".zarray"].to_bytes() + ) + assert "checksum" not in metadata["compressor"] + + +@pytest.mark.parametrize("fill_value", [np.void((0, 0), np.dtype([("foo", "i4"), ("bar", "i4")]))]) +def test_structured_dtype_fill_value_serialization(tmp_path, fill_value): + zarr_format = 2 + group_path = tmp_path / "test.zarr" + root_group = zarr.open_group(group_path, mode="w", zarr_format=zarr_format) + dtype = np.dtype([("foo", "i4"), ("bar", "i4")]) + root_group.create_array( + name="structured_dtype", + shape=(100, 100), + chunks=(100, 100), + dtype=dtype, + fill_value=fill_value, + ) + + zarr.consolidate_metadata(root_group.store, zarr_format=zarr_format) + root_group = zarr.open_group(group_path, mode="r") + observed = root_group.metadata.consolidated_metadata.metadata["structured_dtype"].fill_value + assert observed == fill_value diff --git a/tests/test_metadata/test_v3.py b/tests/test_metadata/test_v3.py new file mode 100644 index 0000000000..4f385afa6d --- /dev/null +++ b/tests/test_metadata/test_v3.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +import json +import re +from typing import TYPE_CHECKING, Literal + +import numpy as np +import pytest + +from zarr.codecs.bytes import BytesCodec +from zarr.core.buffer import default_buffer_prototype +from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding +from zarr.core.config import config +from zarr.core.dtype import get_data_type_from_native_dtype +from zarr.core.dtype.npy.string import _NUMPY_SUPPORTS_VLEN_STRING +from zarr.core.dtype.npy.time import DateTime64 +from zarr.core.group import GroupMetadata, parse_node_type +from zarr.core.metadata.v3 import ( + ArrayV3Metadata, + parse_dimension_names, + parse_zarr_format, +) +from zarr.errors import MetadataValidationError, NodeTypeValidationError + +if TYPE_CHECKING: + from collections.abc import Sequence + from typing import Any + + from zarr.abc.codec import Codec + from zarr.core.common import JSON + + +from zarr.core.metadata.v3 import ( + parse_node_type_array, +) + +bool_dtypes = ("bool",) + +int_dtypes = ( + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", +) + +float_dtypes = ( + "float16", + "float32", + "float64", +) + +complex_dtypes = ("complex64", "complex128") +flexible_dtypes = ("str", "bytes", "void") +if _NUMPY_SUPPORTS_VLEN_STRING: + vlen_string_dtypes = ("T",) +else: + vlen_string_dtypes = ("O",) + +dtypes = ( + *bool_dtypes, + *int_dtypes, + *float_dtypes, + *complex_dtypes, + *flexible_dtypes, + *vlen_string_dtypes, +) + + +@pytest.mark.parametrize("data", [None, 1, 2, 4, 5, "3"]) +def test_parse_zarr_format_invalid(data: Any) -> None: + with pytest.raises( + MetadataValidationError, + match=f"Invalid value for 'zarr_format'. Expected '3'. Got '{data}'.", + ): + parse_zarr_format(data) + + +def test_parse_zarr_format_valid() -> None: + assert parse_zarr_format(3) == 3 + + +def test_parse_node_type_valid() -> None: + assert parse_node_type("array") == "array" + assert parse_node_type("group") == "group" + + +@pytest.mark.parametrize("node_type", [None, 2, "other"]) +def test_parse_node_type_invalid(node_type: Any) -> None: + with pytest.raises( + MetadataValidationError, + match=f"Invalid value for 'node_type'. Expected 'array or group'. Got '{node_type}'.", + ): + parse_node_type(node_type) + + +@pytest.mark.parametrize("data", [None, "group"]) +def test_parse_node_type_array_invalid(data: Any) -> None: + with pytest.raises( + NodeTypeValidationError, + match=f"Invalid value for 'node_type'. Expected 'array'. Got '{data}'.", + ): + parse_node_type_array(data) + + +def test_parse_node_typev_array_alid() -> None: + assert parse_node_type_array("array") == "array" + + +@pytest.mark.parametrize("data", [(), [1, 2, "a"], {"foo": 10}]) +def parse_dimension_names_invalid(data: Any) -> None: + with pytest.raises(TypeError, match="Expected either None or iterable of str,"): + parse_dimension_names(data) + + +@pytest.mark.parametrize("data", [None, ("a", "b", "c"), ["a", "a", "a"]]) +def parse_dimension_names_valid(data: Sequence[str] | None) -> None: + assert parse_dimension_names(data) == data + + +@pytest.mark.parametrize("fill_value", [[1.0, 0.0], [0, 1]]) +@pytest.mark.parametrize("dtype_str", [*complex_dtypes]) +def test_jsonify_fill_value_complex(fill_value: Any, dtype_str: str) -> None: + """ + Test that parse_fill_value(fill_value, dtype) correctly handles complex values represented + as length-2 sequences + """ + zarr_format = 3 + dtype = get_data_type_from_native_dtype(dtype_str) + expected = dtype.to_native_dtype().type(complex(*fill_value)) + observed = dtype.from_json_scalar(fill_value, zarr_format=zarr_format) + assert observed == expected + assert dtype.to_json_scalar(observed, zarr_format=zarr_format) == tuple(fill_value) + + +@pytest.mark.parametrize("fill_value", [{"foo": 10}]) +@pytest.mark.parametrize("dtype_str", [*int_dtypes, *float_dtypes, *complex_dtypes]) +def test_parse_fill_value_invalid_type(fill_value: Any, dtype_str: str) -> None: + """ + Test that parse_fill_value(fill_value, dtype) raises TypeError for invalid non-sequential types. + This test excludes bool because the bool constructor takes anything. + """ + dtype_instance = get_data_type_from_native_dtype(dtype_str) + with pytest.raises(TypeError, match=f"Invalid type: {fill_value}"): + dtype_instance.from_json_scalar(fill_value, zarr_format=3) + + +@pytest.mark.parametrize( + "fill_value", + [ + [ + 1, + ], + (1, 23, 4), + ], +) +@pytest.mark.parametrize("dtype_str", [*int_dtypes, *float_dtypes]) +def test_parse_fill_value_invalid_type_sequence(fill_value: Any, dtype_str: str) -> None: + """ + Test that parse_fill_value(fill_value, dtype) raises TypeError for invalid sequential types. + This test excludes bool because the bool constructor takes anything, and complex because + complex values can be created from length-2 sequences. + """ + dtype_instance = get_data_type_from_native_dtype(dtype_str) + with pytest.raises(TypeError, match=re.escape(f"Invalid type: {fill_value}")): + dtype_instance.from_json_scalar(fill_value, zarr_format=3) + + +@pytest.mark.parametrize("chunk_grid", ["regular"]) +@pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) +@pytest.mark.parametrize("codecs", [[BytesCodec(endian=None)]]) +@pytest.mark.parametrize("fill_value", [0, 1]) +@pytest.mark.parametrize("chunk_key_encoding", ["v2", "default"]) +@pytest.mark.parametrize("dimension_separator", [".", "/", None]) +@pytest.mark.parametrize("dimension_names", ["nones", "strings", "missing"]) +@pytest.mark.parametrize("storage_transformers", [None, ()]) +def test_metadata_to_dict( + chunk_grid: str, + codecs: list[Codec], + fill_value: Any, + chunk_key_encoding: Literal["v2", "default"], + dimension_separator: Literal[".", "/"] | None, + dimension_names: Literal["nones", "strings", "missing"], + attributes: dict[str, Any] | None, + storage_transformers: tuple[dict[str, JSON]] | None, +) -> None: + shape = (1, 2, 3) + data_type_str = "uint8" + if chunk_grid == "regular": + cgrid = {"name": "regular", "configuration": {"chunk_shape": (1, 1, 1)}} + + cke: dict[str, Any] + cke_name_dict = {"name": chunk_key_encoding} + if dimension_separator is not None: + cke = cke_name_dict | {"configuration": {"separator": dimension_separator}} + else: + cke = cke_name_dict + dnames: tuple[str | None, ...] | None + + if dimension_names == "strings": + dnames = tuple(map(str, range(len(shape)))) + elif dimension_names == "missing": + dnames = None + elif dimension_names == "nones": + dnames = (None,) * len(shape) + + metadata_dict = { + "zarr_format": 3, + "node_type": "array", + "shape": shape, + "chunk_grid": cgrid, + "data_type": data_type_str, + "chunk_key_encoding": cke, + "codecs": tuple(c.to_dict() for c in codecs), + "fill_value": fill_value, + "storage_transformers": storage_transformers, + } + + if attributes is not None: + metadata_dict["attributes"] = attributes + if dnames is not None: + metadata_dict["dimension_names"] = dnames + + metadata = ArrayV3Metadata.from_dict(metadata_dict) + observed = metadata.to_dict() + expected = metadata_dict.copy() + + # if unset or None or (), storage_transformers gets normalized to () + assert observed["storage_transformers"] == () + observed.pop("storage_transformers") + expected.pop("storage_transformers") + + if attributes is None: + assert observed["attributes"] == {} + observed.pop("attributes") + + if dimension_separator is None: + if chunk_key_encoding == "default": + expected_cke_dict = DefaultChunkKeyEncoding(separator="/").to_dict() + else: + expected_cke_dict = V2ChunkKeyEncoding(separator=".").to_dict() + assert observed["chunk_key_encoding"] == expected_cke_dict + observed.pop("chunk_key_encoding") + expected.pop("chunk_key_encoding") + assert observed == expected + + +@pytest.mark.parametrize("indent", [2, 4, None]) +def test_json_indent(indent: int): + with config.set({"json_indent": indent}): + m = GroupMetadata() + d = m.to_buffer_dict(default_buffer_prototype())["zarr.json"].to_bytes() + assert d == json.dumps(json.loads(d), indent=indent).encode() + + +@pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897]) +@pytest.mark.parametrize("precision", ["ns", "D"]) +async def test_datetime_metadata(fill_value: int, precision: str) -> None: + dtype = DateTime64(unit=precision) + metadata_dict = { + "zarr_format": 3, + "node_type": "array", + "shape": (1,), + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, + "data_type": dtype.to_json(zarr_format=3), + "chunk_key_encoding": {"name": "default", "separator": "."}, + "codecs": (BytesCodec(),), + "fill_value": dtype.to_json_scalar( + dtype.to_native_dtype().type(fill_value, dtype.unit), zarr_format=3 + ), + } + metadata = ArrayV3Metadata.from_dict(metadata_dict) + # ensure there isn't a TypeError here. + d = metadata.to_buffer_dict(default_buffer_prototype()) + + result = json.loads(d["zarr.json"].to_bytes()) + assert result["fill_value"] == fill_value + + +@pytest.mark.parametrize( + ("data_type", "fill_value"), [("uint8", {}), ("int32", [0, 1]), ("float32", "foo")] +) +async def test_invalid_fill_value_raises(data_type: str, fill_value: float) -> None: + metadata_dict = { + "zarr_format": 3, + "node_type": "array", + "shape": (1,), + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, + "data_type": data_type, + "chunk_key_encoding": {"name": "default", "separator": "."}, + "codecs": ({"name": "bytes"},), + "fill_value": fill_value, # this is not a valid fill value for uint8 + } + # multiple things can go wrong here, so we don't match on the error message. + with pytest.raises(TypeError): + ArrayV3Metadata.from_dict(metadata_dict) + + +@pytest.mark.parametrize("fill_value", [("NaN"), "Infinity", "-Infinity"]) +async def test_special_float_fill_values(fill_value: str) -> None: + metadata_dict = { + "zarr_format": 3, + "node_type": "array", + "shape": (1,), + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, + "data_type": "float64", + "chunk_key_encoding": {"name": "default", "separator": "."}, + "codecs": [{"name": "bytes"}], + "fill_value": fill_value, # this is not a valid fill value for uint8 + } + m = ArrayV3Metadata.from_dict(metadata_dict) + d = json.loads(m.to_buffer_dict(default_buffer_prototype())["zarr.json"].to_bytes()) + assert m.fill_value is not None + if fill_value == "NaN": + assert np.isnan(m.fill_value) + assert d["fill_value"] == "NaN" + elif fill_value == "Infinity": + assert np.isposinf(m.fill_value) + assert d["fill_value"] == "Infinity" + elif fill_value == "-Infinity": + assert np.isneginf(m.fill_value) + assert d["fill_value"] == "-Infinity" diff --git a/tests/test_properties.py b/tests/test_properties.py new file mode 100644 index 0000000000..b8d50ef0b1 --- /dev/null +++ b/tests/test_properties.py @@ -0,0 +1,336 @@ +import json +import numbers +from typing import Any + +import numpy as np +import pytest +from numpy.testing import assert_array_equal + +from zarr.core.buffer import default_buffer_prototype + +pytest.importorskip("hypothesis") + +import hypothesis.extra.numpy as npst +import hypothesis.strategies as st +from hypothesis import assume, given, settings + +from zarr.abc.store import Store +from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON +from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata +from zarr.core.sync import sync +from zarr.testing.strategies import ( + array_metadata, + arrays, + basic_indices, + numpy_arrays, + orthogonal_indices, + simple_arrays, + stores, + zarr_formats, +) + + +def deep_equal(a: Any, b: Any) -> bool: + """Deep equality check with handling of special cases for array metadata classes""" + if isinstance(a, (complex, np.complexfloating)) and isinstance( + b, (complex, np.complexfloating) + ): + a_real, a_imag = float(a.real), float(a.imag) + b_real, b_imag = float(b.real), float(b.imag) + if np.isnan(a_real) and np.isnan(b_real): + real_eq = True + else: + real_eq = a_real == b_real + if np.isnan(a_imag) and np.isnan(b_imag): + imag_eq = True + else: + imag_eq = a_imag == b_imag + return real_eq and imag_eq + + if isinstance(a, (float, np.floating)) and isinstance(b, (float, np.floating)): + if np.isnan(a) and np.isnan(b): + return True + return a == b + + if isinstance(a, np.datetime64) and isinstance(b, np.datetime64): + if np.isnat(a) and np.isnat(b): + return True + return a == b + + if isinstance(a, np.ndarray) and isinstance(b, np.ndarray): + if a.shape != b.shape: + return False + return all(deep_equal(x, y) for x, y in zip(a.flat, b.flat, strict=False)) + + if isinstance(a, dict) and isinstance(b, dict): + if set(a.keys()) != set(b.keys()): + return False + return all(deep_equal(a[k], b[k]) for k in a) + + if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)): + if len(a) != len(b): + return False + return all(deep_equal(x, y) for x, y in zip(a, b, strict=False)) + + return a == b + + +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +@given(data=st.data(), zarr_format=zarr_formats) +def test_array_roundtrip(data: st.DataObject, zarr_format: int) -> None: + nparray = data.draw(numpy_arrays(zarr_formats=st.just(zarr_format))) + zarray = data.draw(arrays(arrays=st.just(nparray), zarr_formats=st.just(zarr_format))) + assert_array_equal(nparray, zarray[:]) + + +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +@given(array=arrays()) +def test_array_creates_implicit_groups(array): + path = array.path + ancestry = path.split("/")[:-1] + for i in range(len(ancestry)): + parent = "/".join(ancestry[: i + 1]) + if array.metadata.zarr_format == 2: + assert ( + sync(array.store.get(f"{parent}/.zgroup", prototype=default_buffer_prototype())) + is not None + ) + elif array.metadata.zarr_format == 3: + assert ( + sync(array.store.get(f"{parent}/zarr.json", prototype=default_buffer_prototype())) + is not None + ) + + +# this decorator removes timeout; not ideal but it should avoid intermittent CI failures + + +@settings(deadline=None) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +@given(data=st.data()) +def test_basic_indexing(data: st.DataObject) -> None: + zarray = data.draw(simple_arrays()) + nparray = zarray[:] + indexer = data.draw(basic_indices(shape=nparray.shape)) + actual = zarray[indexer] + assert_array_equal(nparray[indexer], actual) + + new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) + zarray[indexer] = new_data + nparray[indexer] = new_data + assert_array_equal(nparray, zarray[:]) + + +@given(data=st.data()) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_oindex(data: st.DataObject) -> None: + # integer_array_indices can't handle 0-size dimensions. + zarray = data.draw(simple_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) + nparray = zarray[:] + + zindexer, npindexer = data.draw(orthogonal_indices(shape=nparray.shape)) + actual = zarray.oindex[zindexer] + assert_array_equal(nparray[npindexer], actual) + + assume(zarray.shards is None) # GH2834 + for idxr in npindexer: + if isinstance(idxr, np.ndarray) and idxr.size != np.unique(idxr).size: + # behaviour of setitem with repeated indices is not guaranteed in practice + assume(False) + new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) + nparray[npindexer] = new_data + zarray.oindex[zindexer] = new_data + assert_array_equal(nparray, zarray[:]) + + +@given(data=st.data()) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_vindex(data: st.DataObject) -> None: + # integer_array_indices can't handle 0-size dimensions. + zarray = data.draw(simple_arrays(shapes=npst.array_shapes(max_dims=4, min_side=1))) + nparray = zarray[:] + + indexer = data.draw( + npst.integer_array_indices( + shape=nparray.shape, result_shape=npst.array_shapes(min_side=1, max_dims=None) + ) + ) + actual = zarray.vindex[indexer] + assert_array_equal(nparray[indexer], actual) + + # FIXME! + # when the indexer is such that a value gets overwritten multiple times, + # I think the output depends on chunking. + # new_data = data.draw(npst.arrays(shape=st.just(actual.shape), dtype=nparray.dtype)) + # nparray[indexer] = new_data + # zarray.vindex[indexer] = new_data + # assert_array_equal(nparray, zarray[:]) + + +@given(store=stores, meta=array_metadata()) # type: ignore[misc] +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +async def test_roundtrip_array_metadata_from_store( + store: Store, meta: ArrayV2Metadata | ArrayV3Metadata +) -> None: + """ + Verify that the I/O for metadata in a store are lossless. + + This test serializes an ArrayV2Metadata or ArrayV3Metadata object to a dict + of buffers via `to_buffer_dict`, writes each buffer to a store under keys + prefixed with "0/", and then reads them back. The test asserts that each + retrieved buffer exactly matches the original buffer. + """ + asdict = meta.to_buffer_dict(prototype=default_buffer_prototype()) + for key, expected in asdict.items(): + await store.set(f"0/{key}", expected) + actual = await store.get(f"0/{key}", prototype=default_buffer_prototype()) + assert actual == expected + + +@given(data=st.data(), zarr_format=zarr_formats) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_roundtrip_array_metadata_from_json(data: st.DataObject, zarr_format: int) -> None: + """ + Verify that JSON serialization and deserialization of metadata is lossless. + + For Zarr v2: + - The metadata is split into two JSON documents (one for array data and one + for attributes). The test merges the attributes back before deserialization. + For Zarr v3: + - All metadata is stored in a single JSON document. No manual merger is necessary. + + The test then converts both the original and round-tripped metadata objects + into dictionaries using `dataclasses.asdict` and uses a deep equality check + to verify that the roundtrip has preserved all fields (including special + cases like NaN, Infinity, complex numbers, and datetime values). + """ + metadata = data.draw(array_metadata(zarr_formats=st.just(zarr_format))) + buffer_dict = metadata.to_buffer_dict(prototype=default_buffer_prototype()) + + if zarr_format == 2: + zarray_dict = json.loads(buffer_dict[ZARRAY_JSON].to_bytes().decode()) + zattrs_dict = json.loads(buffer_dict[ZATTRS_JSON].to_bytes().decode()) + # zattrs and zarray are separate in v2, we have to add attributes back prior to `from_dict` + zarray_dict["attributes"] = zattrs_dict + metadata_roundtripped = ArrayV2Metadata.from_dict(zarray_dict) + else: + zarray_dict = json.loads(buffer_dict[ZARR_JSON].to_bytes().decode()) + metadata_roundtripped = ArrayV3Metadata.from_dict(zarray_dict) + + orig = metadata.to_dict() + rt = metadata_roundtripped.to_dict() + + assert deep_equal(orig, rt), f"Roundtrip mismatch:\nOriginal: {orig}\nRoundtripped: {rt}" + + +# @st.composite +# def advanced_indices(draw, *, shape): +# basic_idxr = draw( +# basic_indices( +# shape=shape, min_dims=len(shape), max_dims=len(shape), allow_ellipsis=False +# ).filter(lambda x: isinstance(x, tuple)) +# ) + +# int_idxr = draw( +# npst.integer_array_indices(shape=shape, result_shape=npst.array_shapes(max_dims=1)) +# ) +# args = tuple( +# st.sampled_from((l, r)) for l, r in zip_longest(basic_idxr, int_idxr, fillvalue=slice(None)) +# ) +# return draw(st.tuples(*args)) + + +# @given(st.data()) +# def test_roundtrip_object_array(data): +# nparray = data.draw(np_arrays) +# zarray = data.draw(arrays(arrays=st.just(nparray))) +# assert_array_equal(nparray, zarray[:]) + + +def serialized_complex_float_is_valid( + serialized: tuple[numbers.Real | str, numbers.Real | str], +) -> bool: + """ + Validate that the serialized representation of a complex float conforms to the spec. + + The specification requires that a serialized complex float must be either: + - A JSON number, or + - One of the strings "NaN", "Infinity", or "-Infinity". + + Args: + serialized: The value produced by JSON serialization for a complex floating point number. + + Returns: + bool: True if the serialized value is valid according to the spec, False otherwise. + """ + return ( + isinstance(serialized, tuple) + and len(serialized) == 2 + and all(serialized_float_is_valid(x) for x in serialized) + ) + + +def serialized_float_is_valid(serialized: numbers.Real | str) -> bool: + """ + Validate that the serialized representation of a float conforms to the spec. + + The specification requires that a serialized float must be either: + - A JSON number, or + - One of the strings "NaN", "Infinity", or "-Infinity". + + Args: + serialized: The value produced by JSON serialization for a floating point number. + + Returns: + bool: True if the serialized value is valid according to the spec, False otherwise. + """ + if isinstance(serialized, numbers.Real): + return True + return serialized in ("NaN", "Infinity", "-Infinity") + + +@given(meta=array_metadata()) # type: ignore[misc] +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_array_metadata_meets_spec(meta: ArrayV2Metadata | ArrayV3Metadata) -> None: + """ + Validate that the array metadata produced by the library conforms to the relevant spec (V2 vs V3). + + For ArrayV2Metadata: + - Ensures that 'zarr_format' is 2. + - Verifies that 'filters' is either None or a tuple (and not an empty tuple). + For ArrayV3Metadata: + - Ensures that 'zarr_format' is 3. + + For both versions: + - If the dtype is a floating point of some kind, verifies of fill values: + * NaN is serialized as the string "NaN" + * Positive Infinity is serialized as the string "Infinity" + * Negative Infinity is serialized as the string "-Infinity" + * Other fill values are preserved as-is. + - If the dtype is a complex number of some kind, verifies that each component of the fill + value (real and imaginary) satisfies the serialization rules for floating point numbers. + - If the dtype is a datetime of some kind, verifies that `NaT` values are serialized as "NaT". + + Note: + This test validates spec-compliance for array metadata serialization. + It is a work-in-progress and should be expanded as further edge cases are identified. + """ + asdict_dict = meta.to_dict() + + # version-specific validations + if isinstance(meta, ArrayV2Metadata): + assert asdict_dict["filters"] != () + assert asdict_dict["filters"] is None or isinstance(asdict_dict["filters"], tuple) + assert asdict_dict["zarr_format"] == 2 + else: + assert asdict_dict["zarr_format"] == 3 + + # version-agnostic validations + dtype_native = meta.dtype.to_native_dtype() + if dtype_native.kind == "f": + assert serialized_float_is_valid(asdict_dict["fill_value"]) + elif dtype_native.kind == "c": + # fill_value should be a two-element array [real, imag]. + assert serialized_complex_float_is_valid(asdict_dict["fill_value"]) + elif dtype_native.kind in ("M", "m") and np.isnat(meta.fill_value): + assert asdict_dict["fill_value"] == -9223372036854775808 diff --git a/tests/test_regression/__init__.py b/tests/test_regression/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_regression/scripts/__init__.py b/tests/test_regression/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_regression/scripts/v2.18.py b/tests/test_regression/scripts/v2.18.py new file mode 100644 index 0000000000..39e1c5210c --- /dev/null +++ b/tests/test_regression/scripts/v2.18.py @@ -0,0 +1,81 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "zarr==2.18", +# "numcodecs==0.15" +# ] +# /// + +import argparse + +import zarr +from zarr._storage.store import BaseStore + + +def copy_group( + *, node: zarr.hierarchy.Group, store: zarr.storage.BaseStore, path: str, overwrite: bool +) -> zarr.hierarchy.Group: + result = zarr.group(store=store, path=path, overwrite=overwrite) + result.attrs.put(node.attrs.asdict()) + for key, child in node.items(): + child_path = f"{path}/{key}" + if isinstance(child, zarr.hierarchy.Group): + copy_group(node=child, store=store, path=child_path, overwrite=overwrite) + elif isinstance(child, zarr.core.Array): + copy_array(node=child, store=store, overwrite=overwrite, path=child_path) + return result + + +def copy_array( + *, node: zarr.core.Array, store: BaseStore, path: str, overwrite: bool +) -> zarr.core.Array: + result = zarr.create( + shape=node.shape, + dtype=node.dtype, + fill_value=node.fill_value, + chunks=node.chunks, + compressor=node.compressor, + filters=node.filters, + order=node.order, + dimension_separator=node._dimension_separator, + store=store, + path=path, + overwrite=overwrite, + ) + result.attrs.put(node.attrs.asdict()) + result[:] = node[:] + return result + + +def copy_node( + node: zarr.hierarchy.Group | zarr.core.Array, store: BaseStore, path: str, overwrite: bool +) -> zarr.hierarchy.Group | zarr.core.Array: + if isinstance(node, zarr.hierarchy.Group): + return copy_group(node=node, store=store, path=path, overwrite=overwrite) + elif isinstance(node, zarr.core.Array): + return copy_array(node=node, store=store, path=path, overwrite=overwrite) + else: + raise TypeError(f"Unexpected node type: {type(node)}") # pragma: no cover + + +def cli() -> None: + parser = argparse.ArgumentParser( + description="Copy a zarr hierarchy from one location to another" + ) + parser.add_argument("source", type=str, help="Path to the source zarr hierarchy") + parser.add_argument("destination", type=str, help="Path to the destination zarr hierarchy") + args = parser.parse_args() + + src, dst = args.source, args.destination + root_src = zarr.open(src, mode="r") + result = copy_node(node=root_src, store=zarr.NestedDirectoryStore(dst), path="", overwrite=True) + + print(f"successfully created {result} at {dst}") + + +def main() -> None: + cli() + + +if __name__ == "__main__": + main() diff --git a/tests/test_regression/test_regression.py b/tests/test_regression/test_regression.py new file mode 100644 index 0000000000..34c48a6933 --- /dev/null +++ b/tests/test_regression/test_regression.py @@ -0,0 +1,156 @@ +import subprocess +from dataclasses import dataclass +from itertools import product +from pathlib import Path +from typing import TYPE_CHECKING + +import numcodecs +import numpy as np +import pytest +from numcodecs import LZ4, LZMA, Blosc, GZip, VLenBytes, VLenUTF8, Zstd + +import zarr +from zarr.core.array import Array +from zarr.core.chunk_key_encodings import V2ChunkKeyEncoding +from zarr.core.dtype.npy.bytes import VariableLengthBytes +from zarr.core.dtype.npy.string import VariableLengthUTF8 +from zarr.storage import LocalStore + +if TYPE_CHECKING: + from zarr.core.dtype import ZDTypeLike + + +def runner_installed() -> bool: + """ + Check if a PEP-723 compliant python script runner is installed. + """ + try: + subprocess.check_output(["uv", "--version"]) + return True # noqa: TRY300 + except FileNotFoundError: + return False + + +@dataclass(kw_only=True) +class ArrayParams: + values: np.ndarray[tuple[int], np.dtype[np.generic]] + fill_value: np.generic | str | int | bytes + filters: tuple[numcodecs.abc.Codec, ...] = () + compressor: numcodecs.abc.Codec + + +basic_codecs = GZip(), Blosc(), LZ4(), LZMA(), Zstd() +basic_dtypes = "|b", ">i2", ">i4", ">f4", ">f8", "c8", "c16", "M8[10us]", "m8[4ps]" +string_dtypes = "U4" +bytes_dtypes = ">S1", "V10", " Array: + dest = tmp_path / "in" + store = LocalStore(dest) + array_params: ArrayParams = request.param + compressor = array_params.compressor + chunk_key_encoding = V2ChunkKeyEncoding(separator="/") + dtype: ZDTypeLike + if array_params.values.dtype == np.dtype("|O") and array_params.filters == (VLenUTF8(),): + dtype = VariableLengthUTF8() # type: ignore[assignment] + elif array_params.values.dtype == np.dtype("|O") and array_params.filters == (VLenBytes(),): + dtype = VariableLengthBytes() + else: + dtype = array_params.values.dtype + z = zarr.create_array( + store, + shape=array_params.values.shape, + dtype=dtype, + chunks=array_params.values.shape, + compressors=compressor, + filters=array_params.filters, + fill_value=array_params.fill_value, + order="C", + chunk_key_encoding=chunk_key_encoding, + write_data=True, + zarr_format=2, + ) + z[:] = array_params.values + return z + + +# TODO: make this dynamic based on the installed scripts +script_paths = [Path(__file__).resolve().parent / "scripts" / "v2.18.py"] + + +@pytest.mark.skipif(not runner_installed(), reason="no python script runner installed") +@pytest.mark.parametrize( + "source_array", array_cases, indirect=True, ids=tuple(map(str, array_cases)) +) +@pytest.mark.parametrize("script_path", script_paths) +def test_roundtrip(source_array: Array, tmp_path: Path, script_path: Path) -> None: + out_path = tmp_path / "out" + copy_op = subprocess.run( + [ + "uv", + "run", + script_path, + str(source_array.store).removeprefix("file://"), + str(out_path), + ], + capture_output=True, + text=True, + ) + assert copy_op.returncode == 0 + out_array = zarr.open_array(store=out_path, mode="r", zarr_format=2) + assert source_array.metadata.to_dict() == out_array.metadata.to_dict() + assert np.array_equal(source_array[:], out_array[:]) diff --git a/tests/test_store/__init__.py b/tests/test_store/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py new file mode 100644 index 0000000000..e9c9319ad3 --- /dev/null +++ b/tests/test_store/test_core.py @@ -0,0 +1,270 @@ +import tempfile +from pathlib import Path + +import pytest +from _pytest.compat import LEGACY_PATH + +import zarr +from zarr import Group +from zarr.core.common import AccessModeLiteral, ZarrFormat +from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath +from zarr.storage._common import contains_array, contains_group, make_store_path +from zarr.storage._utils import ( + _join_paths, + _normalize_path_keys, + _normalize_paths, + _relativize_path, + normalize_path, +) + + +@pytest.fixture( + params=["none", "temp_dir_str", "temp_dir_path", "store_path", "memory_store", "dict"] +) +def store_like(request): + if request.param == "none": + yield None + elif request.param == "temp_dir_str": + with tempfile.TemporaryDirectory() as temp_dir: + yield temp_dir + elif request.param == "temp_dir_path": + with tempfile.TemporaryDirectory() as temp_dir: + yield Path(temp_dir) + elif request.param == "store_path": + yield StorePath(store=MemoryStore(store_dict={}), path="/") + elif request.param == "memory_store": + yield MemoryStore(store_dict={}) + elif request.param == "dict": + yield {} + + +@pytest.mark.parametrize("path", ["foo", "foo/bar"]) +@pytest.mark.parametrize("write_group", [True, False]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +async def test_contains_group( + local_store, path: str, write_group: bool, zarr_format: ZarrFormat +) -> None: + """ + Test that the contains_group method correctly reports the existence of a group. + """ + root = Group.from_store(store=local_store, zarr_format=zarr_format) + if write_group: + root.create_group(path) + store_path = StorePath(local_store, path=path) + assert await contains_group(store_path, zarr_format=zarr_format) == write_group + + +@pytest.mark.parametrize("path", ["foo", "foo/bar"]) +@pytest.mark.parametrize("write_array", [True, False]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +async def test_contains_array( + local_store, path: str, write_array: bool, zarr_format: ZarrFormat +) -> None: + """ + Test that the contains array method correctly reports the existence of an array. + """ + root = Group.from_store(store=local_store, zarr_format=zarr_format) + if write_array: + root.create_array(path, shape=(100,), chunks=(10,), dtype="i4") + store_path = StorePath(local_store, path=path) + assert await contains_array(store_path, zarr_format=zarr_format) == write_array + + +@pytest.mark.parametrize("func", [contains_array, contains_group]) +async def test_contains_invalid_format_raises(local_store, func: callable) -> None: + """ + Test contains_group and contains_array raise errors for invalid zarr_formats + """ + store_path = StorePath(local_store) + with pytest.raises(ValueError): + assert await func(store_path, zarr_format="3.0") + + +@pytest.mark.parametrize("path", [None, "", "bar"]) +async def test_make_store_path_none(path: str) -> None: + """ + Test that creating a store_path with None creates a memorystore + """ + store_path = await make_store_path(None, path=path) + assert isinstance(store_path.store, MemoryStore) + assert store_path.path == normalize_path(path) + + +@pytest.mark.parametrize("path", [None, "", "bar"]) +@pytest.mark.parametrize("store_type", [str, Path]) +@pytest.mark.parametrize("mode", ["r", "w"]) +async def test_make_store_path_local( + tmpdir: LEGACY_PATH, + store_type: type[str] | type[Path] | type[LocalStore], + path: str, + mode: AccessModeLiteral, +) -> None: + """ + Test the various ways of invoking make_store_path that create a LocalStore + """ + store_like = store_type(str(tmpdir)) + store_path = await make_store_path(store_like, path=path, mode=mode) + assert isinstance(store_path.store, LocalStore) + assert Path(store_path.store.root) == Path(tmpdir) + assert store_path.path == normalize_path(path) + assert store_path.read_only == (mode == "r") + + +@pytest.mark.parametrize("path", [None, "", "bar"]) +@pytest.mark.parametrize("mode", ["r", "w"]) +async def test_make_store_path_store_path( + tmpdir: LEGACY_PATH, path: str, mode: AccessModeLiteral +) -> None: + """ + Test invoking make_store_path when the input is another store_path. In particular we want to ensure + that a new path is handled correctly. + """ + ro = mode == "r" + store_like = await StorePath.open(LocalStore(str(tmpdir), read_only=ro), path="root", mode=mode) + store_path = await make_store_path(store_like, path=path, mode=mode) + assert isinstance(store_path.store, LocalStore) + assert Path(store_path.store.root) == Path(tmpdir) + path_normalized = normalize_path(path) + assert store_path.path == (store_like / path_normalized).path + assert store_path.read_only == ro + + +@pytest.mark.parametrize("modes", [(True, "w"), (False, "x")]) +async def test_store_path_invalid_mode_raises(tmpdir: LEGACY_PATH, modes: tuple) -> None: + """ + Test that ValueErrors are raise for invalid mode. + """ + with pytest.raises(ValueError): + await StorePath.open(LocalStore(str(tmpdir), read_only=modes[0]), path=None, mode=modes[1]) + + +async def test_make_store_path_invalid() -> None: + """ + Test that invalid types raise TypeError + """ + with pytest.raises(TypeError): + await make_store_path(1) # type: ignore[arg-type] + + +async def test_make_store_path_fsspec(monkeypatch) -> None: + pytest.importorskip("fsspec") + pytest.importorskip("requests") + pytest.importorskip("aiohttp") + store_path = await make_store_path("http://foo.com/bar") + assert isinstance(store_path.store, FsspecStore) + + +async def test_make_store_path_storage_options_raises(store_like: StoreLike) -> None: + with pytest.raises(TypeError, match="storage_options"): + await make_store_path(store_like, storage_options={"foo": "bar"}) + + +async def test_unsupported() -> None: + with pytest.raises(TypeError, match="Unsupported type for store_like: 'int'"): + await make_store_path(1) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + "path", + [ + "/foo/bar", + "//foo/bar", + "foo///bar", + "foo/bar///", + Path("foo/bar"), + b"foo/bar", + ], +) +def test_normalize_path_valid(path: str | bytes | Path) -> None: + assert normalize_path(path) == "foo/bar" + + +def test_normalize_path_upath() -> None: + upath = pytest.importorskip("upath") + assert normalize_path(upath.UPath("foo/bar")) == "foo/bar" + + +def test_normalize_path_none(): + assert normalize_path(None) == "" + + +@pytest.mark.parametrize("path", [".", ".."]) +def test_normalize_path_invalid(path: str): + with pytest.raises(ValueError): + normalize_path(path) + + +@pytest.mark.parametrize("paths", [("", "foo"), ("foo", "bar")]) +def test_join_paths(paths: tuple[str, str]) -> None: + """ + Test that _join_paths joins paths in a way that is robust to an empty string + """ + observed = _join_paths(paths) + if paths[0] == "": + assert observed == paths[1] + else: + assert observed == "/".join(paths) + + +class TestNormalizePaths: + @staticmethod + def test_valid() -> None: + """ + Test that path normalization works as expected + """ + paths = ["a", "b", "c", "d", "", "//a///b//"] + assert _normalize_paths(paths) == tuple(normalize_path(p) for p in paths) + + @staticmethod + @pytest.mark.parametrize("paths", [("", "/"), ("///a", "a")]) + def test_invalid(paths: tuple[str, str]) -> None: + """ + Test that name collisions after normalization raise a ``ValueError`` + """ + msg = ( + f"After normalization, the value '{paths[1]}' collides with '{paths[0]}'. " + f"Both '{paths[1]}' and '{paths[0]}' normalize to the same value: '{normalize_path(paths[0])}'. " + f"You should use either '{paths[1]}' or '{paths[0]}', but not both." + ) + with pytest.raises(ValueError, match=msg): + _normalize_paths(paths) + + +def test_normalize_path_keys(): + """ + Test that ``_normalize_path_keys`` just applies the normalize_path function to each key of its + input + """ + data = {"a": 10, "//b": 10} + assert _normalize_path_keys(data) == {normalize_path(k): v for k, v in data.items()} + + +@pytest.mark.parametrize( + ("path", "prefix", "expected"), + [ + ("a", "", "a"), + ("a/b/c", "a/b", "c"), + ("a/b/c", "a", "b/c"), + ], +) +def test_relativize_path_valid(path: str, prefix: str, expected: str) -> None: + """ + Test the normal behavior of the _relativize_path function. Prefixes should be removed from the + path argument. + """ + assert _relativize_path(path=path, prefix=prefix) == expected + + +def test_relativize_path_invalid() -> None: + path = "a/b/c" + prefix = "b" + msg = f"The first component of {path} does not start with {prefix}." + with pytest.raises(ValueError, match=msg): + _relativize_path(path="a/b/c", prefix="b") + + +def test_invalid_open_mode() -> None: + store = MemoryStore() + zarr.create((100,), store=store, zarr_format=2, path="a") + with pytest.raises(ValueError, match="Store is not read-only but mode is 'r'"): + zarr.open_array(store=store, path="a", zarr_format=2, mode="r") diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py new file mode 100644 index 0000000000..1a989525e3 --- /dev/null +++ b/tests/test_store/test_fsspec.py @@ -0,0 +1,428 @@ +from __future__ import annotations + +import json +import os +import re +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest +from packaging.version import parse as parse_version + +import zarr.api.asynchronous +from zarr import Array +from zarr.abc.store import OffsetByteRequest +from zarr.core.buffer import Buffer, cpu, default_buffer_prototype +from zarr.core.sync import _collect_aiterator, sync +from zarr.storage import FsspecStore +from zarr.storage._fsspec import _make_async +from zarr.testing.store import StoreTests + +if TYPE_CHECKING: + import pathlib + from collections.abc import Generator + from pathlib import Path + + import botocore.client + import s3fs + + from zarr.core.common import JSON + + +# Warning filter due to https://github.com/boto/boto3/issues/3889 +pytestmark = [ + pytest.mark.filterwarnings( + re.escape("ignore:datetime.datetime.utcnow() is deprecated:DeprecationWarning") + ), + # TODO: fix these warnings + pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning"), + pytest.mark.filterwarnings( + "ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited:RuntimeWarning" + ), +] + +fsspec = pytest.importorskip("fsspec") +s3fs = pytest.importorskip("s3fs") +requests = pytest.importorskip("requests") +moto_server = pytest.importorskip("moto.moto_server.threaded_moto_server") +moto = pytest.importorskip("moto") +botocore = pytest.importorskip("botocore") + +# ### amended from s3fs ### # +test_bucket_name = "test" +secure_bucket_name = "test-secure" +port = 5555 +endpoint_url = f"http://127.0.0.1:{port}/" + + +@pytest.fixture(scope="module") +def s3_base() -> Generator[None, None, None]: + # writable local S3 system + + # This fixture is module-scoped, meaning that we can reuse the MotoServer across all tests + server = moto_server.ThreadedMotoServer(ip_address="127.0.0.1", port=port) + server.start() + if "AWS_SECRET_ACCESS_KEY" not in os.environ: + os.environ["AWS_SECRET_ACCESS_KEY"] = "foo" + if "AWS_ACCESS_KEY_ID" not in os.environ: + os.environ["AWS_ACCESS_KEY_ID"] = "foo" + + yield + server.stop() + + +def get_boto3_client() -> botocore.client.BaseClient: + # NB: we use the sync botocore client for setup + session = botocore.session.Session() + return session.create_client("s3", endpoint_url=endpoint_url) + + +@pytest.fixture(autouse=True) +def s3(s3_base: None) -> Generator[s3fs.S3FileSystem, None, None]: + """ + Quoting Martin Durant: + pytest-asyncio creates a new event loop for each async test. + When an async-mode s3fs instance is made from async, it will be assigned to the loop from + which it is made. That means that if you use s3fs again from a subsequent test, + you will have the same identical instance, but be running on a different loop - which fails. + + For the rest: it's very convenient to clean up the state of the store between tests, + make sure we start off blank each time. + + https://github.com/zarr-developers/zarr-python/pull/1785#discussion_r1634856207 + """ + client = get_boto3_client() + client.create_bucket(Bucket=test_bucket_name, ACL="public-read") + s3fs.S3FileSystem.clear_instance_cache() + s3 = s3fs.S3FileSystem(anon=False, client_kwargs={"endpoint_url": endpoint_url}) + session = sync(s3.set_session()) + s3.invalidate_cache() + yield s3 + requests.post(f"{endpoint_url}/moto-api/reset") + client.close() + sync(session.close()) + + +# ### end from s3fs ### # + + +async def test_basic() -> None: + store = FsspecStore.from_url( + f"s3://{test_bucket_name}/foo/spam/", + storage_options={"endpoint_url": endpoint_url, "anon": False}, + ) + assert store.fs.asynchronous + assert store.path == f"{test_bucket_name}/foo/spam" + assert await _collect_aiterator(store.list()) == () + assert not await store.exists("foo") + data = b"hello" + await store.set("foo", cpu.Buffer.from_bytes(data)) + assert await store.exists("foo") + buf = await store.get("foo", prototype=default_buffer_prototype()) + assert buf is not None + assert buf.to_bytes() == data + out = await store.get_partial_values( + prototype=default_buffer_prototype(), key_ranges=[("foo", OffsetByteRequest(1))] + ) + assert out[0] is not None + assert out[0].to_bytes() == data[1:] + + +class TestFsspecStoreS3(StoreTests[FsspecStore, cpu.Buffer]): + store_cls = FsspecStore + buffer_cls = cpu.Buffer + + @pytest.fixture + def store_kwargs(self) -> dict[str, str | bool]: + try: + from fsspec import url_to_fs + except ImportError: + # before fsspec==2024.3.1 + from fsspec.core import url_to_fs + fs, path = url_to_fs( + f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False, asynchronous=True + ) + return {"fs": fs, "path": path} + + @pytest.fixture + async def store(self, store_kwargs: dict[str, Any]) -> FsspecStore: + return self.store_cls(**store_kwargs) + + async def get(self, store: FsspecStore, key: str) -> Buffer: + # make a new, synchronous instance of the filesystem because this test is run in sync code + new_fs = fsspec.filesystem( + "s3", endpoint_url=store.fs.endpoint_url, anon=store.fs.anon, asynchronous=False + ) + return self.buffer_cls.from_bytes(new_fs.cat(f"{store.path}/{key}")) + + async def set(self, store: FsspecStore, key: str, value: Buffer) -> None: + # make a new, synchronous instance of the filesystem because this test is run in sync code + new_fs = fsspec.filesystem( + "s3", endpoint_url=store.fs.endpoint_url, anon=store.fs.anon, asynchronous=False + ) + new_fs.write_bytes(f"{store.path}/{key}", value.to_bytes()) + + def test_store_repr(self, store: FsspecStore) -> None: + assert str(store) == "" + + def test_store_supports_writes(self, store: FsspecStore) -> None: + assert store.supports_writes + + def test_store_supports_partial_writes(self, store: FsspecStore) -> None: + assert not store.supports_partial_writes + + def test_store_supports_listing(self, store: FsspecStore) -> None: + assert store.supports_listing + + async def test_fsspec_store_from_uri(self, store: FsspecStore) -> None: + storage_options = { + "endpoint_url": endpoint_url, + "anon": False, + } + + meta: dict[str, JSON] = { + "attributes": {"key": "value"}, + "zarr_format": 3, + "node_type": "group", + } + + await store.set( + "zarr.json", + self.buffer_cls.from_bytes(json.dumps(meta).encode()), + ) + group = await zarr.api.asynchronous.open_group( + store=f"s3://{test_bucket_name}", storage_options=storage_options + ) + assert dict(group.attrs) == {"key": "value"} + + meta = { + "attributes": {"key": "value-2"}, + "zarr_format": 3, + "node_type": "group", + } + await store.set( + "directory-2/zarr.json", + self.buffer_cls.from_bytes(json.dumps(meta).encode()), + ) + group = await zarr.api.asynchronous.open_group( + store=f"s3://{test_bucket_name}/directory-2", storage_options=storage_options + ) + assert dict(group.attrs) == {"key": "value-2"} + + meta = { + "attributes": {"key": "value-3"}, + "zarr_format": 3, + "node_type": "group", + } + await store.set( + "directory-3/zarr.json", + self.buffer_cls.from_bytes(json.dumps(meta).encode()), + ) + group = await zarr.api.asynchronous.open_group( + store=f"s3://{test_bucket_name}", path="directory-3", storage_options=storage_options + ) + assert dict(group.attrs) == {"key": "value-3"} + + @pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.03.01"), + reason="Prior bug in from_upath", + ) + def test_from_upath(self) -> None: + upath = pytest.importorskip("upath") + path = upath.UPath( + f"s3://{test_bucket_name}/foo/bar/", + endpoint_url=endpoint_url, + anon=False, + asynchronous=True, + ) + result = FsspecStore.from_upath(path) + assert result.fs.endpoint_url == endpoint_url + assert result.fs.asynchronous + assert result.path == f"{test_bucket_name}/foo/bar" + + def test_init_raises_if_path_has_scheme(self, store_kwargs: dict[str, Any]) -> None: + # regression test for https://github.com/zarr-developers/zarr-python/issues/2342 + store_kwargs["path"] = "s3://" + store_kwargs["path"] + with pytest.raises( + ValueError, match="path argument to FsspecStore must not include scheme .*" + ): + self.store_cls(**store_kwargs) + + def test_init_warns_if_fs_asynchronous_is_false(self) -> None: + try: + from fsspec import url_to_fs + except ImportError: + # before fsspec==2024.3.1 + from fsspec.core import url_to_fs + fs, path = url_to_fs( + f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False, asynchronous=False + ) + store_kwargs = {"fs": fs, "path": path} + with pytest.warns(UserWarning, match=r".* was not created with `asynchronous=True`.*"): + self.store_cls(**store_kwargs) + + async def test_empty_nonexistent_path(self, store_kwargs: dict[str, Any]) -> None: + # regression test for https://github.com/zarr-developers/zarr-python/pull/2343 + store_kwargs["path"] += "/abc" + store = await self.store_cls.open(**store_kwargs) + assert await store.is_empty("") + + async def test_delete_dir_unsupported_deletes(self, store: FsspecStore) -> None: + store.supports_deletes = False + with pytest.raises( + NotImplementedError, + match="This method is only available for stores that support deletes.", + ): + await store.delete_dir("test_prefix") + + +def array_roundtrip(store: FsspecStore) -> None: + """ + Round trip an array using a Zarr store + + Args: + store: FsspecStore + """ + data = np.ones((3, 3)) + arr = zarr.create_array(store=store, overwrite=True, data=data) + assert isinstance(arr, Array) + # Read set values + arr2 = zarr.open_array(store=store) + assert isinstance(arr2, Array) + np.testing.assert_array_equal(arr[:], data) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_wrap_sync_filesystem(tmp_path: pathlib.Path) -> None: + """The local fs is not async so we should expect it to be wrapped automatically""" + from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + + store = FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": True}) + assert isinstance(store.fs, AsyncFileSystemWrapper) + assert store.fs.async_impl + array_roundtrip(store) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) >= parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_wrap_sync_filesystem_raises(tmp_path: pathlib.Path) -> None: + """The local fs is not async so we should expect it to be wrapped automatically""" + with pytest.raises(ImportError, match="The filesystem .*"): + FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": True}) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_no_wrap_async_filesystem() -> None: + """An async fs should not be wrapped automatically; fsspec's s3 filesystem is such an fs""" + from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + + store = FsspecStore.from_url( + f"s3://{test_bucket_name}/foo/spam/", + storage_options={"endpoint_url": endpoint_url, "anon": False, "asynchronous": True}, + read_only=False, + ) + assert not isinstance(store.fs, AsyncFileSystemWrapper) + assert store.fs.async_impl + array_roundtrip(store) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_open_fsmap_file(tmp_path: pathlib.Path) -> None: + min_fsspec_with_async_wrapper = parse_version("2024.12.0") + current_version = parse_version(fsspec.__version__) + + fs = fsspec.filesystem("file", auto_mkdir=True) + mapper = fs.get_mapper(tmp_path) + + if current_version < min_fsspec_with_async_wrapper: + # Expect ImportError for older versions + with pytest.raises( + ImportError, + match=r"The filesystem .* is synchronous, and the required AsyncFileSystemWrapper is not available.*", + ): + array_roundtrip(mapper) + else: + # Newer versions should work + array_roundtrip(mapper) + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +def test_open_fsmap_file_raises(tmp_path: pathlib.Path) -> None: + fsspec = pytest.importorskip("fsspec.implementations.local") + fs = fsspec.LocalFileSystem(auto_mkdir=False) + mapper = fs.get_mapper(tmp_path) + with pytest.raises(ValueError, match="LocalFilesystem .*"): + array_roundtrip(mapper) + + +@pytest.mark.parametrize("asynchronous", [True, False]) +def test_open_fsmap_s3(asynchronous: bool) -> None: + s3_filesystem = s3fs.S3FileSystem( + asynchronous=asynchronous, endpoint_url=endpoint_url, anon=False + ) + mapper = s3_filesystem.get_mapper(f"s3://{test_bucket_name}/map/foo/") + array_roundtrip(mapper) + + +def test_open_s3map_raises() -> None: + with pytest.raises(TypeError, match="Unsupported type for store_like:.*"): + zarr.open(store=0, mode="w", shape=(3, 3)) + s3_filesystem = s3fs.S3FileSystem(asynchronous=True, endpoint_url=endpoint_url, anon=False) + mapper = s3_filesystem.get_mapper(f"s3://{test_bucket_name}/map/foo/") + with pytest.raises( + ValueError, match="'path' was provided but is not used for FSMap store_like objects" + ): + zarr.open(store=mapper, path="bar", mode="w", shape=(3, 3)) + with pytest.raises( + ValueError, + match="'storage_options was provided but is not used for FSMap store_like objects", + ): + zarr.open(store=mapper, storage_options={"anon": True}, mode="w", shape=(3, 3)) + + +@pytest.mark.parametrize("asynchronous", [True, False]) +def test_make_async(asynchronous: bool) -> None: + s3_filesystem = s3fs.S3FileSystem( + asynchronous=asynchronous, endpoint_url=endpoint_url, anon=False + ) + fs = _make_async(s3_filesystem) + assert fs.asynchronous + + +@pytest.mark.skipif( + parse_version(fsspec.__version__) < parse_version("2024.12.0"), + reason="No AsyncFileSystemWrapper", +) +async def test_delete_dir_wrapped_filesystem(tmp_path: Path) -> None: + from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper + from fsspec.implementations.local import LocalFileSystem + + wrapped_fs = AsyncFileSystemWrapper(LocalFileSystem(auto_mkdir=True)) + store = FsspecStore(wrapped_fs, read_only=False, path=f"{tmp_path}/test/path") + + assert isinstance(store.fs, AsyncFileSystemWrapper) + assert store.fs.asynchronous + + await store.set("zarr.json", cpu.Buffer.from_bytes(b"root")) + await store.set("foo-bar/zarr.json", cpu.Buffer.from_bytes(b"root")) + await store.set("foo/zarr.json", cpu.Buffer.from_bytes(b"bar")) + await store.set("foo/c/0", cpu.Buffer.from_bytes(b"chunk")) + await store.delete_dir("foo") + assert await store.exists("zarr.json") + assert await store.exists("foo-bar/zarr.json") + assert not await store.exists("foo/zarr.json") + assert not await store.exists("foo/c/0") diff --git a/tests/test_store/test_local.py b/tests/test_store/test_local.py new file mode 100644 index 0000000000..7974d0d633 --- /dev/null +++ b/tests/test_store/test_local.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import pathlib +import re + +import numpy as np +import pytest + +import zarr +from zarr import create_array +from zarr.core.buffer import Buffer, cpu +from zarr.storage import LocalStore +from zarr.testing.store import StoreTests +from zarr.testing.utils import assert_bytes_equal + + +class TestLocalStore(StoreTests[LocalStore, cpu.Buffer]): + store_cls = LocalStore + buffer_cls = cpu.Buffer + + async def get(self, store: LocalStore, key: str) -> Buffer: + return self.buffer_cls.from_bytes((store.root / key).read_bytes()) + + async def set(self, store: LocalStore, key: str, value: Buffer) -> None: + parent = (store.root / key).parent + if not parent.exists(): + parent.mkdir(parents=True) + (store.root / key).write_bytes(value.to_bytes()) + + @pytest.fixture + def store_kwargs(self, tmpdir: str) -> dict[str, str]: + return {"root": str(tmpdir)} + + def test_store_repr(self, store: LocalStore) -> None: + assert str(store) == f"file://{store.root.as_posix()}" + + def test_store_supports_writes(self, store: LocalStore) -> None: + assert store.supports_writes + + def test_store_supports_partial_writes(self, store: LocalStore) -> None: + assert store.supports_partial_writes + + def test_store_supports_listing(self, store: LocalStore) -> None: + assert store.supports_listing + + async def test_empty_with_empty_subdir(self, store: LocalStore) -> None: + assert await store.is_empty("") + (store.root / "foo/bar").mkdir(parents=True) + assert await store.is_empty("") + + def test_creates_new_directory(self, tmp_path: pathlib.Path) -> None: + target = tmp_path.joinpath("a", "b", "c") + assert not target.exists() + + store = self.store_cls(root=target) + zarr.group(store=store) + + def test_invalid_root_raises(self) -> None: + """ + Test that a TypeError is raised when a non-str/Path type is used for the `root` argument + """ + with pytest.raises( + TypeError, + match=r"'root' must be a string or Path instance. Got an instance of instead.", + ): + LocalStore(root=0) # type: ignore[arg-type] + + async def test_get_with_prototype_default(self, store: LocalStore) -> None: + """ + Ensure that data can be read via ``store.get`` if the prototype keyword argument is unspecified, i.e. set to ``None``. + """ + data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04") + key = "c/0" + await self.set(store, key, data_buf) + observed = await store.get(key, prototype=None) + assert_bytes_equal(observed, data_buf) + + @pytest.mark.parametrize("ndim", [0, 1, 3]) + @pytest.mark.parametrize( + "destination", ["destination", "foo/bar/destintion", pathlib.Path("foo/bar/destintion")] + ) + async def test_move( + self, tmp_path: pathlib.Path, ndim: int, destination: pathlib.Path | str + ) -> None: + origin = tmp_path / "origin" + if isinstance(destination, str): + destination = str(tmp_path / destination) + else: + destination = tmp_path / destination + + print(type(destination)) + store = await LocalStore.open(root=origin) + shape = (4,) * ndim + chunks = (2,) * ndim + data = np.arange(4**ndim) + if ndim > 0: + data = data.reshape(*shape) + array = create_array(store, data=data, chunks=chunks or "auto") + + await store.move(destination) + + assert store.root == pathlib.Path(destination) + assert pathlib.Path(destination).exists() + assert not origin.exists() + assert np.array_equal(array[...], data) + + store2 = await LocalStore.open(root=origin) + with pytest.raises( + FileExistsError, match=re.escape(f"Destination root {destination} already exists") + ): + await store2.move(destination) diff --git a/tests/test_store/test_logging.py b/tests/test_store/test_logging.py new file mode 100644 index 0000000000..1a89dca874 --- /dev/null +++ b/tests/test_store/test_logging.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import pytest + +import zarr +from zarr.core.buffer import Buffer, cpu, default_buffer_prototype +from zarr.storage import LocalStore, LoggingStore +from zarr.testing.store import StoreTests + +if TYPE_CHECKING: + from _pytest.compat import LEGACY_PATH + + from zarr.abc.store import Store + + +class TestLoggingStore(StoreTests[LoggingStore, cpu.Buffer]): + store_cls = LoggingStore + buffer_cls = cpu.Buffer + + async def get(self, store: LoggingStore, key: str) -> Buffer: + return self.buffer_cls.from_bytes((store._store.root / key).read_bytes()) + + async def set(self, store: LoggingStore, key: str, value: Buffer) -> None: + parent = (store._store.root / key).parent + if not parent.exists(): + parent.mkdir(parents=True) + (store._store.root / key).write_bytes(value.to_bytes()) + + @pytest.fixture + def store_kwargs(self, tmpdir: LEGACY_PATH) -> dict[str, str]: + return {"store": LocalStore(str(tmpdir)), "log_level": "DEBUG"} + + @pytest.fixture + def open_kwargs(self, tmpdir) -> dict[str, str]: + return {"store_cls": LocalStore, "root": str(tmpdir), "log_level": "DEBUG"} + + @pytest.fixture + def store(self, store_kwargs: str | dict[str, Buffer] | None) -> LoggingStore: + return self.store_cls(**store_kwargs) + + def test_store_supports_writes(self, store: LoggingStore) -> None: + assert store.supports_writes + + def test_store_supports_partial_writes(self, store: LoggingStore) -> None: + assert store.supports_partial_writes + + def test_store_supports_listing(self, store: LoggingStore) -> None: + assert store.supports_listing + + def test_store_repr(self, store: LoggingStore) -> None: + assert f"{store!r}" == f"LoggingStore(LocalStore, 'file://{store._store.root.as_posix()}')" + + def test_store_str(self, store: LoggingStore) -> None: + assert str(store) == f"logging-file://{store._store.root.as_posix()}" + + async def test_default_handler(self, local_store, capsys) -> None: + # Store and then remove existing handlers to enter default handler code path + handlers = logging.getLogger().handlers[:] + for h in handlers: + logging.getLogger().removeHandler(h) + # Test logs are sent to stdout + wrapped = LoggingStore(store=local_store) + buffer = default_buffer_prototype().buffer + res = await wrapped.set("foo/bar/c/0", buffer.from_bytes(b"\x01\x02\x03\x04")) + assert res is None + captured = capsys.readouterr() + assert len(captured) == 2 + assert "Calling LocalStore.set" in captured.out + assert "Finished LocalStore.set" in captured.out + # Restore handlers + for h in handlers: + logging.getLogger().addHandler(h) + + def test_is_open_setter_raises(self, store: LoggingStore) -> None: + "Test that a user cannot change `_is_open` without opening the underlying store." + with pytest.raises( + NotImplementedError, match="LoggingStore must be opened via the `_open` method" + ): + store._is_open = True + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +async def test_logging_store(store: Store, caplog) -> None: + wrapped = LoggingStore(store=store, log_level="DEBUG") + buffer = default_buffer_prototype().buffer + + caplog.clear() + res = await wrapped.set("foo/bar/c/0", buffer.from_bytes(b"\x01\x02\x03\x04")) + assert res is None + assert len(caplog.record_tuples) == 2 + for tup in caplog.record_tuples: + assert str(store) in tup[0] + assert f"Calling {type(store).__name__}.set" in caplog.record_tuples[0][2] + assert f"Finished {type(store).__name__}.set" in caplog.record_tuples[1][2] + + caplog.clear() + keys = [k async for k in wrapped.list()] + assert keys == ["foo/bar/c/0"] + assert len(caplog.record_tuples) == 2 + for tup in caplog.record_tuples: + assert str(store) in tup[0] + assert f"Calling {type(store).__name__}.list" in caplog.record_tuples[0][2] + assert f"Finished {type(store).__name__}.list" in caplog.record_tuples[1][2] + + +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +async def test_logging_store_counter(store: Store) -> None: + wrapped = LoggingStore(store=store, log_level="DEBUG") + + arr = zarr.create(shape=(10,), store=wrapped, overwrite=True) + arr[:] = 1 + + assert wrapped.counter["set"] == 2 + assert wrapped.counter["list"] == 0 + assert wrapped.counter["list_dir"] == 0 + assert wrapped.counter["list_prefix"] == 0 + if store.supports_deletes: + assert wrapped.counter["get"] == 0 # 1 if overwrite=False + assert wrapped.counter["delete_dir"] == 1 + else: + assert wrapped.counter["get"] == 1 + assert wrapped.counter["delete_dir"] == 0 diff --git a/tests/test_store/test_memory.py b/tests/test_store/test_memory.py new file mode 100644 index 0000000000..4fc3f6e698 --- /dev/null +++ b/tests/test_store/test_memory.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any + +import numpy as np +import numpy.typing as npt +import pytest + +import zarr +from zarr.core.buffer import Buffer, cpu, gpu +from zarr.storage import GpuMemoryStore, MemoryStore +from zarr.testing.store import StoreTests +from zarr.testing.utils import gpu_test + +if TYPE_CHECKING: + from zarr.core.common import ZarrFormat + + +# TODO: work out where this warning is coming from and fix it +@pytest.mark.filterwarnings( + re.escape("ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited") +) +class TestMemoryStore(StoreTests[MemoryStore, cpu.Buffer]): + store_cls = MemoryStore + buffer_cls = cpu.Buffer + + async def set(self, store: MemoryStore, key: str, value: Buffer) -> None: + store._store_dict[key] = value + + async def get(self, store: MemoryStore, key: str) -> Buffer: + return store._store_dict[key] + + @pytest.fixture(params=[None, True]) + def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, Any]: + kwargs: dict[str, Any] + if request.param is True: + kwargs = {"store_dict": {}} + else: + kwargs = {"store_dict": None} + return kwargs + + @pytest.fixture + async def store(self, store_kwargs: dict[str, Any]) -> MemoryStore: + return self.store_cls(**store_kwargs) + + def test_store_repr(self, store: MemoryStore) -> None: + assert str(store) == f"memory://{id(store._store_dict)}" + + def test_store_supports_writes(self, store: MemoryStore) -> None: + assert store.supports_writes + + def test_store_supports_listing(self, store: MemoryStore) -> None: + assert store.supports_listing + + def test_store_supports_partial_writes(self, store: MemoryStore) -> None: + assert store.supports_partial_writes + + async def test_list_prefix(self, store: MemoryStore) -> None: + assert True + + @pytest.mark.parametrize("dtype", ["uint8", "float32", "int64"]) + @pytest.mark.parametrize("zarr_format", [2, 3]) + async def test_deterministic_size( + self, store: MemoryStore, dtype: npt.DTypeLike, zarr_format: ZarrFormat + ) -> None: + a = zarr.empty( + store=store, + shape=(3,), + chunks=(1000,), + dtype=dtype, + zarr_format=zarr_format, + overwrite=True, + ) + a[...] = 1 + a.resize((1000,)) + + np.testing.assert_array_equal(a[:3], 1) + np.testing.assert_array_equal(a[3:], 0) + + +# TODO: fix this warning +@pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") +@gpu_test +class TestGpuMemoryStore(StoreTests[GpuMemoryStore, gpu.Buffer]): + store_cls = GpuMemoryStore + buffer_cls = gpu.Buffer + + async def set(self, store: GpuMemoryStore, key: str, value: gpu.Buffer) -> None: # type: ignore[override] + store._store_dict[key] = value + + async def get(self, store: MemoryStore, key: str) -> Buffer: + return store._store_dict[key] + + @pytest.fixture(params=[None, True]) + def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, Any]: + kwargs: dict[str, Any] + if request.param is True: + kwargs = {"store_dict": {}} + else: + kwargs = {"store_dict": None} + return kwargs + + @pytest.fixture + async def store(self, store_kwargs: dict[str, Any]) -> GpuMemoryStore: + return self.store_cls(**store_kwargs) + + def test_store_repr(self, store: GpuMemoryStore) -> None: + assert str(store) == f"gpumemory://{id(store._store_dict)}" + + def test_store_supports_writes(self, store: GpuMemoryStore) -> None: + assert store.supports_writes + + def test_store_supports_listing(self, store: GpuMemoryStore) -> None: + assert store.supports_listing + + def test_store_supports_partial_writes(self, store: GpuMemoryStore) -> None: + assert store.supports_partial_writes + + async def test_list_prefix(self, store: GpuMemoryStore) -> None: + assert True + + def test_dict_reference(self, store: GpuMemoryStore) -> None: + store_dict: dict[str, Any] = {} + result = GpuMemoryStore(store_dict=store_dict) + assert result._store_dict is store_dict + + def test_from_dict(self) -> None: + d = { + "a": gpu.Buffer.from_bytes(b"aaaa"), + "b": cpu.Buffer.from_bytes(b"bbbb"), + } + result = GpuMemoryStore.from_dict(d) + for v in result._store_dict.values(): + assert type(v) is gpu.Buffer diff --git a/tests/test_store/test_object.py b/tests/test_store/test_object.py new file mode 100644 index 0000000000..4d9e8fcc1f --- /dev/null +++ b/tests/test_store/test_object.py @@ -0,0 +1,86 @@ +# ruff: noqa: E402 +from typing import Any + +import pytest + +obstore = pytest.importorskip("obstore") + +from hypothesis.stateful import ( + run_state_machine_as_test, +) +from obstore.store import LocalStore, MemoryStore + +from zarr.core.buffer import Buffer, cpu +from zarr.storage import ObjectStore +from zarr.testing.stateful import ZarrHierarchyStateMachine +from zarr.testing.store import StoreTests + + +class TestObjectStore(StoreTests[ObjectStore, cpu.Buffer]): + store_cls = ObjectStore + buffer_cls = cpu.Buffer + + @pytest.fixture + def store_kwargs(self, tmpdir) -> dict[str, Any]: + store = LocalStore(prefix=tmpdir) + return {"store": store, "read_only": False} + + @pytest.fixture + def store(self, store_kwargs: dict[str, str | bool]) -> ObjectStore: + return self.store_cls(**store_kwargs) + + async def get(self, store: ObjectStore, key: str) -> Buffer: + assert isinstance(store.store, LocalStore) + new_local_store = LocalStore(prefix=store.store.prefix) + return self.buffer_cls.from_bytes(obstore.get(new_local_store, key).bytes()) + + async def set(self, store: ObjectStore, key: str, value: Buffer) -> None: + assert isinstance(store.store, LocalStore) + new_local_store = LocalStore(prefix=store.store.prefix) + obstore.put(new_local_store, key, value.to_bytes()) + + def test_store_repr(self, store: ObjectStore) -> None: + from fnmatch import fnmatch + + pattern = "ObjectStore(object_store://LocalStore(*))" + assert fnmatch(f"{store!r}", pattern) + + def test_store_supports_writes(self, store: ObjectStore) -> None: + assert store.supports_writes + + async def test_store_supports_partial_writes(self, store: ObjectStore) -> None: + assert not store.supports_partial_writes + with pytest.raises(NotImplementedError): + await store.set_partial_values([("foo", 0, b"\x01\x02\x03\x04")]) + + def test_store_supports_listing(self, store: ObjectStore) -> None: + assert store.supports_listing + + def test_store_equal(self, store: ObjectStore) -> None: + """Test store equality""" + # Test equality against a different instance type + assert store != 0 + # Test equality against a different store type + new_memory_store = ObjectStore(MemoryStore()) + assert store != new_memory_store + # Test equality against a read only store + new_local_store = ObjectStore(LocalStore(prefix=store.store.prefix), read_only=True) + assert store != new_local_store + # Test two memory stores cannot be equal + second_memory_store = ObjectStore(MemoryStore()) + assert new_memory_store != second_memory_store + + def test_store_init_raises(self) -> None: + """Test __init__ raises appropriate error for improper store type""" + with pytest.raises(TypeError): + ObjectStore("path/to/store") + + +@pytest.mark.slow_hypothesis +def test_zarr_hierarchy(): + sync_store = ObjectStore(MemoryStore()) + + def mk_test_instance_sync() -> ZarrHierarchyStateMachine: + return ZarrHierarchyStateMachine(sync_store) + + run_state_machine_as_test(mk_test_instance_sync) diff --git a/tests/test_store/test_stateful.py b/tests/test_store/test_stateful.py new file mode 100644 index 0000000000..c0997c3df3 --- /dev/null +++ b/tests/test_store/test_stateful.py @@ -0,0 +1,41 @@ +# Stateful tests for arbitrary Zarr stores. +import pytest +from hypothesis.stateful import ( + run_state_machine_as_test, +) + +from zarr.abc.store import Store +from zarr.storage import LocalStore, ZipStore +from zarr.testing.stateful import ZarrHierarchyStateMachine, ZarrStoreStateMachine + +pytestmark = [ + pytest.mark.slow_hypothesis, + # TODO: work out where this warning is coming from and fix + pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning"), +] + + +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_zarr_hierarchy(sync_store: Store): + def mk_test_instance_sync() -> ZarrHierarchyStateMachine: + return ZarrHierarchyStateMachine(sync_store) + + if isinstance(sync_store, ZipStore): + pytest.skip(reason="ZipStore does not support delete") + + run_state_machine_as_test(mk_test_instance_sync) + + +def test_zarr_store(sync_store: Store) -> None: + def mk_test_instance_sync() -> None: + return ZarrStoreStateMachine(sync_store) + + if isinstance(sync_store, ZipStore): + pytest.skip(reason="ZipStore does not support delete") + + if isinstance(sync_store, LocalStore): + # This test uses arbitrary keys, which are passed to `set` and `delete`. + # It assumes that `set` and `delete` are the only two operations that modify state. + # But LocalStore, directories can hang around even after a key is delete-d. + pytest.skip(reason="Test isn't suitable for LocalStore.") + run_state_machine_as_test(mk_test_instance_sync) diff --git a/tests/test_store/test_wrapper.py b/tests/test_store/test_wrapper.py new file mode 100644 index 0000000000..c6edd4f4dd --- /dev/null +++ b/tests/test_store/test_wrapper.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from zarr.core.buffer.cpu import Buffer, buffer_prototype +from zarr.storage import LocalStore, WrapperStore +from zarr.testing.store import StoreTests + +if TYPE_CHECKING: + from _pytest.compat import LEGACY_PATH + + from zarr.abc.store import Store + from zarr.core.buffer.core import BufferPrototype + + +# TODO: fix this warning +@pytest.mark.filterwarnings( + "ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited:RuntimeWarning" +) +class TestWrapperStore(StoreTests[WrapperStore, Buffer]): + store_cls = WrapperStore + buffer_cls = Buffer + + async def get(self, store: WrapperStore, key: str) -> Buffer: + return self.buffer_cls.from_bytes((store._store.root / key).read_bytes()) + + async def set(self, store: WrapperStore, key: str, value: Buffer) -> None: + parent = (store._store.root / key).parent + if not parent.exists(): + parent.mkdir(parents=True) + (store._store.root / key).write_bytes(value.to_bytes()) + + @pytest.fixture + def store_kwargs(self, tmpdir: LEGACY_PATH) -> dict[str, str]: + return {"store": LocalStore(str(tmpdir))} + + @pytest.fixture + def open_kwargs(self, tmpdir) -> dict[str, str]: + return {"store_cls": LocalStore, "root": str(tmpdir)} + + def test_store_supports_writes(self, store: WrapperStore) -> None: + assert store.supports_writes + + def test_store_supports_partial_writes(self, store: WrapperStore) -> None: + assert store.supports_partial_writes + + def test_store_supports_listing(self, store: WrapperStore) -> None: + assert store.supports_listing + + def test_store_repr(self, store: WrapperStore) -> None: + assert f"{store!r}" == f"WrapperStore(LocalStore, 'file://{store._store.root.as_posix()}')" + + def test_store_str(self, store: WrapperStore) -> None: + assert str(store) == f"wrapping-file://{store._store.root.as_posix()}" + + def test_check_writeable(self, store: WrapperStore) -> None: + """ + Test _check_writeable() runs without errors. + """ + store._check_writable() + + def test_close(self, store: WrapperStore) -> None: + "Test store can be closed" + store.close() + assert not store._is_open + + def test_is_open_setter_raises(self, store: WrapperStore) -> None: + """ + Test that a user cannot change `_is_open` without opening the underlying store. + """ + with pytest.raises( + NotImplementedError, match="WrapperStore must be opened via the `_open` method" + ): + store._is_open = True + + +# TODO: work out where warning is coming from and fix +@pytest.mark.filterwarnings( + "ignore:coroutine 'ClientCreatorContext.__aexit__' was never awaited:RuntimeWarning" +) +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) +async def test_wrapped_set(store: Store, capsys: pytest.CaptureFixture[str]) -> None: + # define a class that prints when it sets + class NoisySetter(WrapperStore): + async def set(self, key: str, value: Buffer) -> None: + print(f"setting {key}") + await super().set(key, value) + + key = "foo" + value = Buffer.from_bytes(b"bar") + store_wrapped = NoisySetter(store) + await store_wrapped.set(key, value) + captured = capsys.readouterr() + assert f"setting {key}" in captured.out + assert await store_wrapped.get(key, buffer_prototype) == value + + +@pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=True) +async def test_wrapped_get(store: Store, capsys: pytest.CaptureFixture[str]) -> None: + # define a class that prints when it sets + class NoisyGetter(WrapperStore): + def get(self, key: str, prototype: BufferPrototype) -> None: + print(f"getting {key}") + return super().get(key, prototype=prototype) + + key = "foo" + value = Buffer.from_bytes(b"bar") + store_wrapped = NoisyGetter(store) + await store_wrapped.set(key, value) + assert await store_wrapped.get(key, buffer_prototype) == value + captured = capsys.readouterr() + assert f"getting {key}" in captured.out diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py new file mode 100644 index 0000000000..24b25ed315 --- /dev/null +++ b/tests/test_store/test_zip.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import os +import shutil +import tempfile +import zipfile +from typing import TYPE_CHECKING + +import numpy as np +import pytest + +import zarr +from zarr import create_array +from zarr.core.buffer import Buffer, cpu, default_buffer_prototype +from zarr.core.group import Group +from zarr.storage import ZipStore +from zarr.testing.store import StoreTests + +if TYPE_CHECKING: + from pathlib import Path + from typing import Any + + +# TODO: work out where this is coming from and fix +pytestmark = [ + pytest.mark.filterwarnings( + "ignore:coroutine method 'aclose' of 'ZipStore.list' was never awaited:RuntimeWarning" + ) +] + + +class TestZipStore(StoreTests[ZipStore, cpu.Buffer]): + store_cls = ZipStore + buffer_cls = cpu.Buffer + + @pytest.fixture + def store_kwargs(self) -> dict[str, str | bool]: + fd, temp_path = tempfile.mkstemp() + os.close(fd) + os.unlink(temp_path) + + return {"path": temp_path, "mode": "w", "read_only": False} + + async def get(self, store: ZipStore, key: str) -> Buffer: + buf = store._get(key, prototype=default_buffer_prototype()) + assert buf is not None + return buf + + async def set(self, store: ZipStore, key: str, value: Buffer) -> None: + return store._set(key, value) + + def test_store_read_only(self, store: ZipStore) -> None: + assert not store.read_only + + async def test_read_only_store_raises(self, store_kwargs: dict[str, Any]) -> None: + # we need to create the zipfile in write mode before switching to read mode + store = await self.store_cls.open(**store_kwargs) + store.close() + + kwargs = {**store_kwargs, "mode": "a", "read_only": True} + store = await self.store_cls.open(**kwargs) + assert store._zmode == "a" + assert store.read_only + + # set + with pytest.raises(ValueError): + await store.set("foo", cpu.Buffer.from_bytes(b"bar")) + + def test_store_repr(self, store: ZipStore) -> None: + assert str(store) == f"zip://{store.path}" + + def test_store_supports_writes(self, store: ZipStore) -> None: + assert store.supports_writes + + def test_store_supports_partial_writes(self, store: ZipStore) -> None: + assert store.supports_partial_writes is False + + def test_store_supports_listing(self, store: ZipStore) -> None: + assert store.supports_listing + + # TODO: fix this warning + @pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") + def test_api_integration(self, store: ZipStore) -> None: + root = zarr.open_group(store=store, mode="a") + + data = np.arange(10000, dtype=np.uint16).reshape(100, 100) + z = root.create_array( + shape=data.shape, chunks=(10, 10), name="foo", dtype=np.uint16, fill_value=99 + ) + z[:] = data + + assert np.array_equal(data, z[:]) + + # you can overwrite existing chunks but zipfile will issue a warning + with pytest.warns(UserWarning, match="Duplicate name: 'foo/c/0/0'"): + z[0, 0] = 100 + + # TODO: assigning an entire chunk to fill value ends up deleting the chunk which is not supported + # a work around will be needed here. + with pytest.raises(NotImplementedError): + z[0:10, 0:10] = 99 + + bar = root.create_group("bar", attributes={"hello": "world"}) + assert "hello" in dict(bar.attrs) + + # keys cannot be deleted + with pytest.raises(NotImplementedError): + del root["bar"] + + store.close() + + @pytest.mark.parametrize("read_only", [True, False]) + async def test_store_open_read_only( + self, store_kwargs: dict[str, Any], read_only: bool + ) -> None: + if read_only: + # create an empty zipfile + with zipfile.ZipFile(store_kwargs["path"], mode="w"): + pass + + await super().test_store_open_read_only(store_kwargs, read_only) + + @pytest.mark.parametrize(("zip_mode", "read_only"), [("w", False), ("a", False), ("x", False)]) + async def test_zip_open_mode_translation( + self, store_kwargs: dict[str, Any], zip_mode: str, read_only: bool + ) -> None: + kws = {**store_kwargs, "mode": zip_mode} + store = await self.store_cls.open(**kws) + assert store.read_only == read_only + + def test_externally_zipped_store(self, tmp_path: Path) -> None: + # See: https://github.com/zarr-developers/zarr-python/issues/2757 + zarr_path = tmp_path / "foo.zarr" + root = zarr.open_group(store=zarr_path, mode="w") + root.require_group("foo") + assert isinstance(foo := root["foo"], Group) # noqa: RUF018 + foo["bar"] = np.array([1]) + shutil.make_archive(str(zarr_path), "zip", zarr_path) + zip_path = tmp_path / "foo.zarr.zip" + zipped = zarr.open_group(ZipStore(zip_path, mode="r"), mode="r") + assert list(zipped.keys()) == list(root.keys()) + assert isinstance(group := zipped["foo"], Group) + assert list(group.keys()) == list(group.keys()) + + async def test_move(self, tmp_path: Path) -> None: + origin = tmp_path / "origin.zip" + destination = tmp_path / "some_folder" / "destination.zip" + + store = await ZipStore.open(path=origin, mode="a") + array = create_array(store, data=np.arange(10)) + + await store.move(str(destination)) + + assert store.path == destination + assert destination.exists() + assert not origin.exists() + assert np.array_equal(array[...], np.arange(10)) diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000000..13b475f8da --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,172 @@ +import asyncio +from collections.abc import AsyncGenerator +from unittest.mock import AsyncMock, patch + +import pytest + +import zarr +from zarr.core.sync import ( + SyncError, + SyncMixin, + _get_executor, + _get_lock, + _get_loop, + cleanup_resources, + loop, + sync, +) +from zarr.storage import MemoryStore + + +@pytest.fixture(params=[True, False]) +def sync_loop(request: pytest.FixtureRequest) -> asyncio.AbstractEventLoop | None: + if request.param is True: + return _get_loop() + else: + return None + + +@pytest.fixture +def clean_state(): + # use this fixture to make sure no existing threads/loops exist in zarr.core.sync + cleanup_resources() + yield + cleanup_resources() + + +def test_get_loop() -> None: + # test that calling _get_loop() twice returns the same loop + loop = _get_loop() + loop2 = _get_loop() + assert loop is loop2 + + +def test_get_lock() -> None: + # test that calling _get_lock() twice returns the same lock + lock = _get_lock() + lock2 = _get_lock() + assert lock is lock2 + + +def test_sync(sync_loop: asyncio.AbstractEventLoop | None) -> None: + foo = AsyncMock(return_value="foo") + assert sync(foo(), loop=sync_loop) == "foo" + foo.assert_awaited_once() + + +def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: + foo = AsyncMock(side_effect=ValueError("foo-bar")) + with pytest.raises(ValueError, match="foo-bar"): + sync(foo(), loop=sync_loop) + foo.assert_awaited_once() + + +def test_sync_timeout() -> None: + duration = 0.02 + + async def foo() -> None: + await asyncio.sleep(duration) + + with pytest.raises(asyncio.TimeoutError): + sync(foo(), timeout=duration / 10) + + +def test_sync_raises_if_no_coroutine(sync_loop: asyncio.AbstractEventLoop | None) -> None: + def foo() -> str: + return "foo" + + with pytest.raises(TypeError): + sync(foo(), loop=sync_loop) # type: ignore[arg-type] + + +@pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") +def test_sync_raises_if_loop_is_closed() -> None: + loop = _get_loop() + + foo = AsyncMock(return_value="foo") + with patch.object(loop, "is_closed", return_value=True): + with pytest.raises(RuntimeError): + sync(foo(), loop=loop) + foo.assert_not_awaited() + + +@pytest.mark.filterwarnings("ignore:Unclosed client session:ResourceWarning") +@pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") +def test_sync_raises_if_calling_sync_from_within_a_running_loop( + sync_loop: asyncio.AbstractEventLoop | None, +) -> None: + def foo() -> str: + # technically, this should be an async function but doing that + # yields a warning because it is never awaited by the inner function + return "foo" + + async def bar() -> str: + return sync(foo(), loop=sync_loop) # type: ignore[arg-type] + + with pytest.raises(SyncError): + sync(bar(), loop=sync_loop) + + +@pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") +def test_sync_raises_if_loop_is_invalid_type() -> None: + foo = AsyncMock(return_value="foo") + with pytest.raises(TypeError): + sync(foo(), loop=1) # type: ignore[arg-type] + foo.assert_not_awaited() + + +def test_sync_mixin(sync_loop) -> None: + class AsyncFoo: + def __init__(self) -> None: + pass + + async def foo(self) -> str: + return "foo" + + async def bar(self) -> AsyncGenerator: + for i in range(10): + yield i + + class SyncFoo(SyncMixin): + def __init__(self, async_foo: AsyncFoo) -> None: + self._async_foo = async_foo + + def foo(self) -> str: + return self._sync(self._async_foo.foo()) + + def bar(self) -> list[int]: + return self._sync_iter(self._async_foo.bar()) + + async_foo = AsyncFoo() + foo = SyncFoo(async_foo) + assert foo.foo() == "foo" + assert foo.bar() == list(range(10)) + + +def test_open_positional_args_deprecate(): + store = MemoryStore() + with pytest.warns(FutureWarning, match="pass"): + zarr.open(store, "w", shape=(1,)) + + +@pytest.mark.parametrize("workers", [None, 1, 2]) +def test_threadpool_executor(clean_state, workers: int | None) -> None: + with zarr.config.set({"threading.max_workers": workers}): + _ = zarr.zeros(shape=(1,)) # trigger executor creation + assert loop != [None] # confirm loop was created + if workers is None: + # confirm no executor was created if no workers were specified + # (this is the default behavior) + assert loop[0]._default_executor is None + else: + # confirm executor was created and attached to loop as the default executor + # note: python doesn't have a direct way to get the default executor so we + # use the private attribute + assert _get_executor() is loop[0]._default_executor + assert _get_executor()._max_workers == workers + + +def test_cleanup_resources_idempotent() -> None: + _get_executor() # trigger resource creation (iothread, loop, thread-pool) + cleanup_resources() + cleanup_resources() diff --git a/tests/test_tree.py b/tests/test_tree.py new file mode 100644 index 0000000000..b4a5106998 --- /dev/null +++ b/tests/test_tree.py @@ -0,0 +1,62 @@ +import os +import textwrap +from typing import Any + +import pytest + +import zarr + +pytest.importorskip("rich") + + +@pytest.mark.parametrize("root_name", [None, "root"]) +def test_tree(root_name: Any) -> None: + os.environ["OVERRIDE_COLOR_SYSTEM"] = "truecolor" + + g = zarr.group(path=root_name) + A = g.create_group("A") + B = g.create_group("B") + C = B.create_group("C") + D = C.create_group("C") + + A.create_array(name="x", shape=(2), dtype="float64") + A.create_array(name="y", shape=(0,), dtype="int8") + B.create_array(name="x", shape=(0,), dtype="float64") + C.create_array(name="x", shape=(0,), dtype="float64") + D.create_array(name="x", shape=(0,), dtype="float64") + + result = repr(g.tree()) + root = root_name or "" + + BOPEN = "\x1b[1m" + BCLOSE = "\x1b[0m" + + expected = textwrap.dedent(f"""\ + {BOPEN}/{root}{BCLOSE} + ├── {BOPEN}A{BCLOSE} + │ ├── {BOPEN}x{BCLOSE} (2,) float64 + │ └── {BOPEN}y{BCLOSE} (0,) int8 + └── {BOPEN}B{BCLOSE} + ├── {BOPEN}C{BCLOSE} + │ ├── {BOPEN}C{BCLOSE} + │ │ └── {BOPEN}x{BCLOSE} (0,) float64 + │ └── {BOPEN}x{BCLOSE} (0,) float64 + └── {BOPEN}x{BCLOSE} (0,) float64 + """) + + assert result == expected + + result = repr(g.tree(level=0)) + expected = textwrap.dedent(f"""\ + {BOPEN}/{root}{BCLOSE} + ├── {BOPEN}A{BCLOSE} + └── {BOPEN}B{BCLOSE} + """) + + assert result == expected + + +def test_expand_not_implemented() -> None: + g = zarr.group() + with pytest.raises(NotImplementedError): + g.tree(expand=True) diff --git a/tests/test_v2.py b/tests/test_v2.py new file mode 100644 index 0000000000..29f031663f --- /dev/null +++ b/tests/test_v2.py @@ -0,0 +1,315 @@ +import json +from pathlib import Path +from typing import Any, Literal + +import numcodecs.abc +import numcodecs.vlen +import numpy as np +import pytest +from numcodecs import Delta +from numcodecs.blosc import Blosc +from numcodecs.zstd import Zstd + +import zarr +import zarr.core.buffer +import zarr.storage +from zarr import config +from zarr.abc.store import Store +from zarr.core.buffer.core import default_buffer_prototype +from zarr.core.dtype import FixedLengthUTF32, Structured, VariableLengthUTF8 +from zarr.core.dtype.npy.bytes import NullTerminatedBytes +from zarr.core.dtype.wrapper import ZDType +from zarr.core.group import Group +from zarr.core.sync import sync +from zarr.storage import MemoryStore, StorePath + + +@pytest.fixture +async def store() -> StorePath: + return StorePath(await MemoryStore.open()) + + +def test_simple(store: StorePath) -> None: + data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) + + a = zarr.create_array( + store / "simple_v2", + zarr_format=2, + shape=data.shape, + chunks=(16, 16), + dtype=data.dtype, + fill_value=0, + ) + + a[:, :] = data + assert np.array_equal(data, a[:, :]) + + +def test_codec_pipeline() -> None: + # https://github.com/zarr-developers/zarr-python/issues/2243 + store = MemoryStore() + array = zarr.create( + store=store, + shape=(1,), + dtype="i4", + zarr_format=2, + filters=[Delta(dtype="i4").get_config()], + compressor=Blosc().get_config(), + ) + array[:] = 1 + result = array[:] + expected = np.ones(1) + np.testing.assert_array_equal(result, expected) + + +@pytest.mark.parametrize( + ("dtype", "expected_dtype", "fill_value", "fill_value_json"), + [ + ("|S1", "|S1", b"X", "WA=="), + ("|V1", "|V1", b"X", "WA=="), + ("|V10", "|V10", b"X", "WAAAAAAAAAAAAA=="), + ], +) +async def test_v2_encode_decode( + dtype: str, expected_dtype: str, fill_value: bytes, fill_value_json: str +) -> None: + with config.set( + { + "array.v2_default_filters.bytes": [{"id": "vlen-bytes"}], + "array.v2_default_compressor.bytes": None, + } + ): + store = zarr.storage.MemoryStore() + g = zarr.group(store=store, zarr_format=2) + g.create_array( + name="foo", shape=(3,), chunks=(3,), dtype=dtype, fill_value=fill_value, compressor=None + ) + + result = await store.get("foo/.zarray", zarr.core.buffer.default_buffer_prototype()) + assert result is not None + + serialized = json.loads(result.to_bytes()) + expected = { + "chunks": [3], + "compressor": None, + "dtype": expected_dtype, + "fill_value": fill_value_json, + "filters": None, + "order": "C", + "shape": [3], + "zarr_format": 2, + "dimension_separator": ".", + } + assert serialized == expected + + data = zarr.open_array(store=store, path="foo")[:] + np.testing.assert_equal(data, np.full((3,), b"X", dtype=dtype)) + + +@pytest.mark.parametrize( + ("dtype", "value"), + [ + (NullTerminatedBytes(length=1), b"Y"), + (FixedLengthUTF32(length=1), "Y"), + (VariableLengthUTF8(), "Y"), + ], +) +def test_v2_encode_decode_with_data(dtype: ZDType[Any, Any], value: str) -> None: + expected = np.full((3,), value, dtype=dtype.to_native_dtype()) + a = zarr.create( + shape=(3,), + zarr_format=2, + dtype=dtype, + ) + a[:] = expected + data = a[:] + np.testing.assert_equal(data, expected) + + +@pytest.mark.parametrize("filters", [[], [numcodecs.Delta(dtype=" None: + array_fixture = [42] + with config.set({"array.order": order}): + arr = zarr.create(shape=1, dtype=" None: + """ + Test that passing compressor=None results in no compressor. Also test that the default value of the compressor + parameter does produce a compressor. + """ + g = zarr.open(store, mode="w", zarr_format=2) + assert isinstance(g, Group) + arr = g.create_array("one", dtype="i8", shape=(1,), chunks=(1,), compressor=None) + assert arr._async_array.compressor is None + assert not (arr.filters) + arr = g.create_array("two", dtype="i8", shape=(1,), chunks=(1,)) + assert arr._async_array.compressor is not None + assert not (arr.filters) + arr = g.create_array("three", dtype="i8", shape=(1,), chunks=(1,), compressor=Zstd()) + assert arr._async_array.compressor is not None + assert not (arr.filters) + with pytest.raises(ValueError): + g.create_array( + "four", dtype="i8", shape=(1,), chunks=(1,), compressor=None, compressors=None + ) + + +@pytest.mark.parametrize("numpy_order", ["C", "F"]) +@pytest.mark.parametrize("zarr_order", ["C", "F"]) +def test_v2_non_contiguous(numpy_order: Literal["C", "F"], zarr_order: Literal["C", "F"]) -> None: + """ + Make sure zarr v2 arrays save data using the memory order given to the zarr array, + not the memory order of the original numpy array. + """ + store = MemoryStore() + arr = zarr.create_array( + store, + shape=(10, 8), + chunks=(3, 3), + fill_value=np.nan, + dtype="float64", + zarr_format=2, + filters=None, + compressors=None, + overwrite=True, + order=zarr_order, + ) + + # Non-contiguous write, using numpy memory order + a = np.arange(arr.shape[0] * arr.shape[1]).reshape(arr.shape, order=numpy_order) + arr[6:9, 3:6] = a[6:9, 3:6] # The slice on the RHS is important + np.testing.assert_array_equal(arr[6:9, 3:6], a[6:9, 3:6]) + + buf = sync(store.get("2.1", default_buffer_prototype())) + assert buf is not None + np.testing.assert_array_equal( + a[6:9, 3:6], + np.frombuffer(buf.to_bytes(), dtype="float64").reshape((3, 3), order=zarr_order), + ) + # After writing and reading from zarr array, order should be same as zarr order + sub_arr = arr[6:9, 3:6] + assert isinstance(sub_arr, np.ndarray) + if zarr_order == "F": + assert (sub_arr).flags.f_contiguous + else: + assert (sub_arr).flags.c_contiguous + + # Contiguous write + store = MemoryStore() + arr = zarr.create_array( + store, + shape=(10, 8), + chunks=(3, 3), + fill_value=np.nan, + dtype="float64", + zarr_format=2, + compressors=None, + filters=None, + overwrite=True, + order=zarr_order, + ) + + a = np.arange(9).reshape((3, 3), order=numpy_order) + arr[6:9, 3:6] = a + np.testing.assert_array_equal(arr[6:9, 3:6], a) + # After writing and reading from zarr array, order should be same as zarr order + sub_arr = arr[6:9, 3:6] + assert isinstance(sub_arr, np.ndarray) + if zarr_order == "F": + assert (sub_arr).flags.f_contiguous + else: + assert (sub_arr).flags.c_contiguous + + +def test_default_compressor_deprecation_warning() -> None: + with pytest.warns(DeprecationWarning, match="default_compressor is deprecated"): + zarr.storage.default_compressor = "zarr.codecs.zstd.ZstdCodec()" # type: ignore[attr-defined] + + +@pytest.mark.parametrize("fill_value", [None, (b"", 0, 0.0)], ids=["no_fill", "fill"]) +def test_structured_dtype_roundtrip(fill_value: float | bytes, tmp_path: Path) -> None: + a = np.array( + [(b"aaa", 1, 4.2), (b"bbb", 2, 8.4), (b"ccc", 3, 12.6)], + dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")], + ) + array_path = tmp_path / "data.zarr" + za = zarr.create( + shape=(3,), + store=array_path, + chunks=(2,), + fill_value=fill_value, + zarr_format=2, + dtype=a.dtype, + ) + if fill_value is not None: + assert (np.array([fill_value] * a.shape[0], dtype=a.dtype) == za[:]).all() + za[...] = a + za = zarr.open_array(store=array_path) + assert (a == za[:]).all() + + +@pytest.mark.parametrize( + ( + "fill_value", + "dtype", + "expected_result", + ), + [ + ( + ("Alice", 30), + np.dtype([("name", "U10"), ("age", "i4")]), + np.array([("Alice", 30)], dtype=[("name", "U10"), ("age", "i4")])[0], + ), + ( + ["Bob", 25], + np.dtype([("name", "U10"), ("age", "i4")]), + np.array([("Bob", 25)], dtype=[("name", "U10"), ("age", "i4")])[0], + ), + ( + b"\x01\x00\x00\x00\x02\x00\x00\x00", + np.dtype([("x", "i4"), ("y", "i4")]), + np.array([(1, 2)], dtype=[("x", "i4"), ("y", "i4")])[0], + ), + ], + ids=[ + "tuple_input", + "list_input", + "bytes_input", + ], +) +def test_parse_structured_fill_value_valid( + fill_value: Any, dtype: np.dtype[Any], expected_result: Any +) -> None: + zdtype = Structured.from_native_dtype(dtype) + result = zdtype.cast_scalar(fill_value) + assert result.dtype == expected_result.dtype + assert result == expected_result + if isinstance(expected_result, np.void): + for name in expected_result.dtype.names or []: + assert result[name] == expected_result[name] + + +@pytest.mark.parametrize("fill_value", [None, b"x"], ids=["no_fill", "fill"]) +def test_other_dtype_roundtrip(fill_value: None | bytes, tmp_path: Path) -> None: + a = np.array([b"a\0\0", b"bb", b"ccc"], dtype="V7") + array_path = tmp_path / "data.zarr" + za = zarr.create( + shape=(3,), + store=array_path, + chunks=(2,), + fill_value=fill_value, + zarr_format=2, + dtype=a.dtype, + ) + if fill_value is not None: + assert (np.array([fill_value] * a.shape[0], dtype=a.dtype) == za[:]).all() + za[...] = a + za = zarr.open_array(store=array_path) + assert (a == za[:]).all() diff --git a/tests/test_zarr.py b/tests/test_zarr.py new file mode 100644 index 0000000000..f49873132e --- /dev/null +++ b/tests/test_zarr.py @@ -0,0 +1,29 @@ +import pytest + +import zarr + + +def test_exports() -> None: + """ + Ensure that everything in __all__ can be imported. + """ + from zarr import __all__ + + for export in __all__: + getattr(zarr, export) + + +def test_print_debug_info(capsys: pytest.CaptureFixture[str]) -> None: + """ + Ensure that print_debug_info does not raise an error + """ + from importlib.metadata import version + + from zarr import __version__, print_debug_info + + print_debug_info() + captured = capsys.readouterr() + # test that at least some of what we expect is + # printed out + assert f"zarr: {__version__}" in captured.out + assert f"numpy: {version('numpy')}" in captured.out diff --git a/tox.ini b/tox.ini deleted file mode 100644 index a9256a9979..0000000000 --- a/tox.ini +++ /dev/null @@ -1,49 +0,0 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - -[tox] -envlist = py37-npy{117,latest}, py38, docs - -[testenv] -install_command = pip install --no-binary=numcodecs {opts} {packages} -setenv = - PYTHONHASHSEED = 42 -passenv = - ZARR_TEST_ABS - ZARR_TEST_MONGO - ZARR_TEST_REDIS -commands = - # clear out any data files generated during tests - python -c 'import glob; import shutil; import os; [(shutil.rmtree(d) if os.path.isdir(d) else os.remove(d) if os.path.isfile(d) else None) for d in glob.glob("./example*")]' - # main unit test runner - py38: pytest -v --cov=zarr --cov-config=.coveragerc zarr - # don't collect coverage when running older numpy versions - py37-npy117: pytest -v zarr - # collect coverage and run doctests under py37 - py37-npylatest: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-plus zarr --remote-data - # generate a coverage report - py37-npylatest,py38: coverage report -m - # run doctests in the tutorial and spec - py38: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst - # pep8 checks - py38: flake8 zarr - # print environment for debugging - pip freeze -deps = - py37-npy117: numpy==1.17.* - py37-npylatest,py38: -rrequirements_dev_numpy.txt - -rrequirements_dev_minimal.txt - -rrequirements_dev_optional.txt - -[testenv:docs] -basepython = python3.7 -changedir = docs -deps = - -rrequirements_rtfd.txt -commands = - sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html - -[flake8] -max-line-length = 100 diff --git a/windows_conda_dev.txt b/windows_conda_dev.txt deleted file mode 100644 index 576674827d..0000000000 --- a/windows_conda_dev.txt +++ /dev/null @@ -1,9 +0,0 @@ -coverage -fasteners -flake8 -monotonic -msgpack-python -numcodecs -numpy -setuptools_scm -twine diff --git a/zarr/__init__.py b/zarr/__init__.py deleted file mode 100644 index 8079bab071..0000000000 --- a/zarr/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# flake8: noqa -from zarr.codecs import * -from zarr.convenience import (consolidate_metadata, copy, copy_all, copy_store, - load, open, open_consolidated, save, save_array, - save_group, tree) -from zarr.core import Array -from zarr.creation import (array, create, empty, empty_like, full, full_like, - ones, ones_like, open_array, open_like, zeros, - zeros_like) -from zarr.errors import CopyError, MetadataError -from zarr.hierarchy import Group, group, open_group -from zarr.n5 import N5Store -from zarr.storage import (ABSStore, DBMStore, DictStore, DirectoryStore, - LMDBStore, LRUStoreCache, MemoryStore, MongoDBStore, - NestedDirectoryStore, RedisStore, SQLiteStore, - TempStore, ZipStore) -from zarr.sync import ProcessSynchronizer, ThreadSynchronizer -from zarr.version import version as __version__ - -# in case setuptools scm screw up and find version to be 0.0.0 -assert not __version__.startswith("0.0.0") diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py deleted file mode 100644 index 0dc5bf1892..0000000000 --- a/zarr/_storage/absstore.py +++ /dev/null @@ -1,200 +0,0 @@ -"""This module contains storage classes related to Azure Blob Storage (ABS)""" - -import warnings -from collections.abc import MutableMapping -from numcodecs.compat import ensure_bytes -from zarr.util import normalize_storage_path - -__doctest_requires__ = { - ('ABSStore', 'ABSStore.*'): ['azure.storage.blob'], -} - - -class ABSStore(MutableMapping): - """Storage class using Azure Blob Storage (ABS). - - Parameters - ---------- - container : string - The name of the ABS container to use. - .. deprecated:: - Use ``client`` instead. - prefix : string - Location of the "directory" to use as the root of the storage hierarchy - within the container. - account_name : string - The Azure blob storage account name. - .. deprecated:: 2.8.3 - Use ``client`` instead. - account_key : string - The Azure blob storage account access key. - .. deprecated:: 2.8.3 - Use ``client`` instead. - blob_service_kwargs : dictionary - Extra arguments to be passed into the azure blob client, for e.g. when - using the emulator, pass in blob_service_kwargs={'is_emulated': True}. - .. deprecated:: 2.8.3 - Use ``client`` instead. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - client : azure.storage.blob.ContainerClient, optional - And ``azure.storage.blob.ContainerClient`` to connect with. See - `here `_ # noqa - for more. - - .. versionadded:: 2.8.3 - - Notes - ----- - In order to use this store, you must install the Microsoft Azure Storage SDK for Python, - ``azure-storage-blob>=12.5.0``. - """ - - def __init__(self, container=None, prefix='', account_name=None, account_key=None, - blob_service_kwargs=None, dimension_separator=None, - client=None, - ): - self._dimension_separator = dimension_separator - self.prefix = normalize_storage_path(prefix) - if client is None: - # deprecated option, try to construct the client for them - msg = ( - "Providing 'container', 'account_name', 'account_key', and 'blob_service_kwargs'" - "is deprecated. Provide and instance of 'azure.storage.blob.ContainerClient' " - "'client' instead." - ) - warnings.warn(msg, FutureWarning, stacklevel=2) - from azure.storage.blob import ContainerClient - blob_service_kwargs = blob_service_kwargs or {} - client = ContainerClient( - "https://{}.blob.core.windows.net/".format(account_name), container, - credential=account_key, **blob_service_kwargs - ) - - self.client = client - self._container = container - self._account_name = account_name - self._account_key = account_key - - def _warn_deprecated(self, property_): - msg = ("The {} property is deprecated and will be removed in a future " - "version. Get the property from 'ABSStore.client' instead.") - warnings.warn(msg.format(property_), FutureWarning, stacklevel=3) - - @property - def container(self): - self._warn_deprecated("container") - return self._container - - @property - def account_name(self): - self._warn_deprecated("account_name") - return self._account_name - - @property - def account_key(self): - self._warn_deprecated("account_key") - return self._account_key - - def _append_path_to_prefix(self, path): - if self.prefix == '': - return normalize_storage_path(path) - else: - return '/'.join([self.prefix, normalize_storage_path(path)]) - - @staticmethod - def _strip_prefix_from_path(path, prefix): - # normalized things will not have any leading or trailing slashes - path_norm = normalize_storage_path(path) - prefix_norm = normalize_storage_path(prefix) - if prefix: - return path_norm[(len(prefix_norm)+1):] - else: - return path_norm - - def __getitem__(self, key): - from azure.core.exceptions import ResourceNotFoundError - blob_name = self._append_path_to_prefix(key) - try: - return self.client.download_blob(blob_name).readall() - except ResourceNotFoundError: - raise KeyError('Blob %s not found' % blob_name) - - def __setitem__(self, key, value): - value = ensure_bytes(value) - blob_name = self._append_path_to_prefix(key) - self.client.upload_blob(blob_name, value, overwrite=True) - - def __delitem__(self, key): - from azure.core.exceptions import ResourceNotFoundError - try: - self.client.delete_blob(self._append_path_to_prefix(key)) - except ResourceNotFoundError: - raise KeyError('Blob %s not found' % key) - - def __eq__(self, other): - return ( - isinstance(other, ABSStore) and - self.client == other.client and - self.prefix == other.prefix - ) - - def keys(self): - return list(self.__iter__()) - - def __iter__(self): - if self.prefix: - list_blobs_prefix = self.prefix + '/' - else: - list_blobs_prefix = None - for blob in self.client.list_blobs(list_blobs_prefix): - yield self._strip_prefix_from_path(blob.name, self.prefix) - - def __len__(self): - return len(self.keys()) - - def __contains__(self, key): - blob_name = self._append_path_to_prefix(key) - return self.client.get_blob_client(blob_name).exists() - - def listdir(self, path=None): - dir_path = normalize_storage_path(self._append_path_to_prefix(path)) - if dir_path: - dir_path += '/' - items = [ - self._strip_prefix_from_path(blob.name, dir_path) - for blob in self.client.walk_blobs(name_starts_with=dir_path, delimiter='/') - ] - return items - - def rmdir(self, path=None): - dir_path = normalize_storage_path(self._append_path_to_prefix(path)) - if dir_path: - dir_path += '/' - for blob in self.client.list_blobs(name_starts_with=dir_path): - self.client.delete_blob(blob) - - def getsize(self, path=None): - store_path = normalize_storage_path(path) - fs_path = self._append_path_to_prefix(store_path) - if fs_path: - blob_client = self.client.get_blob_client(fs_path) - else: - blob_client = None - - if blob_client and blob_client.exists(): - return blob_client.get_blob_properties().size - else: - size = 0 - if fs_path == '': - fs_path = None - elif not fs_path.endswith('/'): - fs_path += '/' - for blob in self.client.walk_blobs(name_starts_with=fs_path, delimiter='/'): - blob_client = self.client.get_blob_client(blob) - if blob_client.exists(): - size += blob_client.get_blob_properties().size - return size - - def clear(self): - self.rmdir() diff --git a/zarr/attrs.py b/zarr/attrs.py deleted file mode 100644 index ea6b831608..0000000000 --- a/zarr/attrs.py +++ /dev/null @@ -1,142 +0,0 @@ -from collections.abc import MutableMapping - -from zarr.meta import parse_metadata -from zarr.util import json_dumps - - -class Attributes(MutableMapping): - """Class providing access to user attributes on an array or group. Should not be - instantiated directly, will be available via the `.attrs` property of an array or - group. - - Parameters - ---------- - store : MutableMapping - The store in which to store the attributes. - key : str, optional - The key under which the attributes will be stored. - read_only : bool, optional - If True, attributes cannot be modified. - cache : bool, optional - If True (default), attributes will be cached locally. - synchronizer : Synchronizer - Only necessary if attributes may be modified from multiple threads or processes. - - """ - - def __init__(self, store, key='.zattrs', read_only=False, cache=True, - synchronizer=None): - self.store = store - self.key = key - self.read_only = read_only - self.cache = cache - self._cached_asdict = None - self.synchronizer = synchronizer - - def _get_nosync(self): - try: - data = self.store[self.key] - except KeyError: - d = dict() - else: - d = parse_metadata(data) - return d - - def asdict(self): - """Retrieve all attributes as a dictionary.""" - if self.cache and self._cached_asdict is not None: - return self._cached_asdict - d = self._get_nosync() - if self.cache: - self._cached_asdict = d - return d - - def refresh(self): - """Refresh cached attributes from the store.""" - if self.cache: - self._cached_asdict = self._get_nosync() - - def __contains__(self, x): - return x in self.asdict() - - def __getitem__(self, item): - return self.asdict()[item] - - def _write_op(self, f, *args, **kwargs): - - # guard condition - if self.read_only: - raise PermissionError('attributes are read-only') - - # synchronization - if self.synchronizer is None: - return f(*args, **kwargs) - else: - with self.synchronizer[self.key]: - return f(*args, **kwargs) - - def __setitem__(self, item, value): - self._write_op(self._setitem_nosync, item, value) - - def _setitem_nosync(self, item, value): - - # load existing data - d = self._get_nosync() - - # set key value - d[item] = value - - # _put modified data - self._put_nosync(d) - - def __delitem__(self, item): - self._write_op(self._delitem_nosync, item) - - def _delitem_nosync(self, key): - - # load existing data - d = self._get_nosync() - - # delete key value - del d[key] - - # _put modified data - self._put_nosync(d) - - def put(self, d): - """Overwrite all attributes with the key/value pairs in the provided dictionary - `d` in a single operation.""" - self._write_op(self._put_nosync, d) - - def _put_nosync(self, d): - self.store[self.key] = json_dumps(d) - if self.cache: - self._cached_asdict = d - - # noinspection PyMethodOverriding - def update(self, *args, **kwargs): - """Update the values of several attributes in a single operation.""" - self._write_op(self._update_nosync, *args, **kwargs) - - def _update_nosync(self, *args, **kwargs): - - # load existing data - d = self._get_nosync() - - # update - d.update(*args, **kwargs) - - # _put modified data - self._put_nosync(d) - - def keys(self): - return self.asdict().keys() - - def __iter__(self): - return iter(self.asdict()) - - def __len__(self): - return len(self.asdict()) - - def _ipython_key_completions_(self): - return sorted(self) diff --git a/zarr/codecs.py b/zarr/codecs.py deleted file mode 100644 index 4ad68b8627..0000000000 --- a/zarr/codecs.py +++ /dev/null @@ -1,4 +0,0 @@ -# flake8: noqa -from numcodecs import * -from numcodecs import get_codec, Blosc, Pickle, Zlib, Delta, AsType, BZ2 -from numcodecs.registry import codec_registry diff --git a/zarr/convenience.py b/zarr/convenience.py deleted file mode 100644 index 80cf7fffd4..0000000000 --- a/zarr/convenience.py +++ /dev/null @@ -1,1187 +0,0 @@ -"""Convenience functions for storing and loading data.""" -import io -import itertools -import os -import re -from collections.abc import Mapping - -from zarr.core import Array -from zarr.creation import array as _create_array -from zarr.creation import normalize_store_arg, open_array -from zarr.errors import CopyError, PathNotFoundError -from zarr.hierarchy import Group -from zarr.hierarchy import group as _create_group -from zarr.hierarchy import open_group -from zarr.meta import json_dumps, json_loads -from zarr.storage import contains_array, contains_group -from zarr.util import TreeViewer, buffer_size, normalize_storage_path - - -# noinspection PyShadowingBuiltins -def open(store=None, mode='a', **kwargs): - """Convenience function to open a group or array using file-mode-like semantics. - - Parameters - ---------- - store : MutableMapping or string, optional - Store or path to directory in file system or name of zip file. - mode : {'r', 'r+', 'a', 'w', 'w-'}, optional - Persistence mode: 'r' means read only (must exist); 'r+' means - read/write (must exist); 'a' means read/write (create if doesn't - exist); 'w' means create (overwrite if exists); 'w-' means create - (fail if exists). - **kwargs - Additional parameters are passed through to :func:`zarr.creation.open_array` or - :func:`zarr.hierarchy.open_group`. - - Returns - ------- - z : :class:`zarr.core.Array` or :class:`zarr.hierarchy.Group` - Array or group, depending on what exists in the given store. - - See Also - -------- - zarr.creation.open_array, zarr.hierarchy.open_group - - Examples - -------- - - Storing data in a directory 'data/example.zarr' on the local file system:: - - >>> import zarr - >>> store = 'data/example.zarr' - >>> zw = zarr.open(store, mode='w', shape=100, dtype='i4') # open new array - >>> zw - - >>> za = zarr.open(store, mode='a') # open existing array for reading and writing - >>> za - - >>> zr = zarr.open(store, mode='r') # open existing array read-only - >>> zr - - >>> gw = zarr.open(store, mode='w') # open new group, overwriting previous data - >>> gw - - >>> ga = zarr.open(store, mode='a') # open existing group for reading and writing - >>> ga - - >>> gr = zarr.open(store, mode='r') # open existing group read-only - >>> gr - - - """ - - path = kwargs.get('path', None) - # handle polymorphic store arg - clobber = mode == 'w' - # we pass storage options explicitly, since normalize_store_arg might construct - # a store if the input is a fsspec-compatible URL - store = normalize_store_arg(store, clobber=clobber, - storage_options=kwargs.pop("storage_options", {})) - path = normalize_storage_path(path) - - if mode in {'w', 'w-', 'x'}: - if 'shape' in kwargs: - return open_array(store, mode=mode, **kwargs) - else: - return open_group(store, mode=mode, **kwargs) - - elif mode == "a": - if "shape" in kwargs or contains_array(store, path): - return open_array(store, mode=mode, **kwargs) - else: - return open_group(store, mode=mode, **kwargs) - - else: - if contains_array(store, path): - return open_array(store, mode=mode, **kwargs) - elif contains_group(store, path): - return open_group(store, mode=mode, **kwargs) - else: - raise PathNotFoundError(path) - - -def _might_close(path): - return isinstance(path, (str, os.PathLike)) - - -def save_array(store, arr, **kwargs): - """Convenience function to save a NumPy array to the local file system, following a - similar API to the NumPy save() function. - - Parameters - ---------- - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - arr : ndarray - NumPy array with data to save. - kwargs - Passed through to :func:`create`, e.g., compressor. - - Examples - -------- - Save an array to a directory on the file system (uses a :class:`DirectoryStore`):: - - >>> import zarr - >>> import numpy as np - >>> arr = np.arange(10000) - >>> zarr.save_array('data/example.zarr', arr) - >>> zarr.load('data/example.zarr') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - - Save an array to a single file (uses a :class:`ZipStore`):: - - >>> zarr.save_array('data/example.zip', arr) - >>> zarr.load('data/example.zip') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - - """ - may_need_closing = _might_close(store) - store = normalize_store_arg(store, clobber=True) - try: - _create_array(arr, store=store, overwrite=True, **kwargs) - finally: - if may_need_closing and hasattr(store, 'close'): - # needed to ensure zip file records are written - store.close() - - -def save_group(store, *args, **kwargs): - """Convenience function to save several NumPy arrays to the local file system, following a - similar API to the NumPy savez()/savez_compressed() functions. - - Parameters - ---------- - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - args : ndarray - NumPy arrays with data to save. - kwargs - NumPy arrays with data to save. - - Examples - -------- - Save several arrays to a directory on the file system (uses a - :class:`DirectoryStore`): - - >>> import zarr - >>> import numpy as np - >>> a1 = np.arange(10000) - >>> a2 = np.arange(10000, 0, -1) - >>> zarr.save_group('data/example.zarr', a1, a2) - >>> loader = zarr.load('data/example.zarr') - >>> loader - - >>> loader['arr_0'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['arr_1'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - Save several arrays using named keyword arguments:: - - >>> zarr.save_group('data/example.zarr', foo=a1, bar=a2) - >>> loader = zarr.load('data/example.zarr') - >>> loader - - >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - Store several arrays in a single zip file (uses a :class:`ZipStore`):: - - >>> zarr.save_group('data/example.zip', foo=a1, bar=a2) - >>> loader = zarr.load('data/example.zip') - >>> loader - - >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - Notes - ----- - Default compression options will be used. - - """ - if len(args) == 0 and len(kwargs) == 0: - raise ValueError('at least one array must be provided') - # handle polymorphic store arg - may_need_closing = _might_close(store) - store = normalize_store_arg(store, clobber=True) - try: - grp = _create_group(store, overwrite=True) - for i, arr in enumerate(args): - k = 'arr_{}'.format(i) - grp.create_dataset(k, data=arr, overwrite=True) - for k, arr in kwargs.items(): - grp.create_dataset(k, data=arr, overwrite=True) - finally: - if may_need_closing and hasattr(store, 'close'): - # needed to ensure zip file records are written - store.close() - - -def save(store, *args, **kwargs): - """Convenience function to save an array or group of arrays to the local file system. - - Parameters - ---------- - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - args : ndarray - NumPy arrays with data to save. - kwargs - NumPy arrays with data to save. - - Examples - -------- - Save an array to a directory on the file system (uses a :class:`DirectoryStore`):: - - >>> import zarr - >>> import numpy as np - >>> arr = np.arange(10000) - >>> zarr.save('data/example.zarr', arr) - >>> zarr.load('data/example.zarr') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - - Save an array to a Zip file (uses a :class:`ZipStore`):: - - >>> zarr.save('data/example.zip', arr) - >>> zarr.load('data/example.zip') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - - Save several arrays to a directory on the file system (uses a - :class:`DirectoryStore` and stores arrays in a group):: - - >>> import zarr - >>> import numpy as np - >>> a1 = np.arange(10000) - >>> a2 = np.arange(10000, 0, -1) - >>> zarr.save('data/example.zarr', a1, a2) - >>> loader = zarr.load('data/example.zarr') - >>> loader - - >>> loader['arr_0'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['arr_1'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - Save several arrays using named keyword arguments:: - - >>> zarr.save('data/example.zarr', foo=a1, bar=a2) - >>> loader = zarr.load('data/example.zarr') - >>> loader - - >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - Store several arrays in a single zip file (uses a :class:`ZipStore`):: - - >>> zarr.save('data/example.zip', foo=a1, bar=a2) - >>> loader = zarr.load('data/example.zip') - >>> loader - - >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - See Also - -------- - save_array, save_group - - """ - if len(args) == 0 and len(kwargs) == 0: - raise ValueError('at least one array must be provided') - if len(args) == 1 and len(kwargs) == 0: - save_array(store, args[0]) - else: - save_group(store, *args, **kwargs) - - -class LazyLoader(Mapping): - - def __init__(self, grp): - self.grp = grp - self.cache = dict() - - def __getitem__(self, item): - try: - return self.cache[item] - except KeyError: - arr = self.grp[item][...] - self.cache[item] = arr - return arr - - def __len__(self): - return len(self.grp) - - def __iter__(self): - return iter(self.grp) - - def __contains__(self, item): - return item in self.grp - - def __repr__(self): - r = '>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g3.create_group('baz') - >>> g5 = g3.create_group('qux') - >>> d1 = g5.create_dataset('baz', shape=100, chunks=10) - >>> g1.tree() - / - ├── bar - │ ├── baz - │ └── qux - │ └── baz (100,) float64 - └── foo - >>> import h5py - >>> h5f = h5py.File('data/example.h5', mode='w') - >>> zarr.copy_all(g1, h5f) - (5, 0, 800) - >>> zarr.tree(h5f) - / - ├── bar - │ ├── baz - │ └── qux - │ └── baz (100,) float64 - └── foo - - See Also - -------- - zarr.hierarchy.Group.tree - - Notes - ----- - Please note that this is an experimental feature. The behaviour of this - function is still evolving and the default output and/or parameters may change - in future versions. - - """ - - return TreeViewer(grp, expand=expand, level=level) - - -class _LogWriter(object): - - def __init__(self, log): - self.log_func = None - self.log_file = None - self.needs_closing = False - if log is None: - # don't do any logging - pass - elif callable(log): - self.log_func = log - elif isinstance(log, str): - self.log_file = io.open(log, mode='w') - self.needs_closing = True - else: - if not hasattr(log, 'write'): - raise TypeError('log must be a callable function, file path or ' - 'file-like object, found %r' % log) - self.log_file = log - self.needs_closing = False - - def __enter__(self): - return self - - def __exit__(self, *args): - if self.log_file is not None and self.needs_closing: - self.log_file.close() - - def __call__(self, *args, **kwargs): - if self.log_file is not None: - kwargs['file'] = self.log_file - print(*args, **kwargs) - if hasattr(self.log_file, 'flush'): - # get immediate feedback - self.log_file.flush() - elif self.log_func is not None: - self.log_func(*args, **kwargs) - - -def _log_copy_summary(log, dry_run, n_copied, n_skipped, n_bytes_copied): - # log a final message with a summary of what happened - if dry_run: - message = 'dry run: ' - else: - message = 'all done: ' - message += '{:,} copied, {:,} skipped'.format(n_copied, n_skipped) - if not dry_run: - message += ', {:,} bytes copied'.format(n_bytes_copied) - log(message) - - -def copy_store(source, dest, source_path='', dest_path='', excludes=None, - includes=None, flags=0, if_exists='raise', dry_run=False, - log=None): - """Copy data directly from the `source` store to the `dest` store. Use this - function when you want to copy a group or array in the most efficient way, - preserving all configuration and attributes. This function is more efficient - than the copy() or copy_all() functions because it avoids de-compressing and - re-compressing data, rather the compressed chunk data for each array are - copied directly between stores. - - Parameters - ---------- - source : Mapping - Store to copy data from. - dest : MutableMapping - Store to copy data into. - source_path : str, optional - Only copy data from under this path in the source store. - dest_path : str, optional - Copy data into this path in the destination store. - excludes : sequence of str, optional - One or more regular expressions which will be matched against keys in - the source store. Any matching key will not be copied. - includes : sequence of str, optional - One or more regular expressions which will be matched against keys in - the source store and will override any excludes also matching. - flags : int, optional - Regular expression flags used for matching excludes and includes. - if_exists : {'raise', 'replace', 'skip'}, optional - How to handle keys that already exist in the destination store. If - 'raise' then a CopyError is raised on the first key already present - in the destination store. If 'replace' then any data will be replaced in - the destination. If 'skip' then any existing keys will not be copied. - dry_run : bool, optional - If True, don't actually copy anything, just log what would have - happened. - log : callable, file path or file-like object, optional - If provided, will be used to log progress information. - - Returns - ------- - n_copied : int - Number of items copied. - n_skipped : int - Number of items skipped. - n_bytes_copied : int - Number of bytes of data that were actually copied. - - Examples - -------- - - >>> import zarr - >>> store1 = zarr.DirectoryStore('data/example.zarr') - >>> root = zarr.group(store1, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.create_group('bar') - >>> baz = bar.create_dataset('baz', shape=100, chunks=50, dtype='i8') - >>> import numpy as np - >>> baz[:] = np.arange(100) - >>> root.tree() - / - └── foo - └── bar - └── baz (100,) int64 - >>> from sys import stdout - >>> store2 = zarr.ZipStore('data/example.zip', mode='w') - >>> zarr.copy_store(store1, store2, log=stdout) - copy .zgroup - copy foo/.zgroup - copy foo/bar/.zgroup - copy foo/bar/baz/.zarray - copy foo/bar/baz/0 - copy foo/bar/baz/1 - all done: 6 copied, 0 skipped, 566 bytes copied - (6, 0, 566) - >>> new_root = zarr.group(store2) - >>> new_root.tree() - / - └── foo - └── bar - └── baz (100,) int64 - >>> new_root['foo/bar/baz'][:] - array([ 0, 1, 2, ..., 97, 98, 99]) - >>> store2.close() # zip stores need to be closed - - Notes - ----- - Please note that this is an experimental feature. The behaviour of this - function is still evolving and the default behaviour and/or parameters may change - in future versions. - - """ - - # normalize paths - source_path = normalize_storage_path(source_path) - dest_path = normalize_storage_path(dest_path) - if source_path: - source_path = source_path + '/' - if dest_path: - dest_path = dest_path + '/' - - # normalize excludes and includes - if excludes is None: - excludes = [] - elif isinstance(excludes, str): - excludes = [excludes] - if includes is None: - includes = [] - elif isinstance(includes, str): - includes = [includes] - excludes = [re.compile(e, flags) for e in excludes] - includes = [re.compile(i, flags) for i in includes] - - # check if_exists parameter - valid_if_exists = ['raise', 'replace', 'skip'] - if if_exists not in valid_if_exists: - raise ValueError('if_exists must be one of {!r}; found {!r}' - .format(valid_if_exists, if_exists)) - - # setup counting variables - n_copied = n_skipped = n_bytes_copied = 0 - - # setup logging - with _LogWriter(log) as log: - - # iterate over source keys - for source_key in sorted(source.keys()): - - # filter to keys under source path - if source_key.startswith(source_path): - - # process excludes and includes - exclude = False - for prog in excludes: - if prog.search(source_key): - exclude = True - break - if exclude: - for prog in includes: - if prog.search(source_key): - exclude = False - break - if exclude: - continue - - # map key to destination path - key_suffix = source_key[len(source_path):] - dest_key = dest_path + key_suffix - - # create a descriptive label for this operation - descr = source_key - if dest_key != source_key: - descr = descr + ' -> ' + dest_key - - # decide what to do - do_copy = True - if if_exists != 'replace': - if dest_key in dest: - if if_exists == 'raise': - raise CopyError('key {!r} exists in destination' - .format(dest_key)) - elif if_exists == 'skip': - do_copy = False - - # take action - if do_copy: - log('copy {}'.format(descr)) - if not dry_run: - data = source[source_key] - n_bytes_copied += buffer_size(data) - dest[dest_key] = data - n_copied += 1 - else: - log('skip {}'.format(descr)) - n_skipped += 1 - - # log a final message with a summary of what happened - _log_copy_summary(log, dry_run, n_copied, n_skipped, n_bytes_copied) - - return n_copied, n_skipped, n_bytes_copied - - -def _check_dest_is_group(dest): - if not hasattr(dest, 'create_dataset'): - raise ValueError('dest must be a group, got {!r}'.format(dest)) - - -def copy(source, dest, name=None, shallow=False, without_attrs=False, log=None, - if_exists='raise', dry_run=False, **create_kws): - """Copy the `source` array or group into the `dest` group. - - Parameters - ---------- - source : group or array/dataset - A zarr group or array, or an h5py group or dataset. - dest : group - A zarr or h5py group. - name : str, optional - Name to copy the object to. - shallow : bool, optional - If True, only copy immediate children of `source`. - without_attrs : bool, optional - Do not copy user attributes. - log : callable, file path or file-like object, optional - If provided, will be used to log progress information. - if_exists : {'raise', 'replace', 'skip', 'skip_initialized'}, optional - How to handle arrays that already exist in the destination group. If - 'raise' then a CopyError is raised on the first array already present - in the destination group. If 'replace' then any array will be - replaced in the destination. If 'skip' then any existing arrays will - not be copied. If 'skip_initialized' then any existing arrays with - all chunks initialized will not be copied (not available when copying to - h5py). - dry_run : bool, optional - If True, don't actually copy anything, just log what would have - happened. - **create_kws - Passed through to the create_dataset method when copying an array/dataset. - - Returns - ------- - n_copied : int - Number of items copied. - n_skipped : int - Number of items skipped. - n_bytes_copied : int - Number of bytes of data that were actually copied. - - Examples - -------- - Here's an example of copying a group named 'foo' from an HDF5 file to a - Zarr group:: - - >>> import h5py - >>> import zarr - >>> import numpy as np - >>> source = h5py.File('data/example.h5', mode='w') - >>> foo = source.create_group('foo') - >>> baz = foo.create_dataset('bar/baz', data=np.arange(100), chunks=(50,)) - >>> spam = source.create_dataset('spam', data=np.arange(100, 200), chunks=(30,)) - >>> zarr.tree(source) - / - ├── foo - │ └── bar - │ └── baz (100,) int64 - └── spam (100,) int64 - >>> dest = zarr.group() - >>> from sys import stdout - >>> zarr.copy(source['foo'], dest, log=stdout) - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - all done: 3 copied, 0 skipped, 800 bytes copied - (3, 0, 800) - >>> dest.tree() # N.B., no spam - / - └── foo - └── bar - └── baz (100,) int64 - >>> source.close() - - The ``if_exists`` parameter provides options for how to handle pre-existing data in - the destination. Here are some examples of these options, also using - ``dry_run=True`` to find out what would happen without actually copying anything:: - - >>> source = zarr.group() - >>> dest = zarr.group() - >>> baz = source.create_dataset('foo/bar/baz', data=np.arange(100)) - >>> spam = source.create_dataset('foo/spam', data=np.arange(1000)) - >>> existing_spam = dest.create_dataset('foo/spam', data=np.arange(1000)) - >>> from sys import stdout - >>> try: - ... zarr.copy(source['foo'], dest, log=stdout, dry_run=True) - ... except zarr.CopyError as e: - ... print(e) - ... - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - an object 'spam' already exists in destination '/foo' - >>> zarr.copy(source['foo'], dest, log=stdout, if_exists='replace', dry_run=True) - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - copy /foo/spam (1000,) int64 - dry run: 4 copied, 0 skipped - (4, 0, 0) - >>> zarr.copy(source['foo'], dest, log=stdout, if_exists='skip', dry_run=True) - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - skip /foo/spam (1000,) int64 - dry run: 3 copied, 1 skipped - (3, 1, 0) - - Notes - ----- - Please note that this is an experimental feature. The behaviour of this - function is still evolving and the default behaviour and/or parameters may change - in future versions. - - """ - - # value checks - _check_dest_is_group(dest) - - # setup logging - with _LogWriter(log) as log: - - # do the copying - n_copied, n_skipped, n_bytes_copied = _copy( - log, source, dest, name=name, root=True, shallow=shallow, - without_attrs=without_attrs, if_exists=if_exists, dry_run=dry_run, - **create_kws - ) - - # log a final message with a summary of what happened - _log_copy_summary(log, dry_run, n_copied, n_skipped, n_bytes_copied) - - return n_copied, n_skipped, n_bytes_copied - - -def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, - dry_run, **create_kws): - # N.B., if this is a dry run, dest may be None - - # setup counting variables - n_copied = n_skipped = n_bytes_copied = 0 - - # are we copying to/from h5py? - source_h5py = source.__module__.startswith('h5py.') - dest_h5py = dest is not None and dest.__module__.startswith('h5py.') - - # check if_exists parameter - valid_if_exists = ['raise', 'replace', 'skip', 'skip_initialized'] - if if_exists not in valid_if_exists: - raise ValueError('if_exists must be one of {!r}; found {!r}' - .format(valid_if_exists, if_exists)) - if dest_h5py and if_exists == 'skip_initialized': - raise ValueError('{!r} can only be used when copying to zarr' - .format(if_exists)) - - # determine name to copy to - if name is None: - name = source.name.split('/')[-1] - if not name: - # this can happen if source is the root group - raise TypeError('source has no name, please provide the `name` ' - 'parameter to indicate a name to copy to') - - if hasattr(source, 'shape'): - # copy a dataset/array - - # check if already exists, decide what to do - do_copy = True - exists = dest is not None and name in dest - if exists: - if if_exists == 'raise': - raise CopyError('an object {!r} already exists in destination ' - '{!r}'.format(name, dest.name)) - elif if_exists == 'skip': - do_copy = False - elif if_exists == 'skip_initialized': - ds = dest[name] - if ds.nchunks_initialized == ds.nchunks: - do_copy = False - - # take action - if do_copy: - - # log a message about what we're going to do - log('copy {} {} {}'.format(source.name, source.shape, source.dtype)) - - if not dry_run: - - # clear the way - if exists: - del dest[name] - - # setup creation keyword arguments - kws = create_kws.copy() - - # setup chunks option, preserve by default - kws.setdefault('chunks', source.chunks) - - # setup compression options - if source_h5py: - if dest_h5py: - # h5py -> h5py; preserve compression options by default - kws.setdefault('compression', source.compression) - kws.setdefault('compression_opts', source.compression_opts) - kws.setdefault('shuffle', source.shuffle) - kws.setdefault('fletcher32', source.fletcher32) - kws.setdefault('fillvalue', source.fillvalue) - else: - # h5py -> zarr; use zarr default compression options - kws.setdefault('fill_value', source.fillvalue) - else: - if dest_h5py: - # zarr -> h5py; use some vaguely sensible defaults - kws.setdefault('chunks', True) - kws.setdefault('compression', 'gzip') - kws.setdefault('compression_opts', 1) - kws.setdefault('shuffle', False) - kws.setdefault('fillvalue', source.fill_value) - else: - # zarr -> zarr; preserve compression options by default - kws.setdefault('compressor', source.compressor) - kws.setdefault('filters', source.filters) - kws.setdefault('order', source.order) - kws.setdefault('fill_value', source.fill_value) - - # create new dataset in destination - ds = dest.create_dataset(name, shape=source.shape, - dtype=source.dtype, **kws) - - # copy data - N.B., go chunk by chunk to avoid loading - # everything into memory - shape = ds.shape - chunks = ds.chunks - chunk_offsets = [range(0, s, c) for s, c in zip(shape, chunks)] - for offset in itertools.product(*chunk_offsets): - sel = tuple(slice(o, min(s, o + c)) - for o, s, c in zip(offset, shape, chunks)) - ds[sel] = source[sel] - n_bytes_copied += ds.size * ds.dtype.itemsize - - # copy attributes - if not without_attrs: - ds.attrs.update(source.attrs) - - n_copied += 1 - - else: - log('skip {} {} {}'.format(source.name, source.shape, source.dtype)) - n_skipped += 1 - - elif root or not shallow: - # copy a group - - # check if an array is in the way - do_copy = True - exists_array = (dest is not None and - name in dest and - hasattr(dest[name], 'shape')) - if exists_array: - if if_exists == 'raise': - raise CopyError('an array {!r} already exists in destination ' - '{!r}'.format(name, dest.name)) - elif if_exists == 'skip': - do_copy = False - - # take action - if do_copy: - - # log action - log('copy {}'.format(source.name)) - - if not dry_run: - - # clear the way - if exists_array: - del dest[name] - - # require group in destination - grp = dest.require_group(name) - - # copy attributes - if not without_attrs: - grp.attrs.update(source.attrs) - - else: - - # setup for dry run without creating any groups in the - # destination - if dest is not None: - grp = dest.get(name, None) - else: - grp = None - - # recurse - for k in source.keys(): - c, s, b = _copy( - log, source[k], grp, name=k, root=False, shallow=shallow, - without_attrs=without_attrs, if_exists=if_exists, - dry_run=dry_run, **create_kws) - n_copied += c - n_skipped += s - n_bytes_copied += b - - n_copied += 1 - - else: - log('skip {}'.format(source.name)) - n_skipped += 1 - - return n_copied, n_skipped, n_bytes_copied - - -def copy_all(source, dest, shallow=False, without_attrs=False, log=None, - if_exists='raise', dry_run=False, **create_kws): - """Copy all children of the `source` group into the `dest` group. - - Parameters - ---------- - source : group or array/dataset - A zarr group or array, or an h5py group or dataset. - dest : group - A zarr or h5py group. - shallow : bool, optional - If True, only copy immediate children of `source`. - without_attrs : bool, optional - Do not copy user attributes. - log : callable, file path or file-like object, optional - If provided, will be used to log progress information. - if_exists : {'raise', 'replace', 'skip', 'skip_initialized'}, optional - How to handle arrays that already exist in the destination group. If - 'raise' then a CopyError is raised on the first array already present - in the destination group. If 'replace' then any array will be - replaced in the destination. If 'skip' then any existing arrays will - not be copied. If 'skip_initialized' then any existing arrays with - all chunks initialized will not be copied (not available when copying to - h5py). - dry_run : bool, optional - If True, don't actually copy anything, just log what would have - happened. - **create_kws - Passed through to the create_dataset method when copying an - array/dataset. - - Returns - ------- - n_copied : int - Number of items copied. - n_skipped : int - Number of items skipped. - n_bytes_copied : int - Number of bytes of data that were actually copied. - - Examples - -------- - >>> import h5py - >>> import zarr - >>> import numpy as np - >>> source = h5py.File('data/example.h5', mode='w') - >>> foo = source.create_group('foo') - >>> baz = foo.create_dataset('bar/baz', data=np.arange(100), chunks=(50,)) - >>> spam = source.create_dataset('spam', data=np.arange(100, 200), chunks=(30,)) - >>> zarr.tree(source) - / - ├── foo - │ └── bar - │ └── baz (100,) int64 - └── spam (100,) int64 - >>> dest = zarr.group() - >>> import sys - >>> zarr.copy_all(source, dest, log=sys.stdout) - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - copy /spam (100,) int64 - all done: 4 copied, 0 skipped, 1,600 bytes copied - (4, 0, 1600) - >>> dest.tree() - / - ├── foo - │ └── bar - │ └── baz (100,) int64 - └── spam (100,) int64 - >>> source.close() - - Notes - ----- - Please note that this is an experimental feature. The behaviour of this - function is still evolving and the default behaviour and/or parameters may change - in future versions. - - """ - - # value checks - _check_dest_is_group(dest) - - # setup counting variables - n_copied = n_skipped = n_bytes_copied = 0 - - # setup logging - with _LogWriter(log) as log: - - for k in source.keys(): - c, s, b = _copy( - log, source[k], dest, name=k, root=False, shallow=shallow, - without_attrs=without_attrs, if_exists=if_exists, - dry_run=dry_run, **create_kws) - n_copied += c - n_skipped += s - n_bytes_copied += b - dest.attrs.update(**source.attrs) - - # log a final message with a summary of what happened - _log_copy_summary(log, dry_run, n_copied, n_skipped, n_bytes_copied) - - return n_copied, n_skipped, n_bytes_copied - - -def consolidate_metadata(store, metadata_key='.zmetadata'): - """ - Consolidate all metadata for groups and arrays within the given store - into a single resource and put it under the given key. - - This produces a single object in the backend store, containing all the - metadata read from all the zarr-related keys that can be found. After - metadata have been consolidated, use :func:`open_consolidated` to open - the root group in optimised, read-only mode, using the consolidated - metadata to reduce the number of read operations on the backend store. - - Note, that if the metadata in the store is changed after this - consolidation, then the metadata read by :func:`open_consolidated` - would be incorrect unless this function is called again. - - .. note:: This is an experimental feature. - - Parameters - ---------- - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - metadata_key : str - Key to put the consolidated metadata under. - - Returns - ------- - g : :class:`zarr.hierarchy.Group` - Group instance, opened with the new consolidated metadata. - - See Also - -------- - open_consolidated - - """ - store = normalize_store_arg(store) - - def is_zarr_key(key): - return (key.endswith('.zarray') or key.endswith('.zgroup') or - key.endswith('.zattrs')) - - out = { - 'zarr_consolidated_format': 1, - 'metadata': { - key: json_loads(store[key]) - for key in store if is_zarr_key(key) - } - } - store[metadata_key] = json_dumps(out) - return open_consolidated(store, metadata_key=metadata_key) - - -def open_consolidated(store, metadata_key='.zmetadata', mode='r+', **kwargs): - """Open group using metadata previously consolidated into a single key. - - This is an optimised method for opening a Zarr group, where instead of - traversing the group/array hierarchy by accessing the metadata keys at - each level, a single key contains all of the metadata for everything. - For remote data sources where the overhead of accessing a key is large - compared to the time to read data. - - The group accessed must have already had its metadata consolidated into a - single key using the function :func:`consolidate_metadata`. - - This optimised method only works in modes which do not change the - metadata, although the data may still be written/updated. - - Parameters - ---------- - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - metadata_key : str - Key to read the consolidated metadata from. The default (.zmetadata) - corresponds to the default used by :func:`consolidate_metadata`. - mode : {'r', 'r+'}, optional - Persistence mode: 'r' means read only (must exist); 'r+' means - read/write (must exist) although only writes to data are allowed, - changes to metadata including creation of new arrays or group - are not allowed. - **kwargs - Additional parameters are passed through to :func:`zarr.creation.open_array` or - :func:`zarr.hierarchy.open_group`. - - Returns - ------- - g : :class:`zarr.hierarchy.Group` - Group instance, opened with the consolidated metadata. - - See Also - -------- - consolidate_metadata - - """ - - from .storage import ConsolidatedMetadataStore - - # normalize parameters - store = normalize_store_arg(store, storage_options=kwargs.get("storage_options", None)) - if mode not in {'r', 'r+'}: - raise ValueError("invalid mode, expected either 'r' or 'r+'; found {!r}" - .format(mode)) - - # setup metadata store - meta_store = ConsolidatedMetadataStore(store, metadata_key=metadata_key) - - # pass through - chunk_store = kwargs.pop('chunk_store', None) or store - return open(store=meta_store, chunk_store=chunk_store, mode=mode, **kwargs) diff --git a/zarr/core.py b/zarr/core.py deleted file mode 100644 index ba3f2c1e2d..0000000000 --- a/zarr/core.py +++ /dev/null @@ -1,2534 +0,0 @@ -import binascii -import hashlib -import itertools -import math -import operator -import re -from functools import reduce - -import numpy as np -from numcodecs.compat import ensure_bytes, ensure_ndarray - -from zarr.attrs import Attributes -from zarr.codecs import AsType, get_codec -from zarr.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError -from zarr.indexing import ( - BasicIndexer, - CoordinateIndexer, - MaskIndexer, - OIndex, - OrthogonalIndexer, - VIndex, - PartialChunkIterator, - check_fields, - check_no_multi_fields, - ensure_tuple, - err_too_many_indices, - is_contiguous_selection, - is_scalar, - pop_fields, -) -from zarr.meta import decode_array_metadata, encode_array_metadata -from zarr.storage import array_meta_key, attrs_key, getsize, listdir -from zarr.util import ( - InfoReporter, - check_array_shape, - human_readable_size, - is_total_slice, - nolock, - normalize_chunks, - normalize_resize_args, - normalize_shape, - normalize_storage_path, - PartialReadBuffer, -) - - -# noinspection PyUnresolvedReferences -class Array: - """Instantiate an array from an initialized store. - - Parameters - ---------- - store : MutableMapping - Array store, already initialized. - path : string, optional - Storage path. - read_only : bool, optional - True if array should be protected against modification. - chunk_store : MutableMapping, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - synchronizer : object, optional - Array synchronizer. - cache_metadata : bool, optional - If True (default), array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - partial_decompress : bool, optional - If True and while the chunk_store is a FSStore and the compresion used - is Blosc, when getting data from the array chunks will be partially - read and decompressed when possible. - - .. versionadded:: 2.7 - - Attributes - ---------- - store - path - name - read_only - chunk_store - shape - chunks - dtype - compression - compression_opts - dimension_separator - fill_value - order - synchronizer - filters - attrs - size - itemsize - nbytes - nbytes_stored - cdata_shape - nchunks - nchunks_initialized - is_view - info - vindex - oindex - - Methods - ------- - __getitem__ - __setitem__ - get_basic_selection - set_basic_selection - get_orthogonal_selection - set_orthogonal_selection - get_mask_selection - set_mask_selection - get_coordinate_selection - set_coordinate_selection - digest - hexdigest - resize - append - view - astype - - """ - - def __init__( - self, - store, - path=None, - read_only=False, - chunk_store=None, - synchronizer=None, - cache_metadata=True, - cache_attrs=True, - partial_decompress=False, - ): - # N.B., expect at this point store is fully initialized with all - # configuration metadata fully specified and normalized - - self._store = store - self._chunk_store = chunk_store - self._path = normalize_storage_path(path) - if self._path: - self._key_prefix = self._path + '/' - else: - self._key_prefix = '' - self._read_only = bool(read_only) - self._synchronizer = synchronizer - self._cache_metadata = cache_metadata - self._is_view = False - self._partial_decompress = partial_decompress - - # initialize metadata - self._load_metadata() - - # initialize attributes - akey = self._key_prefix + attrs_key - self._attrs = Attributes(store, key=akey, read_only=read_only, - synchronizer=synchronizer, cache=cache_attrs) - - # initialize info reporter - self._info_reporter = InfoReporter(self) - - # initialize indexing helpers - self._oindex = OIndex(self) - self._vindex = VIndex(self) - - def _load_metadata(self): - """(Re)load metadata from store.""" - if self._synchronizer is None: - self._load_metadata_nosync() - else: - mkey = self._key_prefix + array_meta_key - with self._synchronizer[mkey]: - self._load_metadata_nosync() - - def _load_metadata_nosync(self): - try: - mkey = self._key_prefix + array_meta_key - meta_bytes = self._store[mkey] - except KeyError: - raise ArrayNotFoundError(self._path) - else: - - # decode and store metadata as instance members - meta = decode_array_metadata(meta_bytes) - self._meta = meta - self._shape = meta['shape'] - self._chunks = meta['chunks'] - self._dtype = meta['dtype'] - self._fill_value = meta['fill_value'] - self._order = meta['order'] - self._dimension_separator = meta.get('dimension_separator', '.') - - # setup compressor - config = meta['compressor'] - if config is None: - self._compressor = None - else: - self._compressor = get_codec(config) - - # setup filters - filters = meta['filters'] - if filters: - filters = [get_codec(config) for config in filters] - self._filters = filters - - def _refresh_metadata(self): - if not self._cache_metadata: - self._load_metadata() - - def _refresh_metadata_nosync(self): - if not self._cache_metadata and not self._is_view: - self._load_metadata_nosync() - - def _flush_metadata_nosync(self): - if self._is_view: - raise PermissionError('operation not permitted for views') - - if self._compressor: - compressor_config = self._compressor.get_config() - else: - compressor_config = None - if self._filters: - filters_config = [f.get_config() for f in self._filters] - else: - filters_config = None - meta = dict(shape=self._shape, chunks=self._chunks, dtype=self._dtype, - compressor=compressor_config, fill_value=self._fill_value, - order=self._order, filters=filters_config) - mkey = self._key_prefix + array_meta_key - self._store[mkey] = encode_array_metadata(meta) - - @property - def store(self): - """A MutableMapping providing the underlying storage for the array.""" - return self._store - - @property - def path(self): - """Storage path.""" - return self._path - - @property - def name(self): - """Array name following h5py convention.""" - if self.path: - # follow h5py convention: add leading slash - name = self.path - if name[0] != '/': - name = '/' + name - return name - return None - - @property - def basename(self): - """Final component of name.""" - if self.name is not None: - return self.name.split('/')[-1] - return None - - @property - def read_only(self): - """A boolean, True if modification operations are not permitted.""" - return self._read_only - - @read_only.setter - def read_only(self, value): - self._read_only = bool(value) - - @property - def chunk_store(self): - """A MutableMapping providing the underlying storage for array chunks.""" - if self._chunk_store is None: - return self._store - else: - return self._chunk_store - - @property - def shape(self): - """A tuple of integers describing the length of each dimension of - the array.""" - # N.B., shape may change if array is resized, hence need to refresh - # metadata - self._refresh_metadata() - return self._shape - - @shape.setter - def shape(self, value): - self.resize(value) - - @property - def chunks(self): - """A tuple of integers describing the length of each dimension of a - chunk of the array.""" - return self._chunks - - @property - def dtype(self): - """The NumPy data type.""" - return self._dtype - - @property - def compressor(self): - """Primary compression codec.""" - return self._compressor - - @property - def fill_value(self): - """A value used for uninitialized portions of the array.""" - return self._fill_value - - @property - def order(self): - """A string indicating the order in which bytes are arranged within - chunks of the array.""" - return self._order - - @property - def filters(self): - """One or more codecs used to transform data prior to compression.""" - return self._filters - - @property - def synchronizer(self): - """Object used to synchronize write access to the array.""" - return self._synchronizer - - @property - def attrs(self): - """A MutableMapping containing user-defined attributes. Note that - attribute values must be JSON serializable.""" - return self._attrs - - @property - def ndim(self): - """Number of dimensions.""" - return len(self.shape) - - @property - def _size(self): - return reduce(operator.mul, self._shape, 1) - - @property - def size(self): - """The total number of elements in the array.""" - # N.B., this property depends on shape, and shape may change if array - # is resized, hence need to refresh metadata - self._refresh_metadata() - return self._size - - @property - def itemsize(self): - """The size in bytes of each item in the array.""" - return self.dtype.itemsize - - @property - def _nbytes(self): - return self._size * self.itemsize - - @property - def nbytes(self): - """The total number of bytes that would be required to store the - array without compression.""" - # N.B., this property depends on shape, and shape may change if array - # is resized, hence need to refresh metadata - self._refresh_metadata() - return self._nbytes - - @property - def nbytes_stored(self): - """The total number of stored bytes of data for the array. This - includes storage required for configuration metadata and user - attributes.""" - m = getsize(self._store, self._path) - if self._chunk_store is None: - return m - else: - n = getsize(self._chunk_store, self._path) - if m < 0 or n < 0: - return -1 - else: - return m + n - - @property - def _cdata_shape(self): - if self._shape == (): - return 1, - else: - return tuple(math.ceil(s / c) - for s, c in zip(self._shape, self._chunks)) - - @property - def cdata_shape(self): - """A tuple of integers describing the number of chunks along each - dimension of the array.""" - self._refresh_metadata() - return self._cdata_shape - - @property - def _nchunks(self): - return reduce(operator.mul, self._cdata_shape, 1) - - @property - def nchunks(self): - """Total number of chunks.""" - self._refresh_metadata() - return self._nchunks - - @property - def nchunks_initialized(self): - """The number of chunks that have been initialized with some data.""" - - # key pattern for chunk keys - prog = re.compile(r'\.'.join([r'\d+'] * min(1, self.ndim))) - - # count chunk keys - return sum(1 for k in listdir(self.chunk_store, self._path) if prog.match(k)) - - # backwards compability - initialized = nchunks_initialized - - @property - def is_view(self): - """A boolean, True if this array is a view on another array.""" - return self._is_view - - @property - def oindex(self): - """Shortcut for orthogonal (outer) indexing, see :func:`get_orthogonal_selection` and - :func:`set_orthogonal_selection` for documentation and examples.""" - return self._oindex - - @property - def vindex(self): - """Shortcut for vectorized (inner) indexing, see :func:`get_coordinate_selection`, - :func:`set_coordinate_selection`, :func:`get_mask_selection` and - :func:`set_mask_selection` for documentation and examples.""" - return self._vindex - - def __eq__(self, other): - return ( - isinstance(other, Array) and - self.store == other.store and - self.read_only == other.read_only and - self.path == other.path and - not self._is_view - # N.B., no need to compare other properties, should be covered by - # store comparison - ) - - def __array__(self, *args): - a = self[...] - if args: - a = a.astype(args[0]) - return a - - def islice(self, start=None, end=None): - """ - Yield a generator for iterating over the entire or parts of the - array. Uses a cache so chunks only have to be decompressed once. - - Parameters - ---------- - start : int, optional - Start index for the generator to start at. Defaults to 0. - end : int, optional - End index for the generator to stop at. Defaults to self.shape[0]. - - Yields - ------ - out : generator - A generator that can be used to iterate over the requested region - the array. - - Examples - -------- - Setup a 1-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.array(np.arange(100)) - - Iterate over part of the array: - >>> for value in z.islice(25, 30): value; - 25 - 26 - 27 - 28 - 29 - """ - - if len(self.shape) == 0: - # Same error as numpy - raise TypeError("iteration over a 0-d array") - if start is None: - start = 0 - if end is None or end > self.shape[0]: - end = self.shape[0] - - if not isinstance(start, int) or start < 0: - raise ValueError('start must be a nonnegative integer') - - if not isinstance(end, int) or end < 0: - raise ValueError('end must be a nonnegative integer') - - # Avoid repeatedly decompressing chunks by iterating over the chunks - # in the first dimension. - chunk_size = self.chunks[0] - chunk = None - for j in range(start, end): - if j % chunk_size == 0: - chunk = self[j: j + chunk_size] - # init chunk if we start offset of chunk borders - elif chunk is None: - chunk_start = j - j % chunk_size - chunk_end = chunk_start + chunk_size - chunk = self[chunk_start:chunk_end] - yield chunk[j % chunk_size] - - def __iter__(self): - return self.islice() - - def __len__(self): - if self.shape: - return self.shape[0] - else: - # 0-dimensional array, same error message as numpy - raise TypeError('len() of unsized object') - - def __getitem__(self, selection): - """Retrieve data for an item or region of the array. - - Parameters - ---------- - selection : tuple - An integer index or slice or tuple of int/slice objects specifying the - requested item or region for each dimension of the array. - - Returns - ------- - out : ndarray - A NumPy array containing the data for the requested region. - - Examples - -------- - Setup a 1-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.array(np.arange(100)) - - Retrieve a single item:: - - >>> z[5] - 5 - - Retrieve a region via slicing:: - - >>> z[:5] - array([0, 1, 2, 3, 4]) - >>> z[-5:] - array([95, 96, 97, 98, 99]) - >>> z[5:10] - array([5, 6, 7, 8, 9]) - >>> z[5:10:2] - array([5, 7, 9]) - >>> z[::2] - array([ 0, 2, 4, ..., 94, 96, 98]) - - Load the entire array into memory:: - - >>> z[...] - array([ 0, 1, 2, ..., 97, 98, 99]) - - Setup a 2-dimensional array:: - - >>> z = zarr.array(np.arange(100).reshape(10, 10)) - - Retrieve an item:: - - >>> z[2, 2] - 22 - - Retrieve a region via slicing:: - - >>> z[1:3, 1:3] - array([[11, 12], - [21, 22]]) - >>> z[1:3, :] - array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]]) - >>> z[:, 1:3] - array([[ 1, 2], - [11, 12], - [21, 22], - [31, 32], - [41, 42], - [51, 52], - [61, 62], - [71, 72], - [81, 82], - [91, 92]]) - >>> z[0:5:2, 0:5:2] - array([[ 0, 2, 4], - [20, 22, 24], - [40, 42, 44]]) - >>> z[::2, ::2] - array([[ 0, 2, 4, 6, 8], - [20, 22, 24, 26, 28], - [40, 42, 44, 46, 48], - [60, 62, 64, 66, 68], - [80, 82, 84, 86, 88]]) - - Load the entire array into memory:: - - >>> z[...] - array([[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], - [30, 31, 32, 33, 34, 35, 36, 37, 38, 39], - [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], - [50, 51, 52, 53, 54, 55, 56, 57, 58, 59], - [60, 61, 62, 63, 64, 65, 66, 67, 68, 69], - [70, 71, 72, 73, 74, 75, 76, 77, 78, 79], - [80, 81, 82, 83, 84, 85, 86, 87, 88, 89], - [90, 91, 92, 93, 94, 95, 96, 97, 98, 99]]) - - For arrays with a structured dtype, specific fields can be retrieved, e.g.:: - - >>> a = np.array([(b'aaa', 1, 4.2), - ... (b'bbb', 2, 8.4), - ... (b'ccc', 3, 12.6)], - ... dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - >>> z = zarr.array(a) - >>> z['foo'] - array([b'aaa', b'bbb', b'ccc'], - dtype='|S3') - - Notes - ----- - Slices with step > 1 are supported, but slices with negative step are not. - - Currently the implementation for __getitem__ is provided by - :func:`get_basic_selection`. For advanced ("fancy") indexing, see the methods - listed under See Also. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, - set_orthogonal_selection, vindex, oindex, __setitem__ - - """ - - fields, selection = pop_fields(selection) - return self.get_basic_selection(selection, fields=fields) - - def get_basic_selection(self, selection=Ellipsis, out=None, fields=None): - """Retrieve data for an item or region of the array. - - Parameters - ---------- - selection : tuple - A tuple specifying the requested item or region for each dimension of the - array. May be any combination of int and/or slice for multidimensional arrays. - out : ndarray, optional - If given, load the selected data directly into this array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to - extract data for. - - Returns - ------- - out : ndarray - A NumPy array containing the data for the requested region. - - Examples - -------- - Setup a 1-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.array(np.arange(100)) - - Retrieve a single item:: - - >>> z.get_basic_selection(5) - 5 - - Retrieve a region via slicing:: - - >>> z.get_basic_selection(slice(5)) - array([0, 1, 2, 3, 4]) - >>> z.get_basic_selection(slice(-5, None)) - array([95, 96, 97, 98, 99]) - >>> z.get_basic_selection(slice(5, 10)) - array([5, 6, 7, 8, 9]) - >>> z.get_basic_selection(slice(5, 10, 2)) - array([5, 7, 9]) - >>> z.get_basic_selection(slice(None, None, 2)) - array([ 0, 2, 4, ..., 94, 96, 98]) - - Setup a 2-dimensional array:: - - >>> z = zarr.array(np.arange(100).reshape(10, 10)) - - Retrieve an item:: - - >>> z.get_basic_selection((2, 2)) - 22 - - Retrieve a region via slicing:: - - >>> z.get_basic_selection((slice(1, 3), slice(1, 3))) - array([[11, 12], - [21, 22]]) - >>> z.get_basic_selection((slice(1, 3), slice(None))) - array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]]) - >>> z.get_basic_selection((slice(None), slice(1, 3))) - array([[ 1, 2], - [11, 12], - [21, 22], - [31, 32], - [41, 42], - [51, 52], - [61, 62], - [71, 72], - [81, 82], - [91, 92]]) - >>> z.get_basic_selection((slice(0, 5, 2), slice(0, 5, 2))) - array([[ 0, 2, 4], - [20, 22, 24], - [40, 42, 44]]) - >>> z.get_basic_selection((slice(None, None, 2), slice(None, None, 2))) - array([[ 0, 2, 4, 6, 8], - [20, 22, 24, 26, 28], - [40, 42, 44, 46, 48], - [60, 62, 64, 66, 68], - [80, 82, 84, 86, 88]]) - - For arrays with a structured dtype, specific fields can be retrieved, e.g.:: - - >>> a = np.array([(b'aaa', 1, 4.2), - ... (b'bbb', 2, 8.4), - ... (b'ccc', 3, 12.6)], - ... dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - >>> z = zarr.array(a) - >>> z.get_basic_selection(slice(2), fields='foo') - array([b'aaa', b'bbb'], - dtype='|S3') - - Notes - ----- - Slices with step > 1 are supported, but slices with negative step are not. - - Currently this method provides the implementation for accessing data via the - square bracket notation (__getitem__). See :func:`__getitem__` for examples - using the alternative notation. - - See Also - -------- - set_basic_selection, get_mask_selection, set_mask_selection, - get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, - set_orthogonal_selection, vindex, oindex, __getitem__, __setitem__ - - """ - - # refresh metadata - if not self._cache_metadata: - self._load_metadata() - - # check args - check_fields(fields, self._dtype) - - # handle zero-dimensional arrays - if self._shape == (): - return self._get_basic_selection_zd(selection=selection, out=out, - fields=fields) - else: - return self._get_basic_selection_nd(selection=selection, out=out, - fields=fields) - - def _get_basic_selection_zd(self, selection, out=None, fields=None): - # special case basic selection for zero-dimensional array - - # check selection is valid - selection = ensure_tuple(selection) - if selection not in ((), (Ellipsis,)): - err_too_many_indices(selection, ()) - - try: - # obtain encoded data for chunk - ckey = self._chunk_key((0,)) - cdata = self.chunk_store[ckey] - - except KeyError: - # chunk not initialized - chunk = np.zeros((), dtype=self._dtype) - if self._fill_value is not None: - chunk.fill(self._fill_value) - - else: - chunk = self._decode_chunk(cdata) - - # handle fields - if fields: - chunk = chunk[fields] - - # handle selection of the scalar value via empty tuple - if out is None: - out = chunk[selection] - else: - out[selection] = chunk[selection] - - return out - - def _get_basic_selection_nd(self, selection, out=None, fields=None): - # implementation of basic selection for array with at least one dimension - - # setup indexer - indexer = BasicIndexer(selection, self) - - return self._get_selection(indexer=indexer, out=out, fields=fields) - - def get_orthogonal_selection(self, selection, out=None, fields=None): - """Retrieve data by making a selection for each dimension of the array. For - example, if an array has 2 dimensions, allows selecting specific rows and/or - columns. The selection for each dimension can be either an integer (indexing a - single item), a slice, an array of integers, or a Boolean array where True - values indicate a selection. - - Parameters - ---------- - selection : tuple - A selection for each dimension of the array. May be any combination of int, - slice, integer array or Boolean array. - out : ndarray, optional - If given, load the selected data directly into this array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to - extract data for. - - Returns - ------- - out : ndarray - A NumPy array containing the data for the requested selection. - - Examples - -------- - Setup a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.array(np.arange(100).reshape(10, 10)) - - Retrieve rows and columns via any combination of int, slice, integer array and/or - Boolean array:: - - >>> z.get_orthogonal_selection(([1, 4], slice(None))) - array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - [40, 41, 42, 43, 44, 45, 46, 47, 48, 49]]) - >>> z.get_orthogonal_selection((slice(None), [1, 4])) - array([[ 1, 4], - [11, 14], - [21, 24], - [31, 34], - [41, 44], - [51, 54], - [61, 64], - [71, 74], - [81, 84], - [91, 94]]) - >>> z.get_orthogonal_selection(([1, 4], [1, 4])) - array([[11, 14], - [41, 44]]) - >>> sel = np.zeros(z.shape[0], dtype=bool) - >>> sel[1] = True - >>> sel[4] = True - >>> z.get_orthogonal_selection((sel, sel)) - array([[11, 14], - [41, 44]]) - - For convenience, the orthogonal selection functionality is also available via the - `oindex` property, e.g.:: - - >>> z.oindex[[1, 4], :] - array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - [40, 41, 42, 43, 44, 45, 46, 47, 48, 49]]) - >>> z.oindex[:, [1, 4]] - array([[ 1, 4], - [11, 14], - [21, 24], - [31, 34], - [41, 44], - [51, 54], - [61, 64], - [71, 74], - [81, 84], - [91, 94]]) - >>> z.oindex[[1, 4], [1, 4]] - array([[11, 14], - [41, 44]]) - >>> sel = np.zeros(z.shape[0], dtype=bool) - >>> sel[1] = True - >>> sel[4] = True - >>> z.oindex[sel, sel] - array([[11, 14], - [41, 44]]) - - Notes - ----- - Orthogonal indexing is also known as outer indexing. - - Slices with step > 1 are supported, but slices with negative step are not. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_coordinate_selection, set_coordinate_selection, set_orthogonal_selection, - vindex, oindex, __getitem__, __setitem__ - - """ - - # refresh metadata - if not self._cache_metadata: - self._load_metadata() - - # check args - check_fields(fields, self._dtype) - - # setup indexer - indexer = OrthogonalIndexer(selection, self) - - return self._get_selection(indexer=indexer, out=out, fields=fields) - - def get_coordinate_selection(self, selection, out=None, fields=None): - """Retrieve a selection of individual items, by providing the indices - (coordinates) for each selected item. - - Parameters - ---------- - selection : tuple - An integer (coordinate) array for each dimension of the array. - out : ndarray, optional - If given, load the selected data directly into this array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to - extract data for. - - Returns - ------- - out : ndarray - A NumPy array containing the data for the requested selection. - - Examples - -------- - Setup a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.array(np.arange(100).reshape(10, 10)) - - Retrieve items by specifying their coordinates:: - - >>> z.get_coordinate_selection(([1, 4], [1, 4])) - array([11, 44]) - - For convenience, the coordinate selection functionality is also available via the - `vindex` property, e.g.:: - - >>> z.vindex[[1, 4], [1, 4]] - array([11, 44]) - - Notes - ----- - Coordinate indexing is also known as point selection, and is a form of vectorized - or inner indexing. - - Slices are not supported. Coordinate arrays must be provided for all dimensions - of the array. - - Coordinate arrays may be multidimensional, in which case the output array will - also be multidimensional. Coordinate arrays are broadcast against each other - before being applied. The shape of the output will be the same as the shape of - each coordinate array after broadcasting. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_orthogonal_selection, set_orthogonal_selection, set_coordinate_selection, - vindex, oindex, __getitem__, __setitem__ - - """ - - # refresh metadata - if not self._cache_metadata: - self._load_metadata() - - # check args - check_fields(fields, self._dtype) - - # setup indexer - indexer = CoordinateIndexer(selection, self) - - # handle output - need to flatten - if out is not None: - out = out.reshape(-1) - - out = self._get_selection(indexer=indexer, out=out, fields=fields) - - # restore shape - out = out.reshape(indexer.sel_shape) - - return out - - def get_mask_selection(self, selection, out=None, fields=None): - """Retrieve a selection of individual items, by providing a Boolean array of the - same shape as the array against which the selection is being made, where True - values indicate a selected item. - - Parameters - ---------- - selection : ndarray, bool - A Boolean array of the same shape as the array against which the selection is - being made. - out : ndarray, optional - If given, load the selected data directly into this array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to - extract data for. - - Returns - ------- - out : ndarray - A NumPy array containing the data for the requested selection. - - Examples - -------- - Setup a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.array(np.arange(100).reshape(10, 10)) - - Retrieve items by specifying a maks:: - - >>> sel = np.zeros_like(z, dtype=bool) - >>> sel[1, 1] = True - >>> sel[4, 4] = True - >>> z.get_mask_selection(sel) - array([11, 44]) - - For convenience, the mask selection functionality is also available via the - `vindex` property, e.g.:: - - >>> z.vindex[sel] - array([11, 44]) - - Notes - ----- - Mask indexing is a form of vectorized or inner indexing, and is equivalent to - coordinate indexing. Internally the mask array is converted to coordinate - arrays by calling `np.nonzero`. - - See Also - -------- - get_basic_selection, set_basic_selection, set_mask_selection, - get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, - set_coordinate_selection, vindex, oindex, __getitem__, __setitem__ - - """ - - # refresh metadata - if not self._cache_metadata: - self._load_metadata() - - # check args - check_fields(fields, self._dtype) - - # setup indexer - indexer = MaskIndexer(selection, self) - - return self._get_selection(indexer=indexer, out=out, fields=fields) - - def _get_selection(self, indexer, out=None, fields=None): - - # We iterate over all chunks which overlap the selection and thus contain data - # that needs to be extracted. Each chunk is processed in turn, extracting the - # necessary data and storing into the correct location in the output array. - - # N.B., it is an important optimisation that we only visit chunks which overlap - # the selection. This minimises the number of iterations in the main for loop. - - # check fields are sensible - out_dtype = check_fields(fields, self._dtype) - - # determine output shape - out_shape = indexer.shape - - # setup output array - if out is None: - out = np.empty(out_shape, dtype=out_dtype, order=self._order) - else: - check_array_shape('out', out, out_shape) - - # iterate over chunks - if not hasattr(self.chunk_store, "getitems") or \ - any(map(lambda x: x == 0, self.shape)): - # sequentially get one key at a time from storage - for chunk_coords, chunk_selection, out_selection in indexer: - - # load chunk selection into output array - self._chunk_getitem(chunk_coords, chunk_selection, out, out_selection, - drop_axes=indexer.drop_axes, fields=fields) - else: - # allow storage to get multiple items at once - lchunk_coords, lchunk_selection, lout_selection = zip(*indexer) - self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection, - drop_axes=indexer.drop_axes, fields=fields) - - if out.shape: - return out - else: - return out[()] - - def __setitem__(self, selection, value): - """Modify data for an item or region of the array. - - Parameters - ---------- - selection : tuple - An integer index or slice or tuple of int/slice specifying the requested - region for each dimension of the array. - value : scalar or array-like - Value to be stored into the array. - - Examples - -------- - Setup a 1-dimensional array:: - - >>> import zarr - >>> z = zarr.zeros(100, dtype=int) - - Set all array elements to the same scalar value:: - - >>> z[...] = 42 - >>> z[...] - array([42, 42, 42, ..., 42, 42, 42]) - - Set a portion of the array:: - - >>> z[:10] = np.arange(10) - >>> z[-10:] = np.arange(10)[::-1] - >>> z[...] - array([ 0, 1, 2, ..., 2, 1, 0]) - - Setup a 2-dimensional array:: - - >>> z = zarr.zeros((5, 5), dtype=int) - - Set all array elements to the same scalar value:: - - >>> z[...] = 42 - - Set a portion of the array:: - - >>> z[0, :] = np.arange(z.shape[1]) - >>> z[:, 0] = np.arange(z.shape[0]) - >>> z[...] - array([[ 0, 1, 2, 3, 4], - [ 1, 42, 42, 42, 42], - [ 2, 42, 42, 42, 42], - [ 3, 42, 42, 42, 42], - [ 4, 42, 42, 42, 42]]) - - For arrays with a structured dtype, specific fields can be modified, e.g.:: - - >>> a = np.array([(b'aaa', 1, 4.2), - ... (b'bbb', 2, 8.4), - ... (b'ccc', 3, 12.6)], - ... dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - >>> z = zarr.array(a) - >>> z['foo'] = b'zzz' - >>> z[...] - array([(b'zzz', 1, 4.2), (b'zzz', 2, 8.4), (b'zzz', 3, 12.6)], - dtype=[('foo', 'S3'), ('bar', ' 1 are supported, but slices with negative step are not. - - Currently the implementation for __setitem__ is provided by - :func:`set_basic_selection`, which means that only integers and slices are - supported within the selection. For advanced ("fancy") indexing, see the - methods listed under See Also. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, - set_orthogonal_selection, vindex, oindex, __getitem__ - - """ - - fields, selection = pop_fields(selection) - self.set_basic_selection(selection, value, fields=fields) - - def set_basic_selection(self, selection, value, fields=None): - """Modify data for an item or region of the array. - - Parameters - ---------- - selection : tuple - An integer index or slice or tuple of int/slice specifying the requested - region for each dimension of the array. - value : scalar or array-like - Value to be stored into the array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to set - data for. - - Examples - -------- - Setup a 1-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.zeros(100, dtype=int) - - Set all array elements to the same scalar value:: - - >>> z.set_basic_selection(..., 42) - >>> z[...] - array([42, 42, 42, ..., 42, 42, 42]) - - Set a portion of the array:: - - >>> z.set_basic_selection(slice(10), np.arange(10)) - >>> z.set_basic_selection(slice(-10, None), np.arange(10)[::-1]) - >>> z[...] - array([ 0, 1, 2, ..., 2, 1, 0]) - - Setup a 2-dimensional array:: - - >>> z = zarr.zeros((5, 5), dtype=int) - - Set all array elements to the same scalar value:: - - >>> z.set_basic_selection(..., 42) - - Set a portion of the array:: - - >>> z.set_basic_selection((0, slice(None)), np.arange(z.shape[1])) - >>> z.set_basic_selection((slice(None), 0), np.arange(z.shape[0])) - >>> z[...] - array([[ 0, 1, 2, 3, 4], - [ 1, 42, 42, 42, 42], - [ 2, 42, 42, 42, 42], - [ 3, 42, 42, 42, 42], - [ 4, 42, 42, 42, 42]]) - - For arrays with a structured dtype, the `fields` parameter can be used to set - data for a specific field, e.g.:: - - >>> a = np.array([(b'aaa', 1, 4.2), - ... (b'bbb', 2, 8.4), - ... (b'ccc', 3, 12.6)], - ... dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - >>> z = zarr.array(a) - >>> z.set_basic_selection(slice(0, 2), b'zzz', fields='foo') - >>> z[:] - array([(b'zzz', 1, 4.2), (b'zzz', 2, 8.4), (b'ccc', 3, 12.6)], - dtype=[('foo', 'S3'), ('bar', '>> import zarr - >>> import numpy as np - >>> z = zarr.zeros((5, 5), dtype=int) - - Set data for a selection of rows:: - - >>> z.set_orthogonal_selection(([1, 4], slice(None)), 1) - >>> z[...] - array([[0, 0, 0, 0, 0], - [1, 1, 1, 1, 1], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [1, 1, 1, 1, 1]]) - - Set data for a selection of columns:: - - >>> z.set_orthogonal_selection((slice(None), [1, 4]), 2) - >>> z[...] - array([[0, 2, 0, 0, 2], - [1, 2, 1, 1, 2], - [0, 2, 0, 0, 2], - [0, 2, 0, 0, 2], - [1, 2, 1, 1, 2]]) - - Set data for a selection of rows and columns:: - - >>> z.set_orthogonal_selection(([1, 4], [1, 4]), 3) - >>> z[...] - array([[0, 2, 0, 0, 2], - [1, 3, 1, 1, 3], - [0, 2, 0, 0, 2], - [0, 2, 0, 0, 2], - [1, 3, 1, 1, 3]]) - - For convenience, this functionality is also available via the `oindex` property. - E.g.:: - - >>> z.oindex[[1, 4], [1, 4]] = 4 - >>> z[...] - array([[0, 2, 0, 0, 2], - [1, 4, 1, 1, 4], - [0, 2, 0, 0, 2], - [0, 2, 0, 0, 2], - [1, 4, 1, 1, 4]]) - - Notes - ----- - Orthogonal indexing is also known as outer indexing. - - Slices with step > 1 are supported, but slices with negative step are not. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, - vindex, oindex, __getitem__, __setitem__ - - """ - - # guard conditions - if self._read_only: - raise ReadOnlyError() - - # refresh metadata - if not self._cache_metadata: - self._load_metadata_nosync() - - # setup indexer - indexer = OrthogonalIndexer(selection, self) - - self._set_selection(indexer, value, fields=fields) - - def set_coordinate_selection(self, selection, value, fields=None): - """Modify a selection of individual items, by providing the indices (coordinates) - for each item to be modified. - - Parameters - ---------- - selection : tuple - An integer (coordinate) array for each dimension of the array. - value : scalar or array-like - Value to be stored into the array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to set - data for. - - Examples - -------- - Setup a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.zeros((5, 5), dtype=int) - - Set data for a selection of items:: - - >>> z.set_coordinate_selection(([1, 4], [1, 4]), 1) - >>> z[...] - array([[0, 0, 0, 0, 0], - [0, 1, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 1]]) - - For convenience, this functionality is also available via the `vindex` property. - E.g.:: - - >>> z.vindex[[1, 4], [1, 4]] = 2 - >>> z[...] - array([[0, 0, 0, 0, 0], - [0, 2, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 2]]) - - Notes - ----- - Coordinate indexing is also known as point selection, and is a form of vectorized - or inner indexing. - - Slices are not supported. Coordinate arrays must be provided for all dimensions - of the array. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, - vindex, oindex, __getitem__, __setitem__ - - """ - - # guard conditions - if self._read_only: - raise ReadOnlyError() - - # refresh metadata - if not self._cache_metadata: - self._load_metadata_nosync() - - # setup indexer - indexer = CoordinateIndexer(selection, self) - - # handle value - need to flatten - if not is_scalar(value, self._dtype): - value = np.asanyarray(value) - if hasattr(value, 'shape') and len(value.shape) > 1: - value = value.reshape(-1) - - self._set_selection(indexer, value, fields=fields) - - def set_mask_selection(self, selection, value, fields=None): - """Modify a selection of individual items, by providing a Boolean array of the - same shape as the array against which the selection is being made, where True - values indicate a selected item. - - Parameters - ---------- - selection : ndarray, bool - A Boolean array of the same shape as the array against which the selection is - being made. - value : scalar or array-like - Value to be stored into the array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to set - data for. - - Examples - -------- - Setup a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.zeros((5, 5), dtype=int) - - Set data for a selection of items:: - - >>> sel = np.zeros_like(z, dtype=bool) - >>> sel[1, 1] = True - >>> sel[4, 4] = True - >>> z.set_mask_selection(sel, 1) - >>> z[...] - array([[0, 0, 0, 0, 0], - [0, 1, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 1]]) - - For convenience, this functionality is also available via the `vindex` property. - E.g.:: - - >>> z.vindex[sel] = 2 - >>> z[...] - array([[0, 0, 0, 0, 0], - [0, 2, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 2]]) - - Notes - ----- - Mask indexing is a form of vectorized or inner indexing, and is equivalent to - coordinate indexing. Internally the mask array is converted to coordinate - arrays by calling `np.nonzero`. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, - get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, - set_coordinate_selection, vindex, oindex, __getitem__, __setitem__ - - """ - - # guard conditions - if self._read_only: - raise ReadOnlyError() - - # refresh metadata - if not self._cache_metadata: - self._load_metadata_nosync() - - # setup indexer - indexer = MaskIndexer(selection, self) - - self._set_selection(indexer, value, fields=fields) - - def _set_basic_selection_zd(self, selection, value, fields=None): - # special case __setitem__ for zero-dimensional array - - # check selection is valid - selection = ensure_tuple(selection) - if selection not in ((), (Ellipsis,)): - err_too_many_indices(selection, self._shape) - - # check fields - check_fields(fields, self._dtype) - fields = check_no_multi_fields(fields) - - # obtain key for chunk - ckey = self._chunk_key((0,)) - - # setup chunk - try: - # obtain compressed data for chunk - cdata = self.chunk_store[ckey] - - except KeyError: - # chunk not initialized - chunk = np.zeros((), dtype=self._dtype) - if self._fill_value is not None: - chunk.fill(self._fill_value) - - else: - # decode chunk - chunk = self._decode_chunk(cdata).copy() - - # set value - if fields: - chunk[fields][selection] = value - else: - chunk[selection] = value - - # encode and store - cdata = self._encode_chunk(chunk) - self.chunk_store[ckey] = cdata - - def _set_basic_selection_nd(self, selection, value, fields=None): - # implementation of __setitem__ for array with at least one dimension - - # setup indexer - indexer = BasicIndexer(selection, self) - - self._set_selection(indexer, value, fields=fields) - - def _set_selection(self, indexer, value, fields=None): - - # We iterate over all chunks which overlap the selection and thus contain data - # that needs to be replaced. Each chunk is processed in turn, extracting the - # necessary data from the value array and storing into the chunk array. - - # N.B., it is an important optimisation that we only visit chunks which overlap - # the selection. This minimises the number of iterations in the main for loop. - - # check fields are sensible - check_fields(fields, self._dtype) - fields = check_no_multi_fields(fields) - - # determine indices of chunks overlapping the selection - sel_shape = indexer.shape - - # check value shape - if sel_shape == (): - # setting a single item - pass - elif is_scalar(value, self._dtype): - # setting a scalar value - pass - else: - if not hasattr(value, 'shape'): - value = np.asanyarray(value) - check_array_shape('value', value, sel_shape) - - # iterate over chunks in range - if not hasattr(self.store, "setitems") or self._synchronizer is not None \ - or any(map(lambda x: x == 0, self.shape)): - # iterative approach - for chunk_coords, chunk_selection, out_selection in indexer: - - # extract data to store - if sel_shape == (): - chunk_value = value - elif is_scalar(value, self._dtype): - chunk_value = value - else: - chunk_value = value[out_selection] - # handle missing singleton dimensions - if indexer.drop_axes: - item = [slice(None)] * self.ndim - for a in indexer.drop_axes: - item[a] = np.newaxis - item = tuple(item) - chunk_value = chunk_value[item] - - # put data - self._chunk_setitem(chunk_coords, chunk_selection, chunk_value, fields=fields) - else: - lchunk_coords, lchunk_selection, lout_selection = zip(*indexer) - chunk_values = [] - for out_selection in lout_selection: - if sel_shape == (): - chunk_values.append(value) - elif is_scalar(value, self._dtype): - chunk_values.append(value) - else: - cv = value[out_selection] - # handle missing singleton dimensions - if indexer.drop_axes: # pragma: no cover - item = [slice(None)] * self.ndim - for a in indexer.drop_axes: - item[a] = np.newaxis - item = tuple(item) - cv = chunk_value[item] - chunk_values.append(cv) - - self._chunk_setitems(lchunk_coords, lchunk_selection, chunk_values, - fields=fields) - - def _process_chunk( - self, - out, - cdata, - chunk_selection, - drop_axes, - out_is_ndarray, - fields, - out_selection, - partial_read_decode=False, - ): - """Take binary data from storage and fill output array""" - if (out_is_ndarray and - not fields and - is_contiguous_selection(out_selection) and - is_total_slice(chunk_selection, self._chunks) and - not self._filters and - self._dtype != object): - - dest = out[out_selection] - write_direct = ( - dest.flags.writeable and - ( - (self._order == 'C' and dest.flags.c_contiguous) or - (self._order == 'F' and dest.flags.f_contiguous) - ) - ) - - if write_direct: - - # optimization: we want the whole chunk, and the destination is - # contiguous, so we can decompress directly from the chunk - # into the destination array - if self._compressor: - if isinstance(cdata, PartialReadBuffer): - cdata = cdata.read_full() - self._compressor.decode(cdata, dest) - else: - chunk = ensure_ndarray(cdata).view(self._dtype) - chunk = chunk.reshape(self._chunks, order=self._order) - np.copyto(dest, chunk) - return - - # decode chunk - try: - if partial_read_decode: - cdata.prepare_chunk() - # size of chunk - tmp = np.empty(self._chunks, dtype=self.dtype) - index_selection = PartialChunkIterator(chunk_selection, self.chunks) - for start, nitems, partial_out_selection in index_selection: - expected_shape = [ - len( - range(*partial_out_selection[i].indices(self.chunks[0] + 1)) - ) - if i < len(partial_out_selection) - else dim - for i, dim in enumerate(self.chunks) - ] - cdata.read_part(start, nitems) - chunk_partial = self._decode_chunk( - cdata.buff, - start=start, - nitems=nitems, - expected_shape=expected_shape, - ) - tmp[partial_out_selection] = chunk_partial - out[out_selection] = tmp[chunk_selection] - return - except ArrayIndexError: - cdata = cdata.read_full() - chunk = self._decode_chunk(cdata) - - # select data from chunk - if fields: - chunk = chunk[fields] - tmp = chunk[chunk_selection] - if drop_axes: - tmp = np.squeeze(tmp, axis=drop_axes) - - # store selected data in output - out[out_selection] = tmp - - def _chunk_getitem(self, chunk_coords, chunk_selection, out, out_selection, - drop_axes=None, fields=None): - """Obtain part or whole of a chunk. - - Parameters - ---------- - chunk_coords : tuple of ints - Indices of the chunk. - chunk_selection : selection - Location of region within the chunk to extract. - out : ndarray - Array to store result in. - out_selection : selection - Location of region within output array to store results in. - drop_axes : tuple of ints - Axes to squeeze out of the chunk. - fields - TODO - - """ - out_is_ndarray = True - try: - out = ensure_ndarray(out) - except TypeError: - out_is_ndarray = False - - assert len(chunk_coords) == len(self._cdata_shape) - - # obtain key for chunk - ckey = self._chunk_key(chunk_coords) - - try: - # obtain compressed data for chunk - cdata = self.chunk_store[ckey] - - except KeyError: - # chunk not initialized - if self._fill_value is not None: - if fields: - fill_value = self._fill_value[fields] - else: - fill_value = self._fill_value - out[out_selection] = fill_value - - else: - self._process_chunk(out, cdata, chunk_selection, drop_axes, - out_is_ndarray, fields, out_selection) - - def _chunk_getitems(self, lchunk_coords, lchunk_selection, out, lout_selection, - drop_axes=None, fields=None): - """As _chunk_getitem, but for lists of chunks - - This gets called where the storage supports ``getitems``, so that - it can decide how to fetch the keys, allowing concurrency. - """ - out_is_ndarray = True - try: - out = ensure_ndarray(out) - except TypeError: # pragma: no cover - out_is_ndarray = False - - ckeys = [self._chunk_key(ch) for ch in lchunk_coords] - if ( - self._partial_decompress - and self._compressor - and self._compressor.codec_id == "blosc" - and hasattr(self._compressor, "decode_partial") - and not fields - and self.dtype != object - and hasattr(self.chunk_store, "getitems") - ): - partial_read_decode = True - cdatas = { - ckey: PartialReadBuffer(ckey, self.chunk_store) - for ckey in ckeys - if ckey in self.chunk_store - } - else: - partial_read_decode = False - cdatas = self.chunk_store.getitems(ckeys, on_error="omit") - for ckey, chunk_select, out_select in zip(ckeys, lchunk_selection, lout_selection): - if ckey in cdatas: - self._process_chunk( - out, - cdatas[ckey], - chunk_select, - drop_axes, - out_is_ndarray, - fields, - out_select, - partial_read_decode=partial_read_decode, - ) - else: - # check exception type - if self._fill_value is not None: - if fields: - fill_value = self._fill_value[fields] - else: - fill_value = self._fill_value - out[out_select] = fill_value - - def _chunk_setitems(self, lchunk_coords, lchunk_selection, values, fields=None): - ckeys = [self._chunk_key(co) for co in lchunk_coords] - cdatas = [self._process_for_setitem(key, sel, val, fields=fields) - for key, sel, val in zip(ckeys, lchunk_selection, values)] - values = {k: v for k, v in zip(ckeys, cdatas)} - self.chunk_store.setitems(values) - - def _chunk_setitem(self, chunk_coords, chunk_selection, value, fields=None): - """Replace part or whole of a chunk. - - Parameters - ---------- - chunk_coords : tuple of ints - Indices of the chunk. - chunk_selection : tuple of slices - Location of region within the chunk. - value : scalar or ndarray - Value to set. - - """ - - if self._synchronizer is None: - # no synchronization - lock = nolock - else: - # synchronize on the chunk - ckey = self._chunk_key(chunk_coords) - lock = self._synchronizer[ckey] - - with lock: - self._chunk_setitem_nosync(chunk_coords, chunk_selection, value, - fields=fields) - - def _chunk_setitem_nosync(self, chunk_coords, chunk_selection, value, fields=None): - ckey = self._chunk_key(chunk_coords) - cdata = self._process_for_setitem(ckey, chunk_selection, value, fields=fields) - # store - self.chunk_store[ckey] = cdata - - def _process_for_setitem(self, ckey, chunk_selection, value, fields=None): - if is_total_slice(chunk_selection, self._chunks) and not fields: - # totally replace chunk - - # optimization: we are completely replacing the chunk, so no need - # to access the existing chunk data - - if is_scalar(value, self._dtype): - - # setup array filled with value - chunk = np.empty(self._chunks, dtype=self._dtype, order=self._order) - chunk.fill(value) - - else: - - # ensure array is contiguous - chunk = value.astype(self._dtype, order=self._order, copy=False) - - else: - # partially replace the contents of this chunk - - try: - - # obtain compressed data for chunk - cdata = self.chunk_store[ckey] - - except KeyError: - - # chunk not initialized - if self._fill_value is not None: - chunk = np.empty(self._chunks, dtype=self._dtype, order=self._order) - chunk.fill(self._fill_value) - elif self._dtype == object: - chunk = np.empty(self._chunks, dtype=self._dtype, order=self._order) - else: - # N.B., use zeros here so any region beyond the array has consistent - # and compressible data - chunk = np.zeros(self._chunks, dtype=self._dtype, order=self._order) - - else: - - # decode chunk - chunk = self._decode_chunk(cdata) - if not chunk.flags.writeable: - chunk = chunk.copy(order='K') - - # modify - if fields: - # N.B., currently multi-field assignment is not supported in numpy, so - # this only works for a single field - chunk[fields][chunk_selection] = value - else: - chunk[chunk_selection] = value - - # encode chunk - return self._encode_chunk(chunk) - - def _chunk_key(self, chunk_coords): - return self._key_prefix + self._dimension_separator.join(map(str, chunk_coords)) - - def _decode_chunk(self, cdata, start=None, nitems=None, expected_shape=None): - # decompress - if self._compressor: - # only decode requested items - if ( - all([x is not None for x in [start, nitems]]) - and self._compressor.codec_id == "blosc" - ) and hasattr(self._compressor, "decode_partial"): - chunk = self._compressor.decode_partial(cdata, start, nitems) - else: - chunk = self._compressor.decode(cdata) - else: - chunk = cdata - - # apply filters - if self._filters: - for f in reversed(self._filters): - chunk = f.decode(chunk) - - # view as numpy array with correct dtype - chunk = ensure_ndarray(chunk) - # special case object dtype, because incorrect handling can lead to - # segfaults and other bad things happening - if self._dtype != object: - chunk = chunk.view(self._dtype) - elif chunk.dtype != object: - # If we end up here, someone must have hacked around with the filters. - # We cannot deal with object arrays unless there is an object - # codec in the filter chain, i.e., a filter that converts from object - # array to something else during encoding, and converts back to object - # array during decoding. - raise RuntimeError('cannot read object array without object codec') - - # ensure correct chunk shape - chunk = chunk.reshape(-1, order='A') - chunk = chunk.reshape(expected_shape or self._chunks, order=self._order) - - return chunk - - def _encode_chunk(self, chunk): - - # apply filters - if self._filters: - for f in self._filters: - chunk = f.encode(chunk) - - # check object encoding - if ensure_ndarray(chunk).dtype == object: - raise RuntimeError('cannot write object array without object codec') - - # compress - if self._compressor: - cdata = self._compressor.encode(chunk) - else: - cdata = chunk - - # ensure in-memory data is immutable and easy to compare - if isinstance(self.chunk_store, dict): - cdata = ensure_bytes(cdata) - - return cdata - - def __repr__(self): - t = type(self) - r = '<{}.{}'.format(t.__module__, t.__name__) - if self.name: - r += ' %r' % self.name - r += ' %s' % str(self.shape) - r += ' %s' % self.dtype - if self._read_only: - r += ' read-only' - r += '>' - return r - - @property - def info(self): - """Report some diagnostic information about the array. - - Examples - -------- - >>> import zarr - >>> z = zarr.zeros(1000000, chunks=100000, dtype='i4') - >>> z.info - Type : zarr.core.Array - Data type : int32 - Shape : (1000000,) - Chunk shape : (100000,) - Order : C - Read-only : False - Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0) - Store type : builtins.dict - No. bytes : 4000000 (3.8M) - No. bytes stored : ... - Storage ratio : ... - Chunks initialized : 0/10 - - """ - return self._info_reporter - - def info_items(self): - return self._synchronized_op(self._info_items_nosync) - - def _info_items_nosync(self): - - def typestr(o): - return '{}.{}'.format(type(o).__module__, type(o).__name__) - - def bytestr(n): - if n > 2**10: - return '{} ({})'.format(n, human_readable_size(n)) - else: - return str(n) - - items = [] - - # basic info - if self.name is not None: - items += [('Name', self.name)] - items += [ - ('Type', typestr(self)), - ('Data type', '%s' % self.dtype), - ('Shape', str(self.shape)), - ('Chunk shape', str(self.chunks)), - ('Order', self.order), - ('Read-only', str(self.read_only)), - ] - - # filters - if self.filters: - for i, f in enumerate(self.filters): - items += [('Filter [%s]' % i, repr(f))] - - # compressor - items += [('Compressor', repr(self.compressor))] - - # synchronizer - if self._synchronizer is not None: - items += [('Synchronizer type', typestr(self._synchronizer))] - - # storage info - items += [('Store type', typestr(self._store))] - if self._chunk_store is not None: - items += [('Chunk store type', typestr(self._chunk_store))] - items += [('No. bytes', bytestr(self.nbytes))] - if self.nbytes_stored > 0: - items += [ - ('No. bytes stored', bytestr(self.nbytes_stored)), - ('Storage ratio', '%.1f' % (self.nbytes / self.nbytes_stored)), - ] - items += [ - ('Chunks initialized', '{}/{}'.format(self.nchunks_initialized, self.nchunks)) - ] - - return items - - def digest(self, hashname="sha1"): - """ - Compute a checksum for the data. Default uses sha1 for speed. - - Examples - -------- - >>> import binascii - >>> import zarr - >>> z = zarr.empty(shape=(10000, 10000), chunks=(1000, 1000)) - >>> binascii.hexlify(z.digest()) - b'041f90bc7a571452af4f850a8ca2c6cddfa8a1ac' - >>> z = zarr.zeros(shape=(10000, 10000), chunks=(1000, 1000)) - >>> binascii.hexlify(z.digest()) - b'7162d416d26a68063b66ed1f30e0a866e4abed60' - >>> z = zarr.zeros(shape=(10000, 10000), dtype="u1", chunks=(1000, 1000)) - >>> binascii.hexlify(z.digest()) - b'cb387af37410ae5a3222e893cf3373e4e4f22816' - """ - - h = hashlib.new(hashname) - - for i in itertools.product(*[range(s) for s in self.cdata_shape]): - h.update(self.chunk_store.get(self._chunk_key(i), b"")) - - h.update(self.store.get(self._key_prefix + array_meta_key, b"")) - - h.update(self.store.get(self.attrs.key, b"")) - - checksum = h.digest() - - return checksum - - def hexdigest(self, hashname="sha1"): - """ - Compute a checksum for the data. Default uses sha1 for speed. - - Examples - -------- - >>> import zarr - >>> z = zarr.empty(shape=(10000, 10000), chunks=(1000, 1000)) - >>> z.hexdigest() - '041f90bc7a571452af4f850a8ca2c6cddfa8a1ac' - >>> z = zarr.zeros(shape=(10000, 10000), chunks=(1000, 1000)) - >>> z.hexdigest() - '7162d416d26a68063b66ed1f30e0a866e4abed60' - >>> z = zarr.zeros(shape=(10000, 10000), dtype="u1", chunks=(1000, 1000)) - >>> z.hexdigest() - 'cb387af37410ae5a3222e893cf3373e4e4f22816' - """ - - checksum = binascii.hexlify(self.digest(hashname=hashname)) - - # This is a bytes object on Python 3 and we want a str. - if type(checksum) is not str: - checksum = checksum.decode('utf8') - - return checksum - - def __getstate__(self): - return (self._store, self._path, self._read_only, self._chunk_store, - self._synchronizer, self._cache_metadata, self._attrs.cache) - - def __setstate__(self, state): - self.__init__(*state) - - def _synchronized_op(self, f, *args, **kwargs): - - if self._synchronizer is None: - # no synchronization - lock = nolock - - else: - # synchronize on the array - mkey = self._key_prefix + array_meta_key - lock = self._synchronizer[mkey] - - with lock: - self._refresh_metadata_nosync() - result = f(*args, **kwargs) - - return result - - def _write_op(self, f, *args, **kwargs): - - # guard condition - if self._read_only: - raise ReadOnlyError() - - return self._synchronized_op(f, *args, **kwargs) - - def resize(self, *args): - """Change the shape of the array by growing or shrinking one or more - dimensions. - - Examples - -------- - >>> import zarr - >>> z = zarr.zeros(shape=(10000, 10000), chunks=(1000, 1000)) - >>> z.shape - (10000, 10000) - >>> z.resize(20000, 10000) - >>> z.shape - (20000, 10000) - >>> z.resize(30000, 1000) - >>> z.shape - (30000, 1000) - - Notes - ----- - When resizing an array, the data are not rearranged in any way. - - If one or more dimensions are shrunk, any chunks falling outside the - new array shape will be deleted from the underlying store. - - """ - - return self._write_op(self._resize_nosync, *args) - - def _resize_nosync(self, *args): - - # normalize new shape argument - old_shape = self._shape - new_shape = normalize_resize_args(old_shape, *args) - old_cdata_shape = self._cdata_shape - - # update metadata - self._shape = new_shape - self._flush_metadata_nosync() - - # determine the new number and arrangement of chunks - chunks = self._chunks - new_cdata_shape = tuple(math.ceil(s / c) - for s, c in zip(new_shape, chunks)) - - # remove any chunks not within range - chunk_store = self.chunk_store - for cidx in itertools.product(*[range(n) for n in old_cdata_shape]): - if all(i < c for i, c in zip(cidx, new_cdata_shape)): - pass # keep the chunk - else: - key = self._chunk_key(cidx) - try: - del chunk_store[key] - except KeyError: - # chunk not initialized - pass - - def append(self, data, axis=0): - """Append `data` to `axis`. - - Parameters - ---------- - data : array_like - Data to be appended. - axis : int - Axis along which to append. - - Returns - ------- - new_shape : tuple - - Notes - ----- - The size of all dimensions other than `axis` must match between this - array and `data`. - - Examples - -------- - >>> import numpy as np - >>> import zarr - >>> a = np.arange(10000000, dtype='i4').reshape(10000, 1000) - >>> z = zarr.array(a, chunks=(1000, 100)) - >>> z.shape - (10000, 1000) - >>> z.append(a) - (20000, 1000) - >>> z.append(np.vstack([a, a]), axis=1) - (20000, 2000) - >>> z.shape - (20000, 2000) - - """ - return self._write_op(self._append_nosync, data, axis=axis) - - def _append_nosync(self, data, axis=0): - - # ensure data is array-like - if not hasattr(data, 'shape'): - data = np.asanyarray(data) - - # ensure shapes are compatible for non-append dimensions - self_shape_preserved = tuple(s for i, s in enumerate(self._shape) - if i != axis) - data_shape_preserved = tuple(s for i, s in enumerate(data.shape) - if i != axis) - if self_shape_preserved != data_shape_preserved: - raise ValueError('shape of data to append is not compatible with the array; ' - 'all dimensions must match except for the dimension being ' - 'appended') - - # remember old shape - old_shape = self._shape - - # determine new shape - new_shape = tuple( - self._shape[i] if i != axis else self._shape[i] + data.shape[i] - for i in range(len(self._shape)) - ) - - # resize - self._resize_nosync(new_shape) - - # store data - # noinspection PyTypeChecker - append_selection = tuple( - slice(None) if i != axis else slice(old_shape[i], new_shape[i]) - for i in range(len(self._shape)) - ) - self[append_selection] = data - - return new_shape - - def view(self, shape=None, chunks=None, dtype=None, - fill_value=None, filters=None, read_only=None, - synchronizer=None): - """Return an array sharing the same data. - - Parameters - ---------- - shape : int or tuple of ints - Array shape. - chunks : int or tuple of ints, optional - Chunk shape. - dtype : string or dtype, optional - NumPy dtype. - fill_value : object - Default value to use for uninitialized portions of the array. - filters : sequence, optional - Sequence of filters to use to encode chunk data prior to - compression. - read_only : bool, optional - True if array should be protected against modification. - synchronizer : object, optional - Array synchronizer. - - Notes - ----- - WARNING: This is an experimental feature and should be used with care. - There are plenty of ways to generate errors and/or cause data - corruption. - - Examples - -------- - - Bypass filters: - - >>> import zarr - >>> import numpy as np - >>> np.random.seed(42) - >>> labels = ['female', 'male'] - >>> data = np.random.choice(labels, size=10000) - >>> filters = [zarr.Categorize(labels=labels, - ... dtype=data.dtype, - ... astype='u1')] - >>> a = zarr.array(data, chunks=1000, filters=filters) - >>> a[:] - array(['female', 'male', 'female', ..., 'male', 'male', 'female'], - dtype='>> v = a.view(dtype='u1', filters=[]) - >>> v.is_view - True - >>> v[:] - array([1, 2, 1, ..., 2, 2, 1], dtype=uint8) - - Views can be used to modify data: - - >>> x = v[:] - >>> x.sort() - >>> v[:] = x - >>> v[:] - array([1, 1, 1, ..., 2, 2, 2], dtype=uint8) - >>> a[:] - array(['female', 'female', 'female', ..., 'male', 'male', 'male'], - dtype='>> data = np.random.randint(0, 2, size=10000, dtype='u1') - >>> a = zarr.array(data, chunks=1000) - >>> a[:] - array([0, 0, 1, ..., 1, 0, 0], dtype=uint8) - >>> v = a.view(dtype=bool) - >>> v[:] - array([False, False, True, ..., True, False, False]) - >>> np.all(a[:].view(dtype=bool) == v[:]) - True - - An array can be viewed with a dtype with a different item size, however - some care is needed to adjust the shape and chunk shape so that chunk - data is interpreted correctly: - - >>> data = np.arange(10000, dtype='u2') - >>> a = zarr.array(data, chunks=1000) - >>> a[:10] - array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=uint16) - >>> v = a.view(dtype='u1', shape=20000, chunks=2000) - >>> v[:10] - array([0, 0, 1, 0, 2, 0, 3, 0, 4, 0], dtype=uint8) - >>> np.all(a[:].view('u1') == v[:]) - True - - Change fill value for uninitialized chunks: - - >>> a = zarr.full(10000, chunks=1000, fill_value=-1, dtype='i1') - >>> a[:] - array([-1, -1, -1, ..., -1, -1, -1], dtype=int8) - >>> v = a.view(fill_value=42) - >>> v[:] - array([42, 42, 42, ..., 42, 42, 42], dtype=int8) - - Note that resizing or appending to views is not permitted: - - >>> a = zarr.empty(10000) - >>> v = a.view() - >>> try: - ... v.resize(20000) - ... except PermissionError as e: - ... print(e) - operation not permitted for views - - """ - - store = self._store - chunk_store = self._chunk_store - path = self._path - if read_only is None: - read_only = self._read_only - if synchronizer is None: - synchronizer = self._synchronizer - a = Array(store=store, path=path, chunk_store=chunk_store, read_only=read_only, - synchronizer=synchronizer, cache_metadata=True) - a._is_view = True - - # allow override of some properties - if dtype is None: - dtype = self._dtype - else: - dtype = np.dtype(dtype) - a._dtype = dtype - if shape is None: - shape = self._shape - else: - shape = normalize_shape(shape) - a._shape = shape - if chunks is not None: - chunks = normalize_chunks(chunks, shape, dtype.itemsize) - a._chunks = chunks - if fill_value is not None: - a._fill_value = fill_value - if filters is not None: - a._filters = filters - - return a - - def astype(self, dtype): - """Returns a view that does on the fly type conversion of the underlying data. - - Parameters - ---------- - dtype : string or dtype - NumPy dtype. - - Notes - ----- - This method returns a new Array object which is a view on the same - underlying chunk data. Modifying any data via the view is currently - not permitted and will result in an error. This is an experimental - feature and its behavior is subject to change in the future. - - See Also - -------- - Array.view - - Examples - -------- - - >>> import zarr - >>> import numpy as np - >>> data = np.arange(100, dtype=np.uint8) - >>> a = zarr.array(data, chunks=10) - >>> a[:] - array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, - 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, - 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - 96, 97, 98, 99], dtype=uint8) - >>> v = a.astype(np.float32) - >>> v.is_view - True - >>> v[:] - array([ 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., - 10., 11., 12., 13., 14., 15., 16., 17., 18., 19., - 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., - 30., 31., 32., 33., 34., 35., 36., 37., 38., 39., - 40., 41., 42., 43., 44., 45., 46., 47., 48., 49., - 50., 51., 52., 53., 54., 55., 56., 57., 58., 59., - 60., 61., 62., 63., 64., 65., 66., 67., 68., 69., - 70., 71., 72., 73., 74., 75., 76., 77., 78., 79., - 80., 81., 82., 83., 84., 85., 86., 87., 88., 89., - 90., 91., 92., 93., 94., 95., 96., 97., 98., 99.], - dtype=float32) - """ - - dtype = np.dtype(dtype) - - filters = [] - if self._filters: - filters.extend(self._filters) - filters.insert(0, AsType(encode_dtype=self._dtype, decode_dtype=dtype)) - - return self.view(filters=filters, dtype=dtype, read_only=True) diff --git a/zarr/creation.py b/zarr/creation.py deleted file mode 100644 index 28f3b7b89d..0000000000 --- a/zarr/creation.py +++ /dev/null @@ -1,603 +0,0 @@ -import os -from warnings import warn - -import numpy as np -from numcodecs.registry import codec_registry - -from zarr.core import Array -from zarr.errors import ( - ArrayNotFoundError, - ContainsArrayError, - ContainsGroupError, -) -from zarr.n5 import N5Store -from zarr.storage import (DirectoryStore, ZipStore, contains_array, - contains_group, default_compressor, init_array, - normalize_storage_path, FSStore) -from zarr.util import normalize_dimension_separator - - -def create(shape, chunks=True, dtype=None, compressor='default', - fill_value=0, order='C', store=None, synchronizer=None, - overwrite=False, path=None, chunk_store=None, filters=None, - cache_metadata=True, cache_attrs=True, read_only=False, - object_codec=None, dimension_separator=None, **kwargs): - """Create an array. - - Parameters - ---------- - shape : int or tuple of ints - Array shape. - chunks : int or tuple of ints, optional - Chunk shape. If True, will be guessed from `shape` and `dtype`. If - False, will be set to `shape`, i.e., single chunk for the whole array. - If an int, the chunk size in each dimension will be given by the value - of `chunks`. Default is True. - dtype : string or dtype, optional - NumPy dtype. - compressor : Codec, optional - Primary compressor. - fill_value : object - Default value to use for uninitialized portions of the array. - order : {'C', 'F'}, optional - Memory layout to be used within each chunk. - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - synchronizer : object, optional - Array synchronizer. - overwrite : bool, optional - If True, delete all pre-existing data in `store` at `path` before - creating the array. - path : string, optional - Path under which array is stored. - chunk_store : MutableMapping, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - filters : sequence of Codecs, optional - Sequence of filters to use to encode chunk data prior to compression. - cache_metadata : bool, optional - If True, array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - read_only : bool, optional - True if array should be protected against modification. - object_codec : Codec, optional - A codec to encode object arrays, only needed if dtype=object. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - .. versionadded:: 2.8 - - Returns - ------- - z : zarr.core.Array - - Examples - -------- - - Create an array with default settings:: - - >>> import zarr - >>> z = zarr.create((10000, 10000), chunks=(1000, 1000)) - >>> z - - - Create an array with different some different configuration options:: - - >>> from numcodecs import Blosc - >>> compressor = Blosc(cname='zstd', clevel=1, shuffle=Blosc.BITSHUFFLE) - >>> z = zarr.create((10000, 10000), chunks=(1000, 1000), dtype='i1', order='F', - ... compressor=compressor) - >>> z - - - To create an array with object dtype requires a filter that can handle Python object - encoding, e.g., `MsgPack` or `Pickle` from `numcodecs`:: - - >>> from numcodecs import MsgPack - >>> z = zarr.create((10000, 10000), chunks=(1000, 1000), dtype=object, - ... object_codec=MsgPack()) - >>> z - - - Example with some filters, and also storing chunks separately from metadata:: - - >>> from numcodecs import Quantize, Adler32 - >>> store, chunk_store = dict(), dict() - >>> z = zarr.create((10000, 10000), chunks=(1000, 1000), dtype='f8', - ... filters=[Quantize(digits=2, dtype='f8'), Adler32()], - ... store=store, chunk_store=chunk_store) - >>> z - - - """ - - # handle polymorphic store arg - store = normalize_store_arg(store) - - # API compatibility with h5py - compressor, fill_value = _kwargs_compat(compressor, fill_value, kwargs) - - # optional array metadata - if dimension_separator is None: - dimension_separator = getattr(store, "_dimension_separator", None) - else: - if getattr(store, "_dimension_separator", None) != dimension_separator: - raise ValueError( - f"Specified dimension_separtor: {dimension_separator}" - f"conflicts with store's separator: " - f"{store._dimension_separator}") - dimension_separator = normalize_dimension_separator(dimension_separator) - - # initialize array metadata - init_array(store, shape=shape, chunks=chunks, dtype=dtype, compressor=compressor, - fill_value=fill_value, order=order, overwrite=overwrite, path=path, - chunk_store=chunk_store, filters=filters, object_codec=object_codec, - dimension_separator=dimension_separator) - - # instantiate array - z = Array(store, path=path, chunk_store=chunk_store, synchronizer=synchronizer, - cache_metadata=cache_metadata, cache_attrs=cache_attrs, read_only=read_only) - - return z - - -def normalize_store_arg(store, clobber=False, storage_options=None, mode='w'): - if store is None: - return dict() - if isinstance(store, os.PathLike): - store = os.fspath(store) - if isinstance(store, str): - mode = mode if clobber else "r" - if "://" in store or "::" in store: - return FSStore(store, mode=mode, **(storage_options or {})) - elif storage_options: - raise ValueError("storage_options passed with non-fsspec path") - if store.endswith('.zip'): - return ZipStore(store, mode=mode) - elif store.endswith('.n5'): - return N5Store(store) - else: - return DirectoryStore(store) - else: - return store - - -def _kwargs_compat(compressor, fill_value, kwargs): - - # to be compatible with h5py, as well as backwards-compatible with Zarr - # 1.x, accept 'compression' and 'compression_opts' keyword arguments - - if compressor != 'default': - # 'compressor' overrides 'compression' - if "compression" in kwargs: - warn( - "'compression' keyword argument overridden by 'compressor'", - stacklevel=3, - ) - del kwargs["compression"] - if "compression_opts" in kwargs: - warn( - "'compression_opts' keyword argument overridden by 'compressor'", - stacklevel=3, - ) - del kwargs["compression_opts"] - - elif 'compression' in kwargs: - compression = kwargs.pop('compression') - compression_opts = kwargs.pop('compression_opts', None) - - if compression is None or compression == 'none': - compressor = None - - elif compression == 'default': - compressor = default_compressor - - elif isinstance(compression, str): - codec_cls = codec_registry[compression] - - # handle compression_opts - if isinstance(compression_opts, dict): - compressor = codec_cls(**compression_opts) - elif isinstance(compression_opts, (list, tuple)): - compressor = codec_cls(*compression_opts) - elif compression_opts is None: - compressor = codec_cls() - else: - # assume single argument, e.g., int - compressor = codec_cls(compression_opts) - - # be lenient here if user gives compressor as 'compression' - elif hasattr(compression, 'get_config'): - compressor = compression - - else: - raise ValueError('bad value for compression: %r' % compression) - - # handle 'fillvalue' - if 'fillvalue' in kwargs: - # to be compatible with h5py, accept 'fillvalue' instead of - # 'fill_value' - fill_value = kwargs.pop('fillvalue') - - # ignore other keyword arguments - for k in kwargs: - warn('ignoring keyword argument %r' % k) - - return compressor, fill_value - - -def empty(shape, **kwargs): - """Create an empty array. - - For parameter definitions see :func:`zarr.creation.create`. - - Notes - ----- - The contents of an empty Zarr array are not defined. On attempting to - retrieve data from an empty Zarr array, any values may be returned, - and these are not guaranteed to be stable from one access to the next. - - """ - return create(shape=shape, fill_value=None, **kwargs) - - -def zeros(shape, **kwargs): - """Create an array, with zero being used as the default value for - uninitialized portions of the array. - - For parameter definitions see :func:`zarr.creation.create`. - - Examples - -------- - >>> import zarr - >>> z = zarr.zeros((10000, 10000), chunks=(1000, 1000)) - >>> z - - >>> z[:2, :2] - array([[0., 0.], - [0., 0.]]) - - """ - - return create(shape=shape, fill_value=0, **kwargs) - - -def ones(shape, **kwargs): - """Create an array, with one being used as the default value for - uninitialized portions of the array. - - For parameter definitions see :func:`zarr.creation.create`. - - Examples - -------- - >>> import zarr - >>> z = zarr.ones((10000, 10000), chunks=(1000, 1000)) - >>> z - - >>> z[:2, :2] - array([[1., 1.], - [1., 1.]]) - - """ - - return create(shape=shape, fill_value=1, **kwargs) - - -def full(shape, fill_value, **kwargs): - """Create an array, with `fill_value` being used as the default value for - uninitialized portions of the array. - - For parameter definitions see :func:`zarr.creation.create`. - - Examples - -------- - >>> import zarr - >>> z = zarr.full((10000, 10000), chunks=(1000, 1000), fill_value=42) - >>> z - - >>> z[:2, :2] - array([[42., 42.], - [42., 42.]]) - - """ - - return create(shape=shape, fill_value=fill_value, **kwargs) - - -def _get_shape_chunks(a): - shape = None - chunks = None - - if hasattr(a, 'shape') and \ - isinstance(a.shape, tuple): - shape = a.shape - - if hasattr(a, 'chunks') and \ - isinstance(a.chunks, tuple) and \ - (len(a.chunks) == len(a.shape)): - chunks = a.chunks - - elif hasattr(a, 'chunklen'): - # bcolz carray - chunks = (a.chunklen,) + a.shape[1:] - - return shape, chunks - - -def array(data, **kwargs): - """Create an array filled with `data`. - - The `data` argument should be a NumPy array or array-like object. For - other parameter definitions see :func:`zarr.creation.create`. - - Examples - -------- - >>> import numpy as np - >>> import zarr - >>> a = np.arange(100000000).reshape(10000, 10000) - >>> z = zarr.array(a, chunks=(1000, 1000)) - >>> z - - - """ - - # ensure data is array-like - if not hasattr(data, 'shape') or not hasattr(data, 'dtype'): - data = np.asanyarray(data) - - # setup dtype - kw_dtype = kwargs.get('dtype', None) - if kw_dtype is None: - kwargs['dtype'] = data.dtype - else: - kwargs['dtype'] = kw_dtype - - # setup shape and chunks - data_shape, data_chunks = _get_shape_chunks(data) - kwargs['shape'] = data_shape - kw_chunks = kwargs.get('chunks', None) - if kw_chunks is None: - kwargs['chunks'] = data_chunks - else: - kwargs['chunks'] = kw_chunks - - # pop read-only to apply after storing the data - read_only = kwargs.pop('read_only', False) - - # instantiate array - z = create(**kwargs) - - # fill with data - z[...] = data - - # set read_only property afterwards - z.read_only = read_only - - return z - - -def open_array( - store=None, - mode="a", - shape=None, - chunks=True, - dtype=None, - compressor="default", - fill_value=0, - order="C", - synchronizer=None, - filters=None, - cache_metadata=True, - cache_attrs=True, - path=None, - object_codec=None, - chunk_store=None, - storage_options=None, - partial_decompress=False, - **kwargs -): - """Open an array using file-mode-like semantics. - - Parameters - ---------- - store : MutableMapping or string, optional - Store or path to directory in file system or name of zip file. - mode : {'r', 'r+', 'a', 'w', 'w-'}, optional - Persistence mode: 'r' means read only (must exist); 'r+' means - read/write (must exist); 'a' means read/write (create if doesn't - exist); 'w' means create (overwrite if exists); 'w-' means create - (fail if exists). - shape : int or tuple of ints, optional - Array shape. - chunks : int or tuple of ints, optional - Chunk shape. If True, will be guessed from `shape` and `dtype`. If - False, will be set to `shape`, i.e., single chunk for the whole array. - If an int, the chunk size in each dimension will be given by the value - of `chunks`. Default is True. - dtype : string or dtype, optional - NumPy dtype. - compressor : Codec, optional - Primary compressor. - fill_value : object, optional - Default value to use for uninitialized portions of the array. - order : {'C', 'F'}, optional - Memory layout to be used within each chunk. - synchronizer : object, optional - Array synchronizer. - filters : sequence, optional - Sequence of filters to use to encode chunk data prior to compression. - cache_metadata : bool, optional - If True, array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - path : string, optional - Array path within store. - object_codec : Codec, optional - A codec to encode object arrays, only needed if dtype=object. - chunk_store : MutableMapping or string, optional - Store or path to directory in file system or name of zip file. - storage_options : dict - If using an fsspec URL to create the store, these will be passed to - the backend implementation. Ignored otherwise. - partial_decompress : bool, optional - If True and while the chunk_store is a FSStore and the compresion used - is Blosc, when getting data from the array chunks will be partially - read and decompressed when possible. - - .. versionadded:: 2.7 - - Returns - ------- - z : zarr.core.Array - - Examples - -------- - >>> import numpy as np - >>> import zarr - >>> z1 = zarr.open_array('data/example.zarr', mode='w', shape=(10000, 10000), - ... chunks=(1000, 1000), fill_value=0) - >>> z1[:] = np.arange(100000000).reshape(10000, 10000) - >>> z1 - - >>> z2 = zarr.open_array('data/example.zarr', mode='r') - >>> z2 - - >>> np.all(z1[:] == z2[:]) - True - - Notes - ----- - There is no need to close an array. Data are automatically flushed to the - file system. - - """ - - # use same mode semantics as h5py - # r : read only, must exist - # r+ : read/write, must exist - # w : create, delete if exists - # w- or x : create, fail if exists - # a : read/write if exists, create otherwise (default) - - # handle polymorphic store arg - clobber = (mode == 'w') - store = normalize_store_arg(store, clobber=clobber, storage_options=storage_options, mode=mode) - if chunk_store is not None: - chunk_store = normalize_store_arg(chunk_store, clobber=clobber, - storage_options=storage_options) - path = normalize_storage_path(path) - - # API compatibility with h5py - compressor, fill_value = _kwargs_compat(compressor, fill_value, kwargs) - - # ensure fill_value of correct type - if fill_value is not None: - fill_value = np.array(fill_value, dtype=dtype)[()] - - # ensure store is initialized - - if mode in ['r', 'r+']: - if not contains_array(store, path=path): - if contains_group(store, path=path): - raise ContainsGroupError(path) - raise ArrayNotFoundError(path) - - elif mode == 'w': - init_array(store, shape=shape, chunks=chunks, dtype=dtype, - compressor=compressor, fill_value=fill_value, - order=order, filters=filters, overwrite=True, path=path, - object_codec=object_codec, chunk_store=chunk_store) - - elif mode == 'a': - if not contains_array(store, path=path): - if contains_group(store, path=path): - raise ContainsGroupError(path) - init_array(store, shape=shape, chunks=chunks, dtype=dtype, - compressor=compressor, fill_value=fill_value, - order=order, filters=filters, path=path, - object_codec=object_codec, chunk_store=chunk_store) - - elif mode in ['w-', 'x']: - if contains_group(store, path=path): - raise ContainsGroupError(path) - elif contains_array(store, path=path): - raise ContainsArrayError(path) - else: - init_array(store, shape=shape, chunks=chunks, dtype=dtype, - compressor=compressor, fill_value=fill_value, - order=order, filters=filters, path=path, - object_codec=object_codec, chunk_store=chunk_store) - - # determine read only status - read_only = mode == 'r' - - # instantiate array - z = Array(store, read_only=read_only, synchronizer=synchronizer, - cache_metadata=cache_metadata, cache_attrs=cache_attrs, path=path, - chunk_store=chunk_store) - - return z - - -def _like_args(a, kwargs): - - shape, chunks = _get_shape_chunks(a) - if shape is not None: - kwargs.setdefault('shape', shape) - if chunks is not None: - kwargs.setdefault('chunks', chunks) - - if hasattr(a, 'dtype'): - kwargs.setdefault('dtype', a.dtype) - - if isinstance(a, Array): - kwargs.setdefault('compressor', a.compressor) - kwargs.setdefault('order', a.order) - kwargs.setdefault('filters', a.filters) - else: - kwargs.setdefault('compressor', 'default') - kwargs.setdefault('order', 'C') - - -def empty_like(a, **kwargs): - """Create an empty array like `a`.""" - _like_args(a, kwargs) - return empty(**kwargs) - - -def zeros_like(a, **kwargs): - """Create an array of zeros like `a`.""" - _like_args(a, kwargs) - return zeros(**kwargs) - - -def ones_like(a, **kwargs): - """Create an array of ones like `a`.""" - _like_args(a, kwargs) - return ones(**kwargs) - - -def full_like(a, **kwargs): - """Create a filled array like `a`.""" - _like_args(a, kwargs) - if isinstance(a, Array): - kwargs.setdefault('fill_value', a.fill_value) - return full(**kwargs) - - -def open_like(a, path, **kwargs): - """Open a persistent array like `a`.""" - _like_args(a, kwargs) - if isinstance(a, Array): - kwargs.setdefault('fill_value', a.fill_value) - return open_array(path, **kwargs) diff --git a/zarr/errors.py b/zarr/errors.py deleted file mode 100644 index 85c28ea8b6..0000000000 --- a/zarr/errors.py +++ /dev/null @@ -1,81 +0,0 @@ - - -class MetadataError(Exception): - pass - - -class CopyError(RuntimeError): - pass - - -class _BaseZarrError(ValueError): - _msg = "" - - def __init__(self, *args): - super().__init__(self._msg.format(*args)) - - -class ArrayIndexError(IndexError): - pass - - -class _BaseZarrIndexError(IndexError): - _msg = "" - - def __init__(self, *args): - super().__init__(self._msg.format(*args)) - - -class ContainsGroupError(_BaseZarrError): - _msg = "path {0!r} contains a group" - - -class ContainsArrayError(_BaseZarrError): - _msg = "path {0!r} contains an array" - - -class ArrayNotFoundError(_BaseZarrError): - _msg = "array not found at path %r' {0!r}" - - -class GroupNotFoundError(_BaseZarrError): - _msg = "group not found at path {0!r}" - - -class PathNotFoundError(_BaseZarrError): - _msg = "nothing found at path {0!r}" - - -class BadCompressorError(_BaseZarrError): - _msg = "bad compressor; expected Codec object, found {0!r}" - - -class FSPathExistNotDir(GroupNotFoundError): - _msg = "path exists but is not a directory: %r" - - -class ReadOnlyError(PermissionError): - def __init__(self): - super().__init__("object is read-only") - - -class BoundsCheckError(_BaseZarrIndexError): - _msg = "index out of bounds for dimension with length {0}" - - -class NegativeStepError(IndexError): - def __init__(self): - super().__init__("only slices with step >= 1 are supported") - - -def err_too_many_indices(selection, shape): - raise IndexError('too many indices for array; expected {}, got {}' - .format(len(shape), len(selection))) - - -class VindexInvalidSelectionError(_BaseZarrIndexError): - _msg = ( - "unsupported selection type for vectorized indexing; only " - "coordinate selection (tuple of integer arrays) and mask selection " - "(single Boolean array) are supported; got {0!r}" - ) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py deleted file mode 100644 index 87c2178e61..0000000000 --- a/zarr/hierarchy.py +++ /dev/null @@ -1,1191 +0,0 @@ -from collections.abc import MutableMapping -from itertools import islice - -import numpy as np - -from zarr.attrs import Attributes -from zarr.core import Array -from zarr.creation import (array, create, empty, empty_like, full, full_like, - normalize_store_arg, ones, ones_like, zeros, - zeros_like) -from zarr.errors import ( - ContainsArrayError, - ContainsGroupError, - GroupNotFoundError, - ReadOnlyError, -) -from zarr.meta import decode_group_metadata -from zarr.storage import (MemoryStore, attrs_key, contains_array, - contains_group, group_meta_key, init_group, listdir, - rename, rmdir) -from zarr.util import (InfoReporter, TreeViewer, is_valid_python_name, nolock, - normalize_shape, normalize_storage_path) - - -class Group(MutableMapping): - """Instantiate a group from an initialized store. - - Parameters - ---------- - store : MutableMapping - Group store, already initialized. - If the Group is used in a context manager, and the store has a ``close`` method, - it will be called on exit. - path : string, optional - Group path. - read_only : bool, optional - True if group should be protected against modification. - chunk_store : MutableMapping, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - synchronizer : object, optional - Array synchronizer. - - Attributes - ---------- - store - path - name - read_only - chunk_store - synchronizer - attrs - info - - Methods - ------- - __len__ - __iter__ - __contains__ - __getitem__ - __enter__ - __exit__ - group_keys - groups - array_keys - arrays - visit - visitkeys - visitvalues - visititems - tree - create_group - require_group - create_groups - require_groups - create_dataset - require_dataset - create - empty - zeros - ones - full - array - empty_like - zeros_like - ones_like - full_like - info - move - - """ - - def __init__(self, store, path=None, read_only=False, chunk_store=None, - cache_attrs=True, synchronizer=None): - self._store = store - self._chunk_store = chunk_store - self._path = normalize_storage_path(path) - if self._path: - self._key_prefix = self._path + '/' - else: - self._key_prefix = '' - self._read_only = read_only - self._synchronizer = synchronizer - - # guard conditions - if contains_array(store, path=self._path): - raise ContainsArrayError(path) - - # initialize metadata - try: - mkey = self._key_prefix + group_meta_key - meta_bytes = store[mkey] - except KeyError: - raise GroupNotFoundError(path) - else: - meta = decode_group_metadata(meta_bytes) - self._meta = meta - - # setup attributes - akey = self._key_prefix + attrs_key - self._attrs = Attributes(store, key=akey, read_only=read_only, - cache=cache_attrs, synchronizer=synchronizer) - - # setup info - self._info = InfoReporter(self) - - @property - def store(self): - """A MutableMapping providing the underlying storage for the group.""" - return self._store - - @property - def path(self): - """Storage path.""" - return self._path - - @property - def name(self): - """Group name following h5py convention.""" - if self._path: - # follow h5py convention: add leading slash - name = self._path - if name[0] != '/': - name = '/' + name - return name - return '/' - - @property - def basename(self): - """Final component of name.""" - return self.name.split('/')[-1] - - @property - def read_only(self): - """A boolean, True if modification operations are not permitted.""" - return self._read_only - - @property - def chunk_store(self): - """A MutableMapping providing the underlying storage for array chunks.""" - if self._chunk_store is None: - return self._store - else: - return self._chunk_store - - @property - def synchronizer(self): - """Object used to synchronize write access to groups and arrays.""" - return self._synchronizer - - @property - def attrs(self): - """A MutableMapping containing user-defined attributes. Note that - attribute values must be JSON serializable.""" - return self._attrs - - @property - def info(self): - """Return diagnostic information about the group.""" - return self._info - - def __eq__(self, other): - return ( - isinstance(other, Group) and - self._store == other.store and - self._read_only == other.read_only and - self._path == other.path - # N.B., no need to compare attributes, should be covered by - # store comparison - ) - - def __iter__(self): - """Return an iterator over group member names. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_dataset('baz', shape=100, chunks=10) - >>> d2 = g1.create_dataset('quux', shape=200, chunks=20) - >>> for name in g1: - ... print(name) - bar - baz - foo - quux - - """ - for key in sorted(listdir(self._store, self._path)): - path = self._key_prefix + key - if (contains_array(self._store, path) or - contains_group(self._store, path)): - yield key - - def __len__(self): - """Number of members.""" - return sum(1 for _ in self) - - def __repr__(self): - t = type(self) - r = '<{}.{}'.format(t.__module__, t.__name__) - if self.name: - r += ' %r' % self.name - if self._read_only: - r += ' read-only' - r += '>' - return r - - def __enter__(self): - """Return the Group for use as a context manager.""" - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """If the underlying Store has a ``close`` method, call it.""" - try: - self.store.close() - except AttributeError: - pass - - def info_items(self): - - def typestr(o): - return '{}.{}'.format(type(o).__module__, type(o).__name__) - - items = [] - - # basic info - if self.name is not None: - items += [('Name', self.name)] - items += [ - ('Type', typestr(self)), - ('Read-only', str(self.read_only)), - ] - - # synchronizer - if self._synchronizer is not None: - items += [('Synchronizer type', typestr(self._synchronizer))] - - # storage info - items += [('Store type', typestr(self._store))] - if self._chunk_store is not None: - items += [('Chunk store type', typestr(self._chunk_store))] - - # members - items += [('No. members', len(self))] - array_keys = sorted(self.array_keys()) - group_keys = sorted(self.group_keys()) - items += [('No. arrays', len(array_keys))] - items += [('No. groups', len(group_keys))] - if array_keys: - items += [('Arrays', ', '.join(array_keys))] - if group_keys: - items += [('Groups', ', '.join(group_keys))] - - return items - - def __getstate__(self): - return (self._store, self._path, self._read_only, self._chunk_store, - self.attrs.cache, self._synchronizer) - - def __setstate__(self, state): - self.__init__(*state) - - def _item_path(self, item): - absolute = isinstance(item, str) and item and item[0] == '/' - path = normalize_storage_path(item) - if not absolute and self._path: - path = self._key_prefix + path - return path - - def __contains__(self, item): - """Test for group membership. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> d1 = g1.create_dataset('bar', shape=100, chunks=10) - >>> 'foo' in g1 - True - >>> 'bar' in g1 - True - >>> 'baz' in g1 - False - - """ - path = self._item_path(item) - return contains_array(self._store, path) or \ - contains_group(self._store, path) - - def __getitem__(self, item): - """Obtain a group member. - - Parameters - ---------- - item : string - Member name or path. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> d1 = g1.create_dataset('foo/bar/baz', shape=100, chunks=10) - >>> g1['foo'] - - >>> g1['foo/bar'] - - >>> g1['foo/bar/baz'] - - - """ - path = self._item_path(item) - if contains_array(self._store, path): - return Array(self._store, read_only=self._read_only, path=path, - chunk_store=self._chunk_store, - synchronizer=self._synchronizer, cache_attrs=self.attrs.cache) - elif contains_group(self._store, path): - return Group(self._store, read_only=self._read_only, path=path, - chunk_store=self._chunk_store, cache_attrs=self.attrs.cache, - synchronizer=self._synchronizer) - else: - raise KeyError(item) - - def __setitem__(self, item, value): - self.array(item, value, overwrite=True) - - def __delitem__(self, item): - return self._write_op(self._delitem_nosync, item) - - def _delitem_nosync(self, item): - path = self._item_path(item) - if contains_array(self._store, path) or \ - contains_group(self._store, path): - rmdir(self._store, path) - else: - raise KeyError(item) - - def __getattr__(self, item): - # allow access to group members via dot notation - try: - return self.__getitem__(item) - except KeyError: - raise AttributeError - - def __dir__(self): - # noinspection PyUnresolvedReferences - base = super().__dir__() - keys = sorted(set(base + list(self))) - keys = [k for k in keys if is_valid_python_name(k)] - return keys - - def _ipython_key_completions_(self): - return sorted(self) - - def group_keys(self): - """Return an iterator over member names for groups only. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_dataset('baz', shape=100, chunks=10) - >>> d2 = g1.create_dataset('quux', shape=200, chunks=20) - >>> sorted(g1.group_keys()) - ['bar', 'foo'] - - """ - for key in sorted(listdir(self._store, self._path)): - path = self._key_prefix + key - if contains_group(self._store, path): - yield key - - def groups(self): - """Return an iterator over (name, value) pairs for groups only. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_dataset('baz', shape=100, chunks=10) - >>> d2 = g1.create_dataset('quux', shape=200, chunks=20) - >>> for n, v in g1.groups(): - ... print(n, type(v)) - bar - foo - - """ - for key in sorted(listdir(self._store, self._path)): - path = self._key_prefix + key - if contains_group(self._store, path): - yield key, Group(self._store, path=path, read_only=self._read_only, - chunk_store=self._chunk_store, - cache_attrs=self.attrs.cache, - synchronizer=self._synchronizer) - - def array_keys(self, recurse=False): - """Return an iterator over member names for arrays only. - - Parameters - ---------- - recurse : recurse, optional - Option to return member names for all arrays, even from groups - below the current one. If False, only member names for arrays in - the current group will be returned. Default value is False. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_dataset('baz', shape=100, chunks=10) - >>> d2 = g1.create_dataset('quux', shape=200, chunks=20) - >>> sorted(g1.array_keys()) - ['baz', 'quux'] - - """ - return self._array_iter(keys_only=True, - method='array_keys', - recurse=recurse) - - def arrays(self, recurse=False): - """Return an iterator over (name, value) pairs for arrays only. - - Parameters - ---------- - recurse : recurse, optional - Option to return (name, value) pairs for all arrays, even from groups - below the current one. If False, only (name, value) pairs for arrays in - the current group will be returned. Default value is False. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_dataset('baz', shape=100, chunks=10) - >>> d2 = g1.create_dataset('quux', shape=200, chunks=20) - >>> for n, v in g1.arrays(): - ... print(n, type(v)) - baz - quux - - """ - return self._array_iter(keys_only=False, - method='arrays', - recurse=recurse) - - def _array_iter(self, keys_only, method, recurse): - for key in sorted(listdir(self._store, self._path)): - path = self._key_prefix + key - if contains_array(self._store, path): - yield key if keys_only else (key, self[key]) - elif recurse and contains_group(self._store, path): - group = self[key] - for i in getattr(group, method)(recurse=recurse): - yield i - - def visitvalues(self, func): - """Run ``func`` on each object. - - Note: If ``func`` returns ``None`` (or doesn't return), - iteration continues. However, if ``func`` returns - anything else, it ceases and returns that value. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g3.create_group('baz') - >>> g5 = g3.create_group('quux') - >>> def print_visitor(obj): - ... print(obj) - >>> g1.visitvalues(print_visitor) - - - - - >>> g3.visitvalues(print_visitor) - - - - """ - - def _visit(obj): - yield obj - keys = sorted(getattr(obj, "keys", lambda: [])()) - for k in keys: - for v in _visit(obj[k]): - yield v - - for each_obj in islice(_visit(self), 1, None): - value = func(each_obj) - if value is not None: - return value - - def visit(self, func): - """Run ``func`` on each object's path. - - Note: If ``func`` returns ``None`` (or doesn't return), - iteration continues. However, if ``func`` returns - anything else, it ceases and returns that value. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g3.create_group('baz') - >>> g5 = g3.create_group('quux') - >>> def print_visitor(name): - ... print(name) - >>> g1.visit(print_visitor) - bar - bar/baz - bar/quux - foo - >>> g3.visit(print_visitor) - baz - quux - - """ - - base_len = len(self.name) - return self.visitvalues(lambda o: func(o.name[base_len:].lstrip("/"))) - - def visitkeys(self, func): - """An alias for :py:meth:`~Group.visit`. - """ - - return self.visit(func) - - def visititems(self, func): - """Run ``func`` on each object's path and the object itself. - - Note: If ``func`` returns ``None`` (or doesn't return), - iteration continues. However, if ``func`` returns - anything else, it ceases and returns that value. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g3.create_group('baz') - >>> g5 = g3.create_group('quux') - >>> def print_visitor(name, obj): - ... print((name, obj)) - >>> g1.visititems(print_visitor) - ('bar', ) - ('bar/baz', ) - ('bar/quux', ) - ('foo', ) - >>> g3.visititems(print_visitor) - ('baz', ) - ('quux', ) - - """ - - base_len = len(self.name) - return self.visitvalues(lambda o: func(o.name[base_len:].lstrip("/"), o)) - - def tree(self, expand=False, level=None): - """Provide a ``print``-able display of the hierarchy. - - Parameters - ---------- - expand : bool, optional - Only relevant for HTML representation. If True, tree will be fully expanded. - level : int, optional - Maximum depth to descend into hierarchy. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g3.create_group('baz') - >>> g5 = g3.create_group('quux') - >>> d1 = g5.create_dataset('baz', shape=100, chunks=10) - >>> g1.tree() - / - ├── bar - │ ├── baz - │ └── quux - │ └── baz (100,) float64 - └── foo - >>> g1.tree(level=2) - / - ├── bar - │ ├── baz - │ └── quux - └── foo - >>> g3.tree() - bar - ├── baz - └── quux - └── baz (100,) float64 - - Notes - ----- - Please note that this is an experimental feature. The behaviour of this - function is still evolving and the default output and/or parameters may change - in future versions. - - """ - - return TreeViewer(self, expand=expand, level=level) - - def _write_op(self, f, *args, **kwargs): - - # guard condition - if self._read_only: - raise ReadOnlyError() - - if self._synchronizer is None: - # no synchronization - lock = nolock - else: - # synchronize on the root group - lock = self._synchronizer[group_meta_key] - - with lock: - return f(*args, **kwargs) - - def create_group(self, name, overwrite=False): - """Create a sub-group. - - Parameters - ---------- - name : string - Group name. - overwrite : bool, optional - If True, overwrite any existing array with the given name. - - Returns - ------- - g : zarr.hierarchy.Group - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g1.create_group('baz/quux') - - """ - - return self._write_op(self._create_group_nosync, name, overwrite=overwrite) - - def _create_group_nosync(self, name, overwrite=False): - path = self._item_path(name) - - # create terminal group - init_group(self._store, path=path, chunk_store=self._chunk_store, - overwrite=overwrite) - - return Group(self._store, path=path, read_only=self._read_only, - chunk_store=self._chunk_store, cache_attrs=self.attrs.cache, - synchronizer=self._synchronizer) - - def create_groups(self, *names, **kwargs): - """Convenience method to create multiple groups in a single call.""" - return tuple(self.create_group(name, **kwargs) for name in names) - - def require_group(self, name, overwrite=False): - """Obtain a sub-group, creating one if it doesn't exist. - - Parameters - ---------- - name : string - Group name. - overwrite : bool, optional - Overwrite any existing array with given `name` if present. - - Returns - ------- - g : zarr.hierarchy.Group - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> g2 = g1.require_group('foo') - >>> g3 = g1.require_group('foo') - >>> g2 == g3 - True - - """ - - return self._write_op(self._require_group_nosync, name, - overwrite=overwrite) - - def _require_group_nosync(self, name, overwrite=False): - path = self._item_path(name) - - # create terminal group if necessary - if not contains_group(self._store, path): - init_group(store=self._store, path=path, chunk_store=self._chunk_store, - overwrite=overwrite) - - return Group(self._store, path=path, read_only=self._read_only, - chunk_store=self._chunk_store, cache_attrs=self.attrs.cache, - synchronizer=self._synchronizer) - - def require_groups(self, *names): - """Convenience method to require multiple groups in a single call.""" - return tuple(self.require_group(name) for name in names) - - # noinspection PyIncorrectDocstring - def create_dataset(self, name, **kwargs): - """Create an array. - - Arrays are known as "datasets" in HDF5 terminology. For compatibility - with h5py, Zarr groups also implement the require_dataset() method. - - Parameters - ---------- - name : string - Array name. - data : array_like, optional - Initial data. - shape : int or tuple of ints - Array shape. - chunks : int or tuple of ints, optional - Chunk shape. If not provided, will be guessed from `shape` and - `dtype`. - dtype : string or dtype, optional - NumPy dtype. - compressor : Codec, optional - Primary compressor. - fill_value : object - Default value to use for uninitialized portions of the array. - order : {'C', 'F'}, optional - Memory layout to be used within each chunk. - synchronizer : zarr.sync.ArraySynchronizer, optional - Array synchronizer. - filters : sequence of Codecs, optional - Sequence of filters to use to encode chunk data prior to - compression. - overwrite : bool, optional - If True, replace any existing array or group with the given name. - cache_metadata : bool, optional - If True, array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - - Returns - ------- - a : zarr.core.Array - - Examples - -------- - >>> import zarr - >>> g1 = zarr.group() - >>> d1 = g1.create_dataset('foo', shape=(10000, 10000), - ... chunks=(1000, 1000)) - >>> d1 - - >>> d2 = g1.create_dataset('bar/baz/qux', shape=(100, 100, 100), - ... chunks=(100, 10, 10)) - >>> d2 - - - """ - - return self._write_op(self._create_dataset_nosync, name, **kwargs) - - def _create_dataset_nosync(self, name, data=None, **kwargs): - - path = self._item_path(name) - - # determine synchronizer - kwargs.setdefault('synchronizer', self._synchronizer) - kwargs.setdefault('cache_attrs', self.attrs.cache) - - # create array - if data is None: - a = create(store=self._store, path=path, chunk_store=self._chunk_store, - **kwargs) - - else: - a = array(data, store=self._store, path=path, chunk_store=self._chunk_store, - **kwargs) - - return a - - def require_dataset(self, name, shape, dtype=None, exact=False, **kwargs): - """Obtain an array, creating if it doesn't exist. - - Arrays are known as "datasets" in HDF5 terminology. For compatibility - with h5py, Zarr groups also implement the create_dataset() method. - - Other `kwargs` are as per :func:`zarr.hierarchy.Group.create_dataset`. - - Parameters - ---------- - name : string - Array name. - shape : int or tuple of ints - Array shape. - dtype : string or dtype, optional - NumPy dtype. - exact : bool, optional - If True, require `dtype` to match exactly. If false, require - `dtype` can be cast from array dtype. - - """ - - return self._write_op(self._require_dataset_nosync, name, shape=shape, - dtype=dtype, exact=exact, **kwargs) - - def _require_dataset_nosync(self, name, shape, dtype=None, exact=False, - **kwargs): - - path = self._item_path(name) - - if contains_array(self._store, path): - - # array already exists at path, validate that it is the right shape and type - - synchronizer = kwargs.get('synchronizer', self._synchronizer) - cache_metadata = kwargs.get('cache_metadata', True) - cache_attrs = kwargs.get('cache_attrs', self.attrs.cache) - a = Array(self._store, path=path, read_only=self._read_only, - chunk_store=self._chunk_store, synchronizer=synchronizer, - cache_metadata=cache_metadata, cache_attrs=cache_attrs) - shape = normalize_shape(shape) - if shape != a.shape: - raise TypeError('shape do not match existing array; expected {}, got {}' - .format(a.shape, shape)) - dtype = np.dtype(dtype) - if exact: - if dtype != a.dtype: - raise TypeError('dtypes do not match exactly; expected {}, got {}' - .format(a.dtype, dtype)) - else: - if not np.can_cast(dtype, a.dtype): - raise TypeError('dtypes ({}, {}) cannot be safely cast' - .format(dtype, a.dtype)) - return a - - else: - return self._create_dataset_nosync(name, shape=shape, dtype=dtype, - **kwargs) - - def create(self, name, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.creation.create`.""" - return self._write_op(self._create_nosync, name, **kwargs) - - def _create_nosync(self, name, **kwargs): - path = self._item_path(name) - kwargs.setdefault('synchronizer', self._synchronizer) - kwargs.setdefault('cache_attrs', self.attrs.cache) - return create(store=self._store, path=path, chunk_store=self._chunk_store, - **kwargs) - - def empty(self, name, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.creation.empty`.""" - return self._write_op(self._empty_nosync, name, **kwargs) - - def _empty_nosync(self, name, **kwargs): - path = self._item_path(name) - kwargs.setdefault('synchronizer', self._synchronizer) - kwargs.setdefault('cache_attrs', self.attrs.cache) - return empty(store=self._store, path=path, chunk_store=self._chunk_store, - **kwargs) - - def zeros(self, name, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.creation.zeros`.""" - return self._write_op(self._zeros_nosync, name, **kwargs) - - def _zeros_nosync(self, name, **kwargs): - path = self._item_path(name) - kwargs.setdefault('synchronizer', self._synchronizer) - kwargs.setdefault('cache_attrs', self.attrs.cache) - return zeros(store=self._store, path=path, chunk_store=self._chunk_store, - **kwargs) - - def ones(self, name, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.creation.ones`.""" - return self._write_op(self._ones_nosync, name, **kwargs) - - def _ones_nosync(self, name, **kwargs): - path = self._item_path(name) - kwargs.setdefault('synchronizer', self._synchronizer) - kwargs.setdefault('cache_attrs', self.attrs.cache) - return ones(store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - - def full(self, name, fill_value, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.creation.full`.""" - return self._write_op(self._full_nosync, name, fill_value, **kwargs) - - def _full_nosync(self, name, fill_value, **kwargs): - path = self._item_path(name) - kwargs.setdefault('synchronizer', self._synchronizer) - kwargs.setdefault('cache_attrs', self.attrs.cache) - return full(store=self._store, path=path, chunk_store=self._chunk_store, - fill_value=fill_value, **kwargs) - - def array(self, name, data, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.creation.array`.""" - return self._write_op(self._array_nosync, name, data, **kwargs) - - def _array_nosync(self, name, data, **kwargs): - path = self._item_path(name) - kwargs.setdefault('synchronizer', self._synchronizer) - kwargs.setdefault('cache_attrs', self.attrs.cache) - return array(data, store=self._store, path=path, chunk_store=self._chunk_store, - **kwargs) - - def empty_like(self, name, data, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.creation.empty_like`.""" - return self._write_op(self._empty_like_nosync, name, data, **kwargs) - - def _empty_like_nosync(self, name, data, **kwargs): - path = self._item_path(name) - kwargs.setdefault('synchronizer', self._synchronizer) - kwargs.setdefault('cache_attrs', self.attrs.cache) - return empty_like(data, store=self._store, path=path, - chunk_store=self._chunk_store, **kwargs) - - def zeros_like(self, name, data, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.creation.zeros_like`.""" - return self._write_op(self._zeros_like_nosync, name, data, **kwargs) - - def _zeros_like_nosync(self, name, data, **kwargs): - path = self._item_path(name) - kwargs.setdefault('synchronizer', self._synchronizer) - kwargs.setdefault('cache_attrs', self.attrs.cache) - return zeros_like(data, store=self._store, path=path, - chunk_store=self._chunk_store, **kwargs) - - def ones_like(self, name, data, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.creation.ones_like`.""" - return self._write_op(self._ones_like_nosync, name, data, **kwargs) - - def _ones_like_nosync(self, name, data, **kwargs): - path = self._item_path(name) - kwargs.setdefault('synchronizer', self._synchronizer) - kwargs.setdefault('cache_attrs', self.attrs.cache) - return ones_like(data, store=self._store, path=path, - chunk_store=self._chunk_store, **kwargs) - - def full_like(self, name, data, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.creation.full_like`.""" - return self._write_op(self._full_like_nosync, name, data, **kwargs) - - def _full_like_nosync(self, name, data, **kwargs): - path = self._item_path(name) - kwargs.setdefault('synchronizer', self._synchronizer) - kwargs.setdefault('cache_attrs', self.attrs.cache) - return full_like(data, store=self._store, path=path, - chunk_store=self._chunk_store, **kwargs) - - def _move_nosync(self, path, new_path): - rename(self._store, path, new_path) - if self._chunk_store is not None: - rename(self._chunk_store, path, new_path) - - def move(self, source, dest): - """Move contents from one path to another relative to the Group. - - Parameters - ---------- - source : string - Name or path to a Zarr object to move. - dest : string - New name or path of the Zarr object. - """ - - source = self._item_path(source) - dest = self._item_path(dest) - - # Check that source exists. - if not (contains_array(self._store, source) or - contains_group(self._store, source)): - raise ValueError('The source, "%s", does not exist.' % source) - if contains_array(self._store, dest) or contains_group(self._store, dest): - raise ValueError('The dest, "%s", already exists.' % dest) - - # Ensure groups needed for `dest` exist. - if "/" in dest: - self.require_group("/" + dest.rsplit("/", 1)[0]) - - self._write_op(self._move_nosync, source, dest) - - -def _normalize_store_arg(store, *, clobber=False, storage_options=None, mode=None): - if store is None: - return MemoryStore() - return normalize_store_arg(store, clobber=clobber, - storage_options=storage_options, mode=mode) - - -def group(store=None, overwrite=False, chunk_store=None, - cache_attrs=True, synchronizer=None, path=None): - """Create a group. - - Parameters - ---------- - store : MutableMapping or string, optional - Store or path to directory in file system. - overwrite : bool, optional - If True, delete any pre-existing data in `store` at `path` before - creating the group. - chunk_store : MutableMapping, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - synchronizer : object, optional - Array synchronizer. - path : string, optional - Group path within store. - - Returns - ------- - g : zarr.hierarchy.Group - - Examples - -------- - Create a group in memory:: - - >>> import zarr - >>> g = zarr.group() - >>> g - - - Create a group with a different store:: - - >>> store = zarr.DirectoryStore('data/example.zarr') - >>> g = zarr.group(store=store, overwrite=True) - >>> g - - - """ - - # handle polymorphic store arg - store = _normalize_store_arg(store) - path = normalize_storage_path(path) - - # require group - if overwrite or not contains_group(store): - init_group(store, overwrite=overwrite, chunk_store=chunk_store, - path=path) - - return Group(store, read_only=False, chunk_store=chunk_store, - cache_attrs=cache_attrs, synchronizer=synchronizer, path=path) - - -def open_group(store=None, mode='a', cache_attrs=True, synchronizer=None, path=None, - chunk_store=None, storage_options=None): - """Open a group using file-mode-like semantics. - - Parameters - ---------- - store : MutableMapping or string, optional - Store or path to directory in file system or name of zip file. - mode : {'r', 'r+', 'a', 'w', 'w-'}, optional - Persistence mode: 'r' means read only (must exist); 'r+' means - read/write (must exist); 'a' means read/write (create if doesn't - exist); 'w' means create (overwrite if exists); 'w-' means create - (fail if exists). - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - synchronizer : object, optional - Array synchronizer. - path : string, optional - Group path within store. - chunk_store : MutableMapping or string, optional - Store or path to directory in file system or name of zip file. - storage_options : dict - If using an fsspec URL to create the store, these will be passed to - the backend implementation. Ignored otherwise. - - Returns - ------- - g : zarr.hierarchy.Group - - Examples - -------- - >>> import zarr - >>> root = zarr.open_group('data/example.zarr', mode='w') - >>> foo = root.create_group('foo') - >>> bar = root.create_group('bar') - >>> root - - >>> root2 = zarr.open_group('data/example.zarr', mode='a') - >>> root2 - - >>> root == root2 - True - - """ - - # handle polymorphic store arg - clobber = mode != "r" - store = _normalize_store_arg( - store, clobber=clobber, storage_options=storage_options, mode=mode - ) - if chunk_store is not None: - chunk_store = _normalize_store_arg(chunk_store, clobber=clobber, - storage_options=storage_options) - path = normalize_storage_path(path) - - # ensure store is initialized - - if mode in ['r', 'r+']: - if not contains_group(store, path=path): - if contains_array(store, path=path): - raise ContainsArrayError(path) - raise GroupNotFoundError(path) - - elif mode == 'w': - init_group(store, overwrite=True, path=path, chunk_store=chunk_store) - - elif mode == 'a': - if not contains_group(store, path=path): - if contains_array(store, path=path): - raise ContainsArrayError(path) - init_group(store, path=path, chunk_store=chunk_store) - - elif mode in ['w-', 'x']: - if contains_array(store, path=path): - raise ContainsArrayError(path) - elif contains_group(store, path=path): - raise ContainsGroupError(path) - else: - init_group(store, path=path, chunk_store=chunk_store) - - # determine read only status - read_only = mode == 'r' - - return Group(store, read_only=read_only, cache_attrs=cache_attrs, - synchronizer=synchronizer, path=path, chunk_store=chunk_store) diff --git a/zarr/indexing.py b/zarr/indexing.py deleted file mode 100644 index e58e7ba339..0000000000 --- a/zarr/indexing.py +++ /dev/null @@ -1,942 +0,0 @@ -import collections -import itertools -import math -import numbers - -import numpy as np - - -from zarr.errors import ( - ArrayIndexError, - NegativeStepError, - err_too_many_indices, - VindexInvalidSelectionError, - BoundsCheckError, -) - - -def is_integer(x): - return isinstance(x, numbers.Integral) - - -def is_integer_array(x, ndim=None): - t = hasattr(x, 'shape') and hasattr(x, 'dtype') and x.dtype.kind in 'ui' - if ndim is not None: - t = t and len(x.shape) == ndim - return t - - -def is_bool_array(x, ndim=None): - t = hasattr(x, 'shape') and hasattr(x, 'dtype') and x.dtype == bool - if ndim is not None: - t = t and len(x.shape) == ndim - return t - - -def is_scalar(value, dtype): - if np.isscalar(value): - return True - if isinstance(value, tuple) and dtype.names and len(value) == len(dtype.names): - return True - return False - - -def normalize_integer_selection(dim_sel, dim_len): - - # normalize type to int - dim_sel = int(dim_sel) - - # handle wraparound - if dim_sel < 0: - dim_sel = dim_len + dim_sel - - # handle out of bounds - if dim_sel >= dim_len or dim_sel < 0: - raise BoundsCheckError(dim_len) - - return dim_sel - - -ChunkDimProjection = collections.namedtuple( - 'ChunkDimProjection', - ('dim_chunk_ix', 'dim_chunk_sel', 'dim_out_sel') -) -"""A mapping from chunk to output array for a single dimension. - -Parameters ----------- -dim_chunk_ix - Index of chunk. -dim_chunk_sel - Selection of items from chunk array. -dim_out_sel - Selection of items in target (output) array. - -""" - - -class IntDimIndexer(object): - - def __init__(self, dim_sel, dim_len, dim_chunk_len): - - # normalize - dim_sel = normalize_integer_selection(dim_sel, dim_len) - - # store attributes - self.dim_sel = dim_sel - self.dim_len = dim_len - self.dim_chunk_len = dim_chunk_len - self.nitems = 1 - - def __iter__(self): - dim_chunk_ix = self.dim_sel // self.dim_chunk_len - dim_offset = dim_chunk_ix * self.dim_chunk_len - dim_chunk_sel = self.dim_sel - dim_offset - dim_out_sel = None - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) - - -def ceildiv(a, b): - return math.ceil(a / b) - - -class SliceDimIndexer(object): - - def __init__(self, dim_sel, dim_len, dim_chunk_len): - - # normalize - self.start, self.stop, self.step = dim_sel.indices(dim_len) - if self.step < 1: - raise NegativeStepError() - - # store attributes - self.dim_len = dim_len - self.dim_chunk_len = dim_chunk_len - self.nitems = max(0, ceildiv((self.stop - self.start), self.step)) - self.nchunks = ceildiv(self.dim_len, self.dim_chunk_len) - - def __iter__(self): - - # figure out the range of chunks we need to visit - dim_chunk_ix_from = self.start // self.dim_chunk_len - dim_chunk_ix_to = ceildiv(self.stop, self.dim_chunk_len) - - # iterate over chunks in range - for dim_chunk_ix in range(dim_chunk_ix_from, dim_chunk_ix_to): - - # compute offsets for chunk within overall array - dim_offset = dim_chunk_ix * self.dim_chunk_len - dim_limit = min(self.dim_len, (dim_chunk_ix + 1) * self.dim_chunk_len) - - # determine chunk length, accounting for trailing chunk - dim_chunk_len = dim_limit - dim_offset - - if self.start < dim_offset: - # selection starts before current chunk - dim_chunk_sel_start = 0 - remainder = (dim_offset - self.start) % self.step - if remainder: - dim_chunk_sel_start += self.step - remainder - # compute number of previous items, provides offset into output array - dim_out_offset = ceildiv((dim_offset - self.start), self.step) - - else: - # selection starts within current chunk - dim_chunk_sel_start = self.start - dim_offset - dim_out_offset = 0 - - if self.stop > dim_limit: - # selection ends after current chunk - dim_chunk_sel_stop = dim_chunk_len - - else: - # selection ends within current chunk - dim_chunk_sel_stop = self.stop - dim_offset - - dim_chunk_sel = slice(dim_chunk_sel_start, dim_chunk_sel_stop, self.step) - dim_chunk_nitems = ceildiv((dim_chunk_sel_stop - dim_chunk_sel_start), - self.step) - dim_out_sel = slice(dim_out_offset, dim_out_offset + dim_chunk_nitems) - - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) - - -def check_selection_length(selection, shape): - if len(selection) > len(shape): - err_too_many_indices(selection, shape) - - -def replace_ellipsis(selection, shape): - - selection = ensure_tuple(selection) - - # count number of ellipsis present - n_ellipsis = sum(1 for i in selection if i is Ellipsis) - - if n_ellipsis > 1: - # more than 1 is an error - raise IndexError("an index can only have a single ellipsis ('...')") - - elif n_ellipsis == 1: - # locate the ellipsis, count how many items to left and right - n_items_l = selection.index(Ellipsis) # items to left of ellipsis - n_items_r = len(selection) - (n_items_l + 1) # items to right of ellipsis - n_items = len(selection) - 1 # all non-ellipsis items - - if n_items >= len(shape): - # ellipsis does nothing, just remove it - selection = tuple(i for i in selection if i != Ellipsis) - - else: - # replace ellipsis with as many slices are needed for number of dims - new_item = selection[:n_items_l] + ((slice(None),) * (len(shape) - n_items)) - if n_items_r: - new_item += selection[-n_items_r:] - selection = new_item - - # fill out selection if not completely specified - if len(selection) < len(shape): - selection += (slice(None),) * (len(shape) - len(selection)) - - # check selection not too long - check_selection_length(selection, shape) - - return selection - - -def replace_lists(selection): - return tuple( - np.asarray(dim_sel) if isinstance(dim_sel, list) else dim_sel - for dim_sel in selection - ) - - -def ensure_tuple(v): - if not isinstance(v, tuple): - v = (v,) - return v - - -ChunkProjection = collections.namedtuple( - 'ChunkProjection', - ('chunk_coords', 'chunk_selection', 'out_selection') -) -"""A mapping of items from chunk to output array. Can be used to extract items from the -chunk array for loading into an output array. Can also be used to extract items from a -value array for setting/updating in a chunk array. - -Parameters ----------- -chunk_coords - Indices of chunk. -chunk_selection - Selection of items from chunk array. -out_selection - Selection of items in target (output) array. - -""" - - -def is_slice(s): - return isinstance(s, slice) - - -def is_contiguous_slice(s): - return is_slice(s) and (s.step is None or s.step == 1) - - -def is_positive_slice(s): - return is_slice(s) and (s.step is None or s.step >= 1) - - -def is_contiguous_selection(selection): - selection = ensure_tuple(selection) - return all([ - (is_integer_array(s) or is_contiguous_slice(s) or s == Ellipsis) - for s in selection - ]) - - -def is_basic_selection(selection): - selection = ensure_tuple(selection) - return all([is_integer(s) or is_positive_slice(s) for s in selection]) - - -# noinspection PyProtectedMember -class BasicIndexer(object): - - def __init__(self, selection, array): - - # handle ellipsis - selection = replace_ellipsis(selection, array._shape) - - # setup per-dimension indexers - dim_indexers = [] - for dim_sel, dim_len, dim_chunk_len in \ - zip(selection, array._shape, array._chunks): - - if is_integer(dim_sel): - dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_chunk_len) - - elif is_slice(dim_sel): - dim_indexer = SliceDimIndexer(dim_sel, dim_len, dim_chunk_len) - - else: - raise IndexError('unsupported selection item for basic indexing; ' - 'expected integer or slice, got {!r}' - .format(type(dim_sel))) - - dim_indexers.append(dim_indexer) - - self.dim_indexers = dim_indexers - self.shape = tuple(s.nitems for s in self.dim_indexers - if not isinstance(s, IntDimIndexer)) - self.drop_axes = None - - def __iter__(self): - for dim_projections in itertools.product(*self.dim_indexers): - - chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) - chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) - out_selection = tuple(p.dim_out_sel for p in dim_projections - if p.dim_out_sel is not None) - - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) - - -class BoolArrayDimIndexer(object): - - def __init__(self, dim_sel, dim_len, dim_chunk_len): - - # check number of dimensions - if not is_bool_array(dim_sel, 1): - raise IndexError('Boolean arrays in an orthogonal selection must ' - 'be 1-dimensional only') - - # check shape - if dim_sel.shape[0] != dim_len: - raise IndexError('Boolean array has the wrong length for dimension; ' - 'expected {}, got {}'.format(dim_len, dim_sel.shape[0])) - - # store attributes - self.dim_sel = dim_sel - self.dim_len = dim_len - self.dim_chunk_len = dim_chunk_len - self.nchunks = ceildiv(self.dim_len, self.dim_chunk_len) - - # precompute number of selected items for each chunk - self.chunk_nitems = np.zeros(self.nchunks, dtype='i8') - for dim_chunk_ix in range(self.nchunks): - dim_offset = dim_chunk_ix * self.dim_chunk_len - self.chunk_nitems[dim_chunk_ix] = np.count_nonzero( - self.dim_sel[dim_offset:dim_offset + self.dim_chunk_len] - ) - self.chunk_nitems_cumsum = np.cumsum(self.chunk_nitems) - self.nitems = self.chunk_nitems_cumsum[-1] - self.dim_chunk_ixs = np.nonzero(self.chunk_nitems)[0] - - def __iter__(self): - - # iterate over chunks with at least one item - for dim_chunk_ix in self.dim_chunk_ixs: - - # find region in chunk - dim_offset = dim_chunk_ix * self.dim_chunk_len - dim_chunk_sel = self.dim_sel[dim_offset:dim_offset + self.dim_chunk_len] - - # pad out if final chunk - if dim_chunk_sel.shape[0] < self.dim_chunk_len: - tmp = np.zeros(self.dim_chunk_len, dtype=bool) - tmp[:dim_chunk_sel.shape[0]] = dim_chunk_sel - dim_chunk_sel = tmp - - # find region in output - if dim_chunk_ix == 0: - start = 0 - else: - start = self.chunk_nitems_cumsum[dim_chunk_ix - 1] - stop = self.chunk_nitems_cumsum[dim_chunk_ix] - dim_out_sel = slice(start, stop) - - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) - - -class Order: - UNKNOWN = 0 - INCREASING = 1 - DECREASING = 2 - UNORDERED = 3 - - @staticmethod - def check(a): - diff = np.diff(a) - diff_positive = diff >= 0 - n_diff_positive = np.count_nonzero(diff_positive) - all_increasing = n_diff_positive == len(diff_positive) - any_increasing = n_diff_positive > 0 - if all_increasing: - order = Order.INCREASING - elif any_increasing: - order = Order.UNORDERED - else: - order = Order.DECREASING - return order - - -def wraparound_indices(x, dim_len): - loc_neg = x < 0 - if np.any(loc_neg): - x[loc_neg] = x[loc_neg] + dim_len - - -def boundscheck_indices(x, dim_len): - if np.any(x < 0) or np.any(x >= dim_len): - raise BoundsCheckError(dim_len) - - -class IntArrayDimIndexer(object): - """Integer array selection against a single dimension.""" - - def __init__(self, dim_sel, dim_len, dim_chunk_len, wraparound=True, boundscheck=True, - order=Order.UNKNOWN): - - # ensure 1d array - dim_sel = np.asanyarray(dim_sel) - if not is_integer_array(dim_sel, 1): - raise IndexError('integer arrays in an orthogonal selection must be ' - '1-dimensional only') - - # handle wraparound - if wraparound: - wraparound_indices(dim_sel, dim_len) - - # handle out of bounds - if boundscheck: - boundscheck_indices(dim_sel, dim_len) - - # store attributes - self.dim_len = dim_len - self.dim_chunk_len = dim_chunk_len - self.nchunks = ceildiv(self.dim_len, self.dim_chunk_len) - self.nitems = len(dim_sel) - - # determine which chunk is needed for each selection item - # note: for dense integer selections, the division operation here is the - # bottleneck - dim_sel_chunk = dim_sel // dim_chunk_len - - # determine order of indices - if order == Order.UNKNOWN: - order = Order.check(dim_sel) - self.order = order - - if self.order == Order.INCREASING: - self.dim_sel = dim_sel - self.dim_out_sel = None - elif self.order == Order.DECREASING: - self.dim_sel = dim_sel[::-1] - # TODO should be possible to do this without creating an arange - self.dim_out_sel = np.arange(self.nitems - 1, -1, -1) - else: - # sort indices to group by chunk - self.dim_out_sel = np.argsort(dim_sel_chunk) - self.dim_sel = np.take(dim_sel, self.dim_out_sel) - - # precompute number of selected items for each chunk - self.chunk_nitems = np.bincount(dim_sel_chunk, minlength=self.nchunks) - - # find chunks that we need to visit - self.dim_chunk_ixs = np.nonzero(self.chunk_nitems)[0] - - # compute offsets into the output array - self.chunk_nitems_cumsum = np.cumsum(self.chunk_nitems) - - def __iter__(self): - - for dim_chunk_ix in self.dim_chunk_ixs: - - # find region in output - if dim_chunk_ix == 0: - start = 0 - else: - start = self.chunk_nitems_cumsum[dim_chunk_ix - 1] - stop = self.chunk_nitems_cumsum[dim_chunk_ix] - if self.order == Order.INCREASING: - dim_out_sel = slice(start, stop) - else: - dim_out_sel = self.dim_out_sel[start:stop] - - # find region in chunk - dim_offset = dim_chunk_ix * self.dim_chunk_len - dim_chunk_sel = self.dim_sel[start:stop] - dim_offset - - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) - - -def slice_to_range(s: slice, l: int): # noqa: E741 - return range(*s.indices(l)) - - -def ix_(selection, shape): - """Convert an orthogonal selection to a numpy advanced (fancy) selection, like numpy.ix_ - but with support for slices and single ints.""" - - # normalisation - selection = replace_ellipsis(selection, shape) - - # replace slice and int as these are not supported by numpy.ix_ - selection = [slice_to_range(dim_sel, dim_len) if isinstance(dim_sel, slice) - else [dim_sel] if is_integer(dim_sel) - else dim_sel - for dim_sel, dim_len in zip(selection, shape)] - - # now get numpy to convert to a coordinate selection - selection = np.ix_(*selection) - - return selection - - -def oindex(a, selection): - """Implementation of orthogonal indexing with slices and ints.""" - selection = replace_ellipsis(selection, a.shape) - drop_axes = tuple([i for i, s in enumerate(selection) if is_integer(s)]) - selection = ix_(selection, a.shape) - result = a[selection] - if drop_axes: - result = result.squeeze(axis=drop_axes) - return result - - -def oindex_set(a, selection, value): - selection = replace_ellipsis(selection, a.shape) - drop_axes = tuple([i for i, s in enumerate(selection) if is_integer(s)]) - selection = ix_(selection, a.shape) - if not np.isscalar(value) and drop_axes: - value = np.asanyarray(value) - value_selection = [slice(None)] * len(a.shape) - for i in drop_axes: - value_selection[i] = np.newaxis - value_selection = tuple(value_selection) - value = value[value_selection] - a[selection] = value - - -# noinspection PyProtectedMember -class OrthogonalIndexer(object): - - def __init__(self, selection, array): - - # handle ellipsis - selection = replace_ellipsis(selection, array._shape) - - # normalize list to array - selection = replace_lists(selection) - - # setup per-dimension indexers - dim_indexers = [] - for dim_sel, dim_len, dim_chunk_len in \ - zip(selection, array._shape, array._chunks): - - if is_integer(dim_sel): - dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_chunk_len) - - elif isinstance(dim_sel, slice): - dim_indexer = SliceDimIndexer(dim_sel, dim_len, dim_chunk_len) - - elif is_integer_array(dim_sel): - dim_indexer = IntArrayDimIndexer(dim_sel, dim_len, dim_chunk_len) - - elif is_bool_array(dim_sel): - dim_indexer = BoolArrayDimIndexer(dim_sel, dim_len, dim_chunk_len) - - else: - raise IndexError('unsupported selection item for orthogonal indexing; ' - 'expected integer, slice, integer array or Boolean ' - 'array, got {!r}' - .format(type(dim_sel))) - - dim_indexers.append(dim_indexer) - - self.array = array - self.dim_indexers = dim_indexers - self.shape = tuple(s.nitems for s in self.dim_indexers - if not isinstance(s, IntDimIndexer)) - self.is_advanced = not is_basic_selection(selection) - if self.is_advanced: - self.drop_axes = tuple([i for i, dim_indexer in enumerate(self.dim_indexers) - if isinstance(dim_indexer, IntDimIndexer)]) - else: - self.drop_axes = None - - def __iter__(self): - for dim_projections in itertools.product(*self.dim_indexers): - - chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) - chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) - out_selection = tuple(p.dim_out_sel for p in dim_projections - if p.dim_out_sel is not None) - - # handle advanced indexing arrays orthogonally - if self.is_advanced: - - # N.B., numpy doesn't support orthogonal indexing directly as yet, - # so need to work around via np.ix_. Also np.ix_ does not support a - # mixture of arrays and slices or integers, so need to convert slices - # and integers into ranges. - chunk_selection = ix_(chunk_selection, self.array._chunks) - - # special case for non-monotonic indices - if not is_basic_selection(out_selection): - out_selection = ix_(out_selection, self.shape) - - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) - - -class OIndex(object): - - def __init__(self, array): - self.array = array - - def __getitem__(self, selection): - fields, selection = pop_fields(selection) - selection = ensure_tuple(selection) - selection = replace_lists(selection) - return self.array.get_orthogonal_selection(selection, fields=fields) - - def __setitem__(self, selection, value): - fields, selection = pop_fields(selection) - selection = ensure_tuple(selection) - selection = replace_lists(selection) - return self.array.set_orthogonal_selection(selection, value, fields=fields) - - -# noinspection PyProtectedMember -def is_coordinate_selection(selection, array): - return ( - (len(selection) == len(array._shape)) and - all([is_integer(dim_sel) or is_integer_array(dim_sel) - for dim_sel in selection]) - ) - - -# noinspection PyProtectedMember -def is_mask_selection(selection, array): - return ( - len(selection) == 1 and - is_bool_array(selection[0]) and - selection[0].shape == array._shape - ) - - -# noinspection PyProtectedMember -class CoordinateIndexer(object): - - def __init__(self, selection, array): - - # some initial normalization - selection = ensure_tuple(selection) - selection = tuple([i] if is_integer(i) else i for i in selection) - selection = replace_lists(selection) - - # validation - if not is_coordinate_selection(selection, array): - raise IndexError('invalid coordinate selection; expected one integer ' - '(coordinate) array per dimension of the target array, ' - 'got {!r}'.format(selection)) - - # handle wraparound, boundscheck - for dim_sel, dim_len in zip(selection, array.shape): - - # handle wraparound - wraparound_indices(dim_sel, dim_len) - - # handle out of bounds - boundscheck_indices(dim_sel, dim_len) - - # compute chunk index for each point in the selection - chunks_multi_index = tuple( - dim_sel // dim_chunk_len - for (dim_sel, dim_chunk_len) in zip(selection, array._chunks) - ) - - # broadcast selection - this will raise error if array dimensions don't match - selection = np.broadcast_arrays(*selection) - chunks_multi_index = np.broadcast_arrays(*chunks_multi_index) - - # remember shape of selection, because we will flatten indices for processing - self.sel_shape = selection[0].shape if selection[0].shape else (1,) - - # flatten selection - selection = [dim_sel.reshape(-1) for dim_sel in selection] - chunks_multi_index = [dim_chunks.reshape(-1) for dim_chunks in chunks_multi_index] - - # ravel chunk indices - chunks_raveled_indices = np.ravel_multi_index(chunks_multi_index, - dims=array._cdata_shape) - - # group points by chunk - if np.any(np.diff(chunks_raveled_indices) < 0): - # optimisation, only sort if needed - sel_sort = np.argsort(chunks_raveled_indices) - selection = tuple(dim_sel[sel_sort] for dim_sel in selection) - else: - sel_sort = None - - # store attributes - self.selection = selection - self.sel_sort = sel_sort - self.shape = selection[0].shape if selection[0].shape else (1,) - self.drop_axes = None - self.array = array - - # precompute number of selected items for each chunk - self.chunk_nitems = np.bincount(chunks_raveled_indices, minlength=array.nchunks) - self.chunk_nitems_cumsum = np.cumsum(self.chunk_nitems) - # locate the chunks we need to process - self.chunk_rixs = np.nonzero(self.chunk_nitems)[0] - - # unravel chunk indices - self.chunk_mixs = np.unravel_index(self.chunk_rixs, array._cdata_shape) - - def __iter__(self): - - # iterate over chunks - for i, chunk_rix in enumerate(self.chunk_rixs): - - chunk_coords = tuple(m[i] for m in self.chunk_mixs) - if chunk_rix == 0: - start = 0 - else: - start = self.chunk_nitems_cumsum[chunk_rix - 1] - stop = self.chunk_nitems_cumsum[chunk_rix] - if self.sel_sort is None: - out_selection = slice(start, stop) - else: - out_selection = self.sel_sort[start:stop] - - chunk_offsets = tuple( - dim_chunk_ix * dim_chunk_len - for dim_chunk_ix, dim_chunk_len in zip(chunk_coords, self.array._chunks) - ) - chunk_selection = tuple( - dim_sel[start:stop] - dim_chunk_offset - for (dim_sel, dim_chunk_offset) in zip(self.selection, chunk_offsets) - ) - - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) - - -# noinspection PyProtectedMember -class MaskIndexer(CoordinateIndexer): - - def __init__(self, selection, array): - - # some initial normalization - selection = ensure_tuple(selection) - selection = replace_lists(selection) - - # validation - if not is_mask_selection(selection, array): - raise IndexError('invalid mask selection; expected one Boolean (mask)' - 'array with the same shape as the target array, got {!r}' - .format(selection)) - - # convert to indices - selection = np.nonzero(selection[0]) - - # delegate the rest to superclass - super().__init__(selection, array) - - -class VIndex(object): - - def __init__(self, array): - self.array = array - - def __getitem__(self, selection): - fields, selection = pop_fields(selection) - selection = ensure_tuple(selection) - selection = replace_lists(selection) - if is_coordinate_selection(selection, self.array): - return self.array.get_coordinate_selection(selection, fields=fields) - elif is_mask_selection(selection, self.array): - return self.array.get_mask_selection(selection, fields=fields) - else: - raise VindexInvalidSelectionError(selection) - - def __setitem__(self, selection, value): - fields, selection = pop_fields(selection) - selection = ensure_tuple(selection) - selection = replace_lists(selection) - if is_coordinate_selection(selection, self.array): - self.array.set_coordinate_selection(selection, value, fields=fields) - elif is_mask_selection(selection, self.array): - self.array.set_mask_selection(selection, value, fields=fields) - else: - raise VindexInvalidSelectionError(selection) - - -def check_fields(fields, dtype): - # early out - if fields is None: - return dtype - # check type - if not isinstance(fields, (str, list, tuple)): - raise IndexError("'fields' argument must be a string or list of strings; found " - "{!r}".format(type(fields))) - if fields: - if dtype.names is None: - raise IndexError("invalid 'fields' argument, array does not have any fields") - try: - if isinstance(fields, str): - # single field selection - out_dtype = dtype[fields] - else: - # multiple field selection - out_dtype = np.dtype([(f, dtype[f]) for f in fields]) - except KeyError as e: - raise IndexError("invalid 'fields' argument, field not found: {!r}".format(e)) - else: - return out_dtype - else: - return dtype - - -def check_no_multi_fields(fields): - if isinstance(fields, list): - if len(fields) == 1: - return fields[0] - elif len(fields) > 1: - raise IndexError('multiple fields are not supported for this operation') - return fields - - -def pop_fields(selection): - if isinstance(selection, str): - # single field selection - fields = selection - selection = () - elif not isinstance(selection, tuple): - # single selection item, no fields - fields = None - # leave selection as-is - else: - # multiple items, split fields from selection items - fields = [f for f in selection if isinstance(f, str)] - fields = fields[0] if len(fields) == 1 else fields - selection = tuple(s for s in selection if not isinstance(s, str)) - selection = selection[0] if len(selection) == 1 else selection - return fields, selection - - -def make_slice_selection(selection): - ls = [] - for dim_selection in selection: - if is_integer(dim_selection): - ls.append(slice(dim_selection, dim_selection + 1, 1)) - elif isinstance(dim_selection, np.ndarray): - if len(dim_selection) == 1: - ls.append(slice(dim_selection[0], dim_selection[0] + 1, 1)) - else: - raise ArrayIndexError() - else: - ls.append(dim_selection) - return ls - - -class PartialChunkIterator(object): - """Iterator to retrieve the specific coordinates of requested data - from within a compressed chunk. - - Parameters - ---------- - selection : tuple - tuple of slice objects to take from the chunk - arr_shape : shape of chunk to select data from - - Attributes - ----------- - arr_shape - selection - - Returns - ------- - Tuple with 3 elements: - - start: int - elements offset in the chunk to read from - nitems: int - number of elements to read in the chunk from start - partial_out_selection: list of slices - indices of a temporary empty array of size `Array._chunks` to assign - the decompressed data to after the partial read. - - Notes - ----- - An array is flattened when compressed with blosc, so this iterator takes - the wanted selection of an array and determines the wanted coordinates - of the flattened, compressed data to be read and then decompressed. The - decompressed data is then placed in a temporary empty array of size - `Array._chunks` at the indices yielded as partial_out_selection. - Once all the slices yielded by this iterator have been read, decompressed - and written to the temporary array, the wanted slice of the chunk can be - indexed from the temporary array and written to the out_selection slice - of the out array. - - """ - - def __init__(self, selection, arr_shape): - selection = make_slice_selection(selection) - self.arr_shape = arr_shape - - # number of selection dimensions can't be greater than the number of chunk dimensions - if len(selection) > len(self.arr_shape): - raise ValueError( - "Selection has more dimensions then the array:\n" - f"selection dimensions = {len(selection)}\n" - f"array dimensions = {len(self.arr_shape)}" - ) - - # any selection can not be out of the range of the chunk - selection_shape = np.empty(self.arr_shape)[tuple(selection)].shape - if any( - [ - selection_dim < 0 or selection_dim > arr_dim - for selection_dim, arr_dim in zip(selection_shape, self.arr_shape) - ] - ): - raise IndexError( - "a selection index is out of range for the dimension" - ) # pragma: no cover - - for i, dim_size in enumerate(self.arr_shape[::-1]): - index = len(self.arr_shape) - (i + 1) - if index <= len(selection) - 1: - slice_size = selection_shape[index] - if slice_size == dim_size and index > 0: - selection.pop() - else: - break - - chunk_loc_slices = [] - last_dim_slice = None if selection[-1].step > 1 else selection.pop() - for arr_shape_i, sl in zip(arr_shape, selection): - dim_chunk_loc_slices = [] - assert isinstance(sl, slice) - for x in slice_to_range(sl, arr_shape_i): - dim_chunk_loc_slices.append(slice(x, x + 1, 1)) - chunk_loc_slices.append(dim_chunk_loc_slices) - if last_dim_slice: - chunk_loc_slices.append([last_dim_slice]) - self.chunk_loc_slices = list(itertools.product(*chunk_loc_slices)) - - def __iter__(self): - chunk1 = self.chunk_loc_slices[0] - nitems = (chunk1[-1].stop - chunk1[-1].start) * np.prod( - self.arr_shape[len(chunk1):], dtype=int - ) - for partial_out_selection in self.chunk_loc_slices: - start = 0 - for i, sl in enumerate(partial_out_selection): - start += sl.start * np.prod(self.arr_shape[i + 1:], dtype=int) - yield start, nitems, partial_out_selection diff --git a/zarr/meta.py b/zarr/meta.py deleted file mode 100644 index 53b2b779db..0000000000 --- a/zarr/meta.py +++ /dev/null @@ -1,228 +0,0 @@ -import base64 -from collections.abc import Mapping - -import numpy as np - -from zarr.errors import MetadataError -from zarr.util import json_dumps, json_loads - -from typing import cast, Union, Any, List, Mapping as MappingType - -ZARR_FORMAT = 2 - - -def parse_metadata(s: Union[MappingType, str]) -> MappingType[str, Any]: - - # Here we allow that a store may return an already-parsed metadata object, - # or a string of JSON that we will parse here. We allow for an already-parsed - # object to accommodate a consolidated metadata store, where all the metadata for - # all groups and arrays will already have been parsed from JSON. - - if isinstance(s, Mapping): - # assume metadata has already been parsed into a mapping object - meta = s - - else: - # assume metadata needs to be parsed as JSON - meta = json_loads(s) - - return meta - - -def decode_array_metadata(s: Union[MappingType, str]) -> MappingType[str, Any]: - meta = parse_metadata(s) - - # check metadata format - zarr_format = meta.get('zarr_format', None) - if zarr_format != ZARR_FORMAT: - raise MetadataError('unsupported zarr format: %s' % zarr_format) - - # extract array metadata fields - try: - dtype = decode_dtype(meta['dtype']) - - if dtype.hasobject: - import numcodecs - object_codec = numcodecs.get_codec(meta['filters'][0]) - else: - object_codec = None - - fill_value = decode_fill_value(meta['fill_value'], dtype, object_codec) - meta = dict( - zarr_format=meta['zarr_format'], - shape=tuple(meta['shape']), - chunks=tuple(meta['chunks']), - dtype=dtype, - compressor=meta['compressor'], - fill_value=fill_value, - order=meta['order'], - filters=meta['filters'], - dimension_separator=meta.get('dimension_separator', '.'), - ) - - except Exception as e: - raise MetadataError('error decoding metadata: %s' % e) - else: - return meta - - -def encode_array_metadata(meta: MappingType[str, Any]) -> bytes: - dtype = meta['dtype'] - sdshape = () - if dtype.subdtype is not None: - dtype, sdshape = dtype.subdtype - - dimension_separator = meta.get('dimension_separator') - if dtype.hasobject: - import numcodecs - object_codec = numcodecs.get_codec(meta['filters'][0]) - else: - object_codec = None - meta = dict( - zarr_format=ZARR_FORMAT, - shape=meta['shape'] + sdshape, - chunks=meta['chunks'], - dtype=encode_dtype(dtype), - compressor=meta['compressor'], - fill_value=encode_fill_value(meta['fill_value'], dtype, object_codec), - order=meta['order'], - filters=meta['filters'], - ) - - if dimension_separator: - meta['dimension_separator'] = dimension_separator - - return json_dumps(meta) - - -def encode_dtype(d: np.dtype): - if d.fields is None: - return d.str - else: - return d.descr - - -def _decode_dtype_descr(d) -> List[Any]: - # need to convert list of lists to list of tuples - if isinstance(d, list): - # recurse to handle nested structures - d = [(k[0], _decode_dtype_descr(k[1])) + tuple(k[2:]) for k in d] - return d - - -def decode_dtype(d) -> np.dtype: - d = _decode_dtype_descr(d) - return np.dtype(d) - - -def decode_group_metadata(s: Union[MappingType, str]) -> MappingType[str, Any]: - meta = parse_metadata(s) - - # check metadata format version - zarr_format = meta.get('zarr_format', None) - if zarr_format != ZARR_FORMAT: - raise MetadataError('unsupported zarr format: %s' % zarr_format) - - meta = dict(zarr_format=zarr_format) - return meta - - -# N.B., keep `meta` parameter as a placeholder for future -# noinspection PyUnusedLocal -def encode_group_metadata(meta=None) -> bytes: - meta = dict( - zarr_format=ZARR_FORMAT, - ) - return json_dumps(meta) - - -FLOAT_FILLS = { - 'NaN': np.nan, - 'Infinity': np.PINF, - '-Infinity': np.NINF -} - - -def decode_fill_value(v, dtype, object_codec=None): - # early out - if v is None: - return v - if dtype.kind == 'V' and dtype.hasobject: - if object_codec is None: - raise ValueError('missing object_codec for object array') - v = base64.standard_b64decode(v) - v = object_codec.decode(v) - v = np.array(v, dtype=dtype)[()] - return v - if dtype.kind == 'f': - if v == 'NaN': - return np.nan - elif v == 'Infinity': - return np.PINF - elif v == '-Infinity': - return np.NINF - else: - return np.array(v, dtype=dtype)[()] - elif dtype.kind in 'c': - v = (decode_fill_value(v[0], dtype.type().real.dtype), - decode_fill_value(v[1], dtype.type().imag.dtype)) - v = v[0] + 1j * v[1] - return np.array(v, dtype=dtype)[()] - elif dtype.kind == 'S': - # noinspection PyBroadException - try: - v = base64.standard_b64decode(v) - except Exception: - # be lenient, allow for other values that may have been used before base64 - # encoding and may work as fill values, e.g., the number 0 - pass - v = np.array(v, dtype=dtype)[()] - return v - elif dtype.kind == 'V': - v = base64.standard_b64decode(v) - v = np.array(v, dtype=dtype.str).view(dtype)[()] - return v - elif dtype.kind == 'U': - # leave as-is - return v - else: - return np.array(v, dtype=dtype)[()] - - -def encode_fill_value(v: Any, dtype: np.dtype, object_codec: Any = None) -> Any: - # early out - if v is None: - return v - if dtype.kind == 'V' and dtype.hasobject: - if object_codec is None: - raise ValueError('missing object_codec for object array') - v = object_codec.encode(v) - v = str(base64.standard_b64encode(v), 'ascii') - return v - if dtype.kind == 'f': - if np.isnan(v): - return 'NaN' - elif np.isposinf(v): - return 'Infinity' - elif np.isneginf(v): - return '-Infinity' - else: - return float(v) - elif dtype.kind in 'ui': - return int(v) - elif dtype.kind == 'b': - return bool(v) - elif dtype.kind in 'c': - c = cast(np.complex128, np.dtype(complex).type()) - v = (encode_fill_value(v.real, c.real.dtype, object_codec), - encode_fill_value(v.imag, c.imag.dtype, object_codec)) - return v - elif dtype.kind in 'SV': - v = str(base64.standard_b64encode(v), 'ascii') - return v - elif dtype.kind == 'U': - return v - elif dtype.kind in 'mM': - return int(v.view('i8')) - else: - return v diff --git a/zarr/meta_v1.py b/zarr/meta_v1.py deleted file mode 100644 index bc4ae12228..0000000000 --- a/zarr/meta_v1.py +++ /dev/null @@ -1,64 +0,0 @@ -import json - -import numpy as np - -from zarr.errors import MetadataError - - -def decode_metadata(b): - s = str(b, 'ascii') - meta = json.loads(s) - zarr_format = meta.get('zarr_format', None) - if zarr_format != 1: - raise MetadataError('unsupported zarr format: %s' % zarr_format) - try: - meta = dict( - zarr_format=meta['zarr_format'], - shape=tuple(meta['shape']), - chunks=tuple(meta['chunks']), - dtype=decode_dtype(meta['dtype']), - compression=meta['compression'], - compression_opts=meta['compression_opts'], - fill_value=meta['fill_value'], - order=meta['order'], - ) - except Exception as e: - raise MetadataError('error decoding metadata: %s' % e) - else: - return meta - - -def encode_metadata(meta): - meta = dict( - zarr_format=1, - shape=meta['shape'], - chunks=meta['chunks'], - dtype=encode_dtype(meta['dtype']), - compression=meta['compression'], - compression_opts=meta['compression_opts'], - fill_value=meta['fill_value'], - order=meta['order'], - ) - s = json.dumps(meta, indent=4, sort_keys=True, ensure_ascii=True) - b = s.encode('ascii') - return b - - -def encode_dtype(d): - if d.fields is None: - return d.str - else: - return d.descr - - -def _decode_dtype_descr(d): - # need to convert list of lists to list of tuples - if isinstance(d, list): - # recurse to handle nested structures - d = [(f, _decode_dtype_descr(v)) for f, v in d] - return d - - -def decode_dtype(d): - d = _decode_dtype_descr(d) - return np.dtype(d) diff --git a/zarr/n5.py b/zarr/n5.py deleted file mode 100644 index 45e2cdda95..0000000000 --- a/zarr/n5.py +++ /dev/null @@ -1,654 +0,0 @@ -"""This module contains a storage class and codec to support the N5 format. -""" -import os -import struct -import sys -import warnings - -import numpy as np -from numcodecs.abc import Codec -from numcodecs.compat import ndarray_copy -from numcodecs.registry import get_codec, register_codec - -from .meta import ZARR_FORMAT, json_dumps, json_loads -from .storage import NestedDirectoryStore, _prog_ckey, _prog_number -from .storage import array_meta_key as zarr_array_meta_key -from .storage import attrs_key as zarr_attrs_key -from .storage import group_meta_key as zarr_group_meta_key - -zarr_to_n5_keys = [ - ('chunks', 'blockSize'), - ('dtype', 'dataType'), - ('compressor', 'compression'), - ('shape', 'dimensions') -] -n5_attrs_key = 'attributes.json' -n5_keywords = ['n5', 'dataType', 'dimensions', 'blockSize', 'compression'] - - -class N5Store(NestedDirectoryStore): - """Storage class using directories and files on a standard file system, - following the N5 format (https://github.com/saalfeldlab/n5). - - Parameters - ---------- - path : string - Location of directory to use as the root of the storage hierarchy. - normalize_keys : bool, optional - If True, all store keys will be normalized to use lower case characters - (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be - useful to avoid potential discrepancies between case-senstive and - case-insensitive file system. Default value is False. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.N5Store('data/array.n5') - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - - Store a group:: - - >>> store = zarr.N5Store('data/group.n5') - >>> root = zarr.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - - Notes - ----- - - This is an experimental feature. - - Safe to write in multiple threads or processes. - - """ - - def __getitem__(self, key): - - if key.endswith(zarr_group_meta_key): - - key = key.replace(zarr_group_meta_key, n5_attrs_key) - value = group_metadata_to_zarr(self._load_n5_attrs(key)) - - return json_dumps(value) - - elif key.endswith(zarr_array_meta_key): - - key = key.replace(zarr_array_meta_key, n5_attrs_key) - value = array_metadata_to_zarr(self._load_n5_attrs(key)) - - return json_dumps(value) - - elif key.endswith(zarr_attrs_key): - - key = key.replace(zarr_attrs_key, n5_attrs_key) - value = attrs_to_zarr(self._load_n5_attrs(key)) - - if len(value) == 0: - raise KeyError(key) - else: - return json_dumps(value) - - elif is_chunk_key(key): - - key = invert_chunk_coords(key) - - return super().__getitem__(key) - - def __setitem__(self, key, value): - - if key.endswith(zarr_group_meta_key): - - key = key.replace(zarr_group_meta_key, n5_attrs_key) - - n5_attrs = self._load_n5_attrs(key) - n5_attrs.update(**group_metadata_to_n5(json_loads(value))) - - value = json_dumps(n5_attrs) - - elif key.endswith(zarr_array_meta_key): - - key = key.replace(zarr_array_meta_key, n5_attrs_key) - - n5_attrs = self._load_n5_attrs(key) - n5_attrs.update(**array_metadata_to_n5(json_loads(value))) - - value = json_dumps(n5_attrs) - - elif key.endswith(zarr_attrs_key): - - key = key.replace(zarr_attrs_key, n5_attrs_key) - - n5_attrs = self._load_n5_attrs(key) - zarr_attrs = json_loads(value) - - for k in n5_keywords: - if k in zarr_attrs.keys(): - raise ValueError("Can not set attribute %s, this is a reserved N5 keyword" % k) - - # replace previous user attributes - for k in list(n5_attrs.keys()): - if k not in n5_keywords: - del n5_attrs[k] - - # add new user attributes - n5_attrs.update(**zarr_attrs) - - value = json_dumps(n5_attrs) - - elif is_chunk_key(key): - - key = invert_chunk_coords(key) - - super().__setitem__(key, value) - - def __delitem__(self, key): - - if key.endswith(zarr_group_meta_key): # pragma: no cover - key = key.replace(zarr_group_meta_key, n5_attrs_key) - elif key.endswith(zarr_array_meta_key): # pragma: no cover - key = key.replace(zarr_array_meta_key, n5_attrs_key) - elif key.endswith(zarr_attrs_key): # pragma: no cover - key = key.replace(zarr_attrs_key, n5_attrs_key) - elif is_chunk_key(key): - key = invert_chunk_coords(key) - - super().__delitem__(key) - - def __contains__(self, key): - - if key.endswith(zarr_group_meta_key): - - key = key.replace(zarr_group_meta_key, n5_attrs_key) - if key not in self: - return False - # group if not a dataset (attributes do not contain 'dimensions') - return 'dimensions' not in self._load_n5_attrs(key) - - elif key.endswith(zarr_array_meta_key): - - key = key.replace(zarr_array_meta_key, n5_attrs_key) - # array if attributes contain 'dimensions' - return 'dimensions' in self._load_n5_attrs(key) - - elif key.endswith(zarr_attrs_key): - - key = key.replace(zarr_attrs_key, n5_attrs_key) - return self._contains_attrs(key) - - elif is_chunk_key(key): - - key = invert_chunk_coords(key) - - return super().__contains__(key) - - def __eq__(self, other): - return ( - isinstance(other, N5Store) and - self.path == other.path - ) - - def listdir(self, path=None): - - if path is not None: - path = invert_chunk_coords(path) - - # We can't use NestedDirectoryStore's listdir, as it requires - # array_meta_key to be present in array directories, which this store - # doesn't provide. - children = super().listdir(path=path) - - if self._is_array(path): - - # replace n5 attribute file with respective zarr attribute files - children.remove(n5_attrs_key) - children.append(zarr_array_meta_key) - if self._contains_attrs(path): - children.append(zarr_attrs_key) - - # special handling of directories containing an array to map - # inverted nested chunk keys back to standard chunk keys - new_children = [] - root_path = self.dir_path(path) - for entry in children: - entry_path = os.path.join(root_path, entry) - if _prog_number.match(entry) and os.path.isdir(entry_path): - for dir_path, _, file_names in os.walk(entry_path): - for file_name in file_names: - file_path = os.path.join(dir_path, file_name) - rel_path = file_path.split(root_path + os.path.sep)[1] - new_child = rel_path.replace(os.path.sep, '.') - new_children.append(invert_chunk_coords(new_child)) - else: - new_children.append(entry) - - return sorted(new_children) - - elif self._is_group(path): - - # replace n5 attribute file with respective zarr attribute files - children.remove(n5_attrs_key) - children.append(zarr_group_meta_key) - if self._contains_attrs(path): # pragma: no cover - children.append(zarr_attrs_key) - - return sorted(children) - - else: - - return children - - def _load_n5_attrs(self, path): - try: - s = super().__getitem__(path) - return json_loads(s) - except KeyError: - return {} - - def _is_group(self, path): - - if path is None: - attrs_key = n5_attrs_key - else: - attrs_key = os.path.join(path, n5_attrs_key) - - n5_attrs = self._load_n5_attrs(attrs_key) - return len(n5_attrs) > 0 and 'dimensions' not in n5_attrs - - def _is_array(self, path): - - if path is None: - attrs_key = n5_attrs_key - else: - attrs_key = os.path.join(path, n5_attrs_key) - - return 'dimensions' in self._load_n5_attrs(attrs_key) - - def _contains_attrs(self, path): - - if path is None: - attrs_key = n5_attrs_key - else: - if not path.endswith(n5_attrs_key): - attrs_key = os.path.join(path, n5_attrs_key) - else: # pragma: no cover - attrs_key = path - - attrs = attrs_to_zarr(self._load_n5_attrs(attrs_key)) - return len(attrs) > 0 - - -def is_chunk_key(key): - segments = list(key.split('/')) - if segments: - last_segment = segments[-1] - return _prog_ckey.match(last_segment) - return False # pragma: no cover - - -def invert_chunk_coords(key): - segments = list(key.split('/')) - if segments: - last_segment = segments[-1] - if _prog_ckey.match(last_segment): - coords = list(last_segment.split('.')) - last_segment = '/'.join(coords[::-1]) - segments = segments[:-1] + [last_segment] - key = '/'.join(segments) - return key - - -def group_metadata_to_n5(group_metadata): - '''Convert group metadata from zarr to N5 format.''' - del group_metadata['zarr_format'] - # TODO: This should only exist at the top-level - group_metadata['n5'] = '2.0.0' - return group_metadata - - -def group_metadata_to_zarr(group_metadata): - '''Convert group metadata from N5 to zarr format.''' - # This only exists at the top level - group_metadata.pop('n5', None) - group_metadata['zarr_format'] = ZARR_FORMAT - return group_metadata - - -def array_metadata_to_n5(array_metadata): - '''Convert array metadata from zarr to N5 format.''' - - for f, t in zarr_to_n5_keys: - array_metadata[t] = array_metadata[f] - del array_metadata[f] - del array_metadata['zarr_format'] - - try: - dtype = np.dtype(array_metadata['dataType']) - except TypeError: # pragma: no cover - raise TypeError( - "data type %s not supported by N5" % array_metadata['dataType']) - - array_metadata['dataType'] = dtype.name - array_metadata['dimensions'] = array_metadata['dimensions'][::-1] - array_metadata['blockSize'] = array_metadata['blockSize'][::-1] - - if 'fill_value' in array_metadata: - if array_metadata['fill_value'] != 0 and array_metadata['fill_value'] is not None: - raise ValueError("N5 only supports fill_value == 0 (for now)") - del array_metadata['fill_value'] - - if 'order' in array_metadata: - if array_metadata['order'] != 'C': - raise ValueError("zarr N5 storage only stores arrays in C order (for now)") - del array_metadata['order'] - - if 'filters' in array_metadata: - if array_metadata['filters'] != [] and array_metadata['filters'] is not None: - raise ValueError("N5 storage does not support zarr filters") - del array_metadata['filters'] - - assert 'compression' in array_metadata - compressor_config = array_metadata['compression'] - compressor_config = compressor_config_to_n5(compressor_config) - array_metadata['compression'] = compressor_config - - if 'dimension_separator' in array_metadata: - del array_metadata['dimension_separator'] - - return array_metadata - - -def array_metadata_to_zarr(array_metadata): - '''Convert array metadata from N5 to zarr format.''' - for t, f in zarr_to_n5_keys: - array_metadata[t] = array_metadata[f] - del array_metadata[f] - array_metadata['zarr_format'] = ZARR_FORMAT - - array_metadata['shape'] = array_metadata['shape'][::-1] - array_metadata['chunks'] = array_metadata['chunks'][::-1] - array_metadata['fill_value'] = 0 # also if None was requested - array_metadata['order'] = 'C' - array_metadata['filters'] = [] - - compressor_config = array_metadata['compressor'] - compressor_config = compressor_config_to_zarr(compressor_config) - array_metadata['compressor'] = { - 'id': N5ChunkWrapper.codec_id, - 'compressor_config': compressor_config, - 'dtype': array_metadata['dtype'], - 'chunk_shape': array_metadata['chunks'] - } - - return array_metadata - - -def attrs_to_zarr(attrs): - '''Get all zarr attributes from an N5 attributes dictionary (i.e., - all non-keyword attributes).''' - - # remove all N5 keywords - for n5_key in n5_keywords: - if n5_key in attrs: - del attrs[n5_key] - - return attrs - - -def compressor_config_to_n5(compressor_config): - - if compressor_config is None: - return {'type': 'raw'} - - # peel wrapper, if present - if compressor_config['id'] == N5ChunkWrapper.codec_id: - compressor_config = compressor_config['compressor_config'] - - codec_id = compressor_config['id'] - n5_config = {'type': codec_id} - - if codec_id == 'bz2': - - n5_config['type'] = 'bzip2' - n5_config['blockSize'] = compressor_config['level'] - - elif codec_id == 'blosc': - - warnings.warn( - "Not all N5 implementations support blosc compression (yet). You " - "might not be able to open the dataset with another N5 library.", - RuntimeWarning - ) - - n5_config['cname'] = compressor_config['cname'] - n5_config['clevel'] = compressor_config['clevel'] - n5_config['shuffle'] = compressor_config['shuffle'] - n5_config['blocksize'] = compressor_config['blocksize'] - - elif codec_id == 'lzma': - - # Switch to XZ for N5 if we are using the default XZ format. - # Note: 4 is the default, which is lzma.CHECK_CRC64. - if compressor_config['format'] == 1 and compressor_config['check'] in [-1, 4]: - n5_config['type'] = 'xz' - else: - warnings.warn( - "Not all N5 implementations support lzma compression (yet). You " - "might not be able to open the dataset with another N5 library.", - RuntimeWarning - ) - n5_config['format'] = compressor_config['format'] - n5_config['check'] = compressor_config['check'] - n5_config['filters'] = compressor_config['filters'] - - # The default is lzma.PRESET_DEFAULT, which is 6. - if compressor_config['preset']: - n5_config['preset'] = compressor_config['preset'] - else: - n5_config['preset'] = 6 - - elif codec_id == 'zlib': - - n5_config['type'] = 'gzip' - n5_config['level'] = compressor_config['level'] - n5_config['useZlib'] = True - - elif codec_id == 'gzip': # pragma: no cover - - n5_config['type'] = 'gzip' - n5_config['level'] = compressor_config['level'] - n5_config['useZlib'] = False - - else: # pragma: no cover - - n5_config.update({k: v for k, v in compressor_config.items() if k != 'type'}) - - return n5_config - - -def compressor_config_to_zarr(compressor_config): - - codec_id = compressor_config['type'] - zarr_config = {'id': codec_id} - - if codec_id == 'bzip2': - - zarr_config['id'] = 'bz2' - zarr_config['level'] = compressor_config['blockSize'] - - elif codec_id == 'blosc': - - zarr_config['cname'] = compressor_config['cname'] - zarr_config['clevel'] = compressor_config['clevel'] - zarr_config['shuffle'] = compressor_config['shuffle'] - zarr_config['blocksize'] = compressor_config['blocksize'] - - elif codec_id == 'lzma': - - zarr_config['format'] = compressor_config['format'] - zarr_config['check'] = compressor_config['check'] - zarr_config['preset'] = compressor_config['preset'] - zarr_config['filters'] = compressor_config['filters'] - - elif codec_id == 'xz': - - zarr_config['id'] = 'lzma' - zarr_config['format'] = 1 # lzma.FORMAT_XZ - zarr_config['check'] = -1 - zarr_config['preset'] = compressor_config['preset'] - zarr_config['filters'] = None - - elif codec_id == 'gzip': - - if 'useZlib' in compressor_config and compressor_config['useZlib']: - zarr_config['id'] = 'zlib' - zarr_config['level'] = compressor_config['level'] - else: # pragma: no cover - zarr_config['id'] = 'gzip' - zarr_config['level'] = compressor_config['level'] - - elif codec_id == 'raw': - - return None - - else: # pragma: no cover - - zarr_config.update({k: v for k, v in compressor_config.items() if k != 'type'}) - - return zarr_config - - -class N5ChunkWrapper(Codec): - - codec_id = 'n5_wrapper' - - def __init__(self, dtype, chunk_shape, compressor_config=None, compressor=None): - - self.dtype = np.dtype(dtype) - self.chunk_shape = tuple(chunk_shape) - # is the dtype a little endian format? - self._little_endian = ( - self.dtype.byteorder == '<' or - (self.dtype.byteorder == '=' and sys.byteorder == 'little') - ) - - if compressor: # pragma: no cover - if compressor_config is not None: - raise ValueError("Only one of compressor_config or compressor should be given.") - compressor_config = compressor.get_config() - - if ( - compressor_config is None and compressor is None or - compressor_config['id'] == 'raw'): - self.compressor_config = None - self._compressor = None - else: - self._compressor = get_codec(compressor_config) - self.compressor_config = self._compressor.get_config() - - def get_config(self): - config = { - 'id': self.codec_id, - 'compressor_config': self.compressor_config - } - return config - - def encode(self, chunk): - - assert chunk.flags.c_contiguous - - header = self._create_header(chunk) - chunk = self._to_big_endian(chunk) - - if self._compressor: - return header + self._compressor.encode(chunk) - else: - return header + chunk.tobytes(order='A') - - def decode(self, chunk, out=None): - - len_header, chunk_shape = self._read_header(chunk) - chunk = chunk[len_header:] - - if out is not None: - - # out should only be used if we read a complete chunk - assert chunk_shape == self.chunk_shape, ( - "Expected chunk of shape {}, found {}".format( - self.chunk_shape, - chunk_shape)) - - if self._compressor: - self._compressor.decode(chunk, out) - else: - ndarray_copy(chunk, out) - - # we can byteswap in-place - if self._little_endian: - out.byteswap(True) - - return out - - else: - - if self._compressor: - chunk = self._compressor.decode(chunk) - - # more expensive byteswap - chunk = self._from_big_endian(chunk) - - # read partial chunk - if chunk_shape != self.chunk_shape: # pragma: no cover - chunk = np.frombuffer(chunk, dtype=self.dtype) - chunk = chunk.reshape(chunk_shape) - complete_chunk = np.zeros(self.chunk_shape, dtype=self.dtype) - target_slices = tuple(slice(0, s) for s in chunk_shape) - complete_chunk[target_slices] = chunk - chunk = complete_chunk - - return chunk - - def _create_header(self, chunk): - - mode = struct.pack('>H', 0) - num_dims = struct.pack('>H', len(chunk.shape)) - shape = b''.join( - struct.pack('>I', d) - for d in chunk.shape[::-1] - ) - - return mode + num_dims + shape - - def _read_header(self, chunk): - - num_dims = struct.unpack('>H', chunk[2:4])[0] - shape = tuple( - struct.unpack('>I', chunk[i:i+4])[0] - for i in range(4, num_dims*4 + 4, 4) - )[::-1] - - len_header = 4 + num_dims*4 - - return len_header, shape - - def _to_big_endian(self, data): - # assumes data is ndarray - - if self._little_endian: - return data.byteswap() - return data - - def _from_big_endian(self, data): - # assumes data is byte array in big endian - - if not self._little_endian: - return data - - a = np.frombuffer(data, self.dtype.newbyteorder('>')) - return a.astype(self.dtype) - - -register_codec(N5ChunkWrapper, N5ChunkWrapper.codec_id) diff --git a/zarr/storage.py b/zarr/storage.py deleted file mode 100644 index 6ca6271dbf..0000000000 --- a/zarr/storage.py +++ /dev/null @@ -1,2618 +0,0 @@ -"""This module contains storage classes for use with Zarr arrays and groups. - -Note that any object implementing the :class:`MutableMapping` interface from the -:mod:`collections` module in the Python standard library can be used as a Zarr -array store, as long as it accepts string (str) keys and bytes values. - -In addition to the :class:`MutableMapping` interface, store classes may also implement -optional methods `listdir` (list members of a "directory") and `rmdir` (remove all -members of a "directory"). These methods should be implemented if the store class is -aware of the hierarchical organisation of resources within the store and can provide -efficient implementations. If these methods are not available, Zarr will fall back to -slower implementations that work via the :class:`MutableMapping` interface. Store -classes may also optionally implement a `rename` method (rename all members under a given -path) and a `getsize` method (return the size in bytes of a given value). - -""" -import atexit -import errno -import glob -import multiprocessing -import operator -import os -import re -import shutil -import sys -import tempfile -import warnings -import zipfile -from collections import OrderedDict -from collections.abc import MutableMapping -from os import scandir -from pickle import PicklingError -from threading import Lock, RLock -from typing import Optional, Union, List, Tuple, Dict -import uuid -import time - -from numcodecs.compat import ( - ensure_bytes, - ensure_text, - ensure_contiguous_ndarray -) -from numcodecs.registry import codec_registry - -from zarr.errors import ( - MetadataError, - BadCompressorError, - ContainsArrayError, - ContainsGroupError, - FSPathExistNotDir, - ReadOnlyError, -) -from zarr.meta import encode_array_metadata, encode_group_metadata -from zarr.util import (buffer_size, json_loads, nolock, normalize_chunks, - normalize_dimension_separator, - normalize_dtype, normalize_fill_value, normalize_order, - normalize_shape, normalize_storage_path, retry_call) - -from zarr._storage.absstore import ABSStore # noqa: F401 - -__doctest_requires__ = { - ('RedisStore', 'RedisStore.*'): ['redis'], - ('MongoDBStore', 'MongoDBStore.*'): ['pymongo'], - ('LRUStoreCache', 'LRUStoreCache.*'): ['s3fs'], -} - - -array_meta_key = '.zarray' -group_meta_key = '.zgroup' -attrs_key = '.zattrs' -try: - # noinspection PyUnresolvedReferences - from zarr.codecs import Blosc - default_compressor = Blosc() -except ImportError: # pragma: no cover - from zarr.codecs import Zlib - default_compressor = Zlib() - - -Path = Union[str, bytes, None] - - -def _path_to_prefix(path: Optional[str]) -> str: - # assume path already normalized - if path: - prefix = path + '/' - else: - prefix = '' - return prefix - - -def contains_array(store: MutableMapping, path: Path = None) -> bool: - """Return True if the store contains an array at the given logical path.""" - path = normalize_storage_path(path) - prefix = _path_to_prefix(path) - key = prefix + array_meta_key - return key in store - - -def contains_group(store: MutableMapping, path: Path = None) -> bool: - """Return True if the store contains a group at the given logical path.""" - path = normalize_storage_path(path) - prefix = _path_to_prefix(path) - key = prefix + group_meta_key - return key in store - - -def _rmdir_from_keys(store: MutableMapping, path: Optional[str] = None) -> None: - # assume path already normalized - prefix = _path_to_prefix(path) - for key in list(store.keys()): - if key.startswith(prefix): - del store[key] - - -def rmdir(store, path: Path = None): - """Remove all items under the given path. If `store` provides a `rmdir` method, - this will be called, otherwise will fall back to implementation via the - `MutableMapping` interface.""" - path = normalize_storage_path(path) - if hasattr(store, 'rmdir'): - # pass through - store.rmdir(path) - else: - # slow version, delete one key at a time - _rmdir_from_keys(store, path) - - -def _rename_from_keys(store: MutableMapping, src_path: str, dst_path: str) -> None: - # assume path already normalized - src_prefix = _path_to_prefix(src_path) - dst_prefix = _path_to_prefix(dst_path) - for key in list(store.keys()): - if key.startswith(src_prefix): - new_key = dst_prefix + key.lstrip(src_prefix) - store[new_key] = store.pop(key) - - -def rename(store, src_path: Path, dst_path: Path): - """Rename all items under the given path. If `store` provides a `rename` method, - this will be called, otherwise will fall back to implementation via the - `MutableMapping` interface.""" - src_path = normalize_storage_path(src_path) - dst_path = normalize_storage_path(dst_path) - if hasattr(store, 'rename'): - # pass through - store.rename(src_path, dst_path) - else: - # slow version, delete one key at a time - _rename_from_keys(store, src_path, dst_path) - - -def _listdir_from_keys(store: MutableMapping, path: Optional[str] = None) -> List[str]: - # assume path already normalized - prefix = _path_to_prefix(path) - children = set() - for key in list(store.keys()): - if key.startswith(prefix) and len(key) > len(prefix): - suffix = key[len(prefix):] - child = suffix.split('/')[0] - children.add(child) - return sorted(children) - - -def listdir(store, path: Path = None): - """Obtain a directory listing for the given path. If `store` provides a `listdir` - method, this will be called, otherwise will fall back to implementation via the - `MutableMapping` interface.""" - path = normalize_storage_path(path) - if hasattr(store, 'listdir'): - # pass through - return store.listdir(path) - else: - # slow version, iterate through all keys - return _listdir_from_keys(store, path) - - -def getsize(store, path: Path = None) -> int: - """Compute size of stored items for a given path. If `store` provides a `getsize` - method, this will be called, otherwise will return -1.""" - path = normalize_storage_path(path) - if hasattr(store, 'getsize'): - # pass through - return store.getsize(path) - elif isinstance(store, dict): - # compute from size of values - if path in store: - v = store[path] - size = buffer_size(v) - else: - members = listdir(store, path) - prefix = _path_to_prefix(path) - size = 0 - for k in members: - try: - v = store[prefix + k] - except KeyError: - pass - else: - try: - size += buffer_size(v) - except TypeError: - return -1 - return size - else: - return -1 - - -def _require_parent_group( - path: Optional[str], - store: MutableMapping, - chunk_store: Optional[MutableMapping], - overwrite: bool, -): - # assume path is normalized - if path: - segments = path.split('/') - for i in range(len(segments)): - p = '/'.join(segments[:i]) - if contains_array(store, p): - _init_group_metadata(store, path=p, chunk_store=chunk_store, - overwrite=overwrite) - elif not contains_group(store, p): - _init_group_metadata(store, path=p, chunk_store=chunk_store) - - -def init_array( - store: MutableMapping, - shape: Tuple[int, ...], - chunks: Union[bool, int, Tuple[int, ...]] = True, - dtype=None, - compressor="default", - fill_value=None, - order: str = "C", - overwrite: bool = False, - path: Path = None, - chunk_store: MutableMapping = None, - filters=None, - object_codec=None, - dimension_separator=None, -): - """Initialize an array store with the given configuration. Note that this is a low-level - function and there should be no need to call this directly from user code. - - Parameters - ---------- - store : MutableMapping - A mapping that supports string keys and bytes-like values. - shape : int or tuple of ints - Array shape. - chunks : bool, int or tuple of ints, optional - Chunk shape. If True, will be guessed from `shape` and `dtype`. If - False, will be set to `shape`, i.e., single chunk for the whole array. - dtype : string or dtype, optional - NumPy dtype. - compressor : Codec, optional - Primary compressor. - fill_value : object - Default value to use for uninitialized portions of the array. - order : {'C', 'F'}, optional - Memory layout to be used within each chunk. - overwrite : bool, optional - If True, erase all data in `store` prior to initialisation. - path : string, bytes, optional - Path under which array is stored. - chunk_store : MutableMapping, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - filters : sequence, optional - Sequence of filters to use to encode chunk data prior to compression. - object_codec : Codec, optional - A codec to encode object arrays, only needed if dtype=object. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - - Examples - -------- - Initialize an array store:: - - >>> from zarr.storage import init_array - >>> store = dict() - >>> init_array(store, shape=(10000, 10000), chunks=(1000, 1000)) - >>> sorted(store.keys()) - ['.zarray'] - - Array metadata is stored as JSON:: - - >>> print(store['.zarray'].decode()) - { - "chunks": [ - 1000, - 1000 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": ">> store = dict() - >>> init_array(store, shape=100000000, chunks=1000000, dtype='i1', path='foo') - >>> sorted(store.keys()) - ['.zgroup', 'foo/.zarray'] - >>> print(store['foo/.zarray'].decode()) - { - "chunks": [ - 1000000 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "|i1", - "fill_value": null, - "filters": null, - "order": "C", - "shape": [ - 100000000 - ], - "zarr_format": 2 - } - - Notes - ----- - The initialisation process involves normalising all array metadata, encoding - as JSON and storing under the '.zarray' key. - - """ - - # normalize path - path = normalize_storage_path(path) - - # ensure parent group initialized - _require_parent_group(path, store=store, chunk_store=chunk_store, overwrite=overwrite) - - _init_array_metadata(store, shape=shape, chunks=chunks, dtype=dtype, - compressor=compressor, fill_value=fill_value, - order=order, overwrite=overwrite, path=path, - chunk_store=chunk_store, filters=filters, - object_codec=object_codec, - dimension_separator=dimension_separator) - - -def _init_array_metadata( - store, - shape, - chunks=None, - dtype=None, - compressor="default", - fill_value=None, - order="C", - overwrite=False, - path: Optional[str] = None, - chunk_store=None, - filters=None, - object_codec=None, - dimension_separator=None, -): - - # guard conditions - if overwrite: - # attempt to delete any pre-existing items in store - rmdir(store, path) - if chunk_store is not None: - rmdir(chunk_store, path) - elif contains_array(store, path): - raise ContainsArrayError(path) - elif contains_group(store, path): - raise ContainsGroupError(path) - - # normalize metadata - dtype, object_codec = normalize_dtype(dtype, object_codec) - shape = normalize_shape(shape) + dtype.shape - dtype = dtype.base - chunks = normalize_chunks(chunks, shape, dtype.itemsize) - order = normalize_order(order) - fill_value = normalize_fill_value(fill_value, dtype) - - # optional array metadata - if dimension_separator is None: - dimension_separator = getattr(store, "_dimension_separator", None) - dimension_separator = normalize_dimension_separator(dimension_separator) - - # compressor prep - if shape == (): - # no point in compressing a 0-dimensional array, only a single value - compressor = None - elif compressor == 'none': - # compatibility - compressor = None - elif compressor == 'default': - compressor = default_compressor - - # obtain compressor config - compressor_config = None - if compressor: - try: - compressor_config = compressor.get_config() - except AttributeError as e: - raise BadCompressorError(compressor) from e - - # obtain filters config - if filters: - filters_config = [f.get_config() for f in filters] - else: - filters_config = [] - - # deal with object encoding - if dtype.hasobject: - if object_codec is None: - if not filters: - # there are no filters so we can be sure there is no object codec - raise ValueError('missing object_codec for object array') - else: - # one of the filters may be an object codec, issue a warning rather - # than raise an error to maintain backwards-compatibility - warnings.warn('missing object_codec for object array; this will raise a ' - 'ValueError in version 3.0', FutureWarning) - else: - filters_config.insert(0, object_codec.get_config()) - elif object_codec is not None: - warnings.warn('an object_codec is only needed for object arrays') - - # use null to indicate no filters - if not filters_config: - filters_config = None # type: ignore - - # initialize metadata - meta = dict(shape=shape, chunks=chunks, dtype=dtype, - compressor=compressor_config, fill_value=fill_value, - order=order, filters=filters_config, - dimension_separator=dimension_separator) - key = _path_to_prefix(path) + array_meta_key - store[key] = encode_array_metadata(meta) - - -# backwards compatibility -init_store = init_array - - -def init_group( - store: MutableMapping, - overwrite: bool = False, - path: Path = None, - chunk_store: MutableMapping = None, -): - """Initialize a group store. Note that this is a low-level function and there should be no - need to call this directly from user code. - - Parameters - ---------- - store : MutableMapping - A mapping that supports string keys and byte sequence values. - overwrite : bool, optional - If True, erase all data in `store` prior to initialisation. - path : string, optional - Path under which array is stored. - chunk_store : MutableMapping, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - - """ - - # normalize path - path = normalize_storage_path(path) - - # ensure parent group initialized - _require_parent_group(path, store=store, chunk_store=chunk_store, - overwrite=overwrite) - - # initialise metadata - _init_group_metadata(store=store, overwrite=overwrite, path=path, - chunk_store=chunk_store) - - -def _init_group_metadata( - store: MutableMapping, - overwrite: Optional[bool] = False, - path: Optional[str] = None, - chunk_store: MutableMapping = None, -): - - # guard conditions - if overwrite: - # attempt to delete any pre-existing items in store - rmdir(store, path) - if chunk_store is not None: - rmdir(chunk_store, path) - elif contains_array(store, path): - raise ContainsArrayError(path) - elif contains_group(store, path): - raise ContainsGroupError(path) - - # initialize metadata - # N.B., currently no metadata properties are needed, however there may - # be in future - meta = dict() # type: ignore - key = _path_to_prefix(path) + group_meta_key - store[key] = encode_group_metadata(meta) - - -def _dict_store_keys(d: Dict, prefix="", cls=dict): - for k in d.keys(): - v = d[k] - if isinstance(v, cls): - for sk in _dict_store_keys(v, prefix + k + '/', cls): - yield sk - else: - yield prefix + k - - -class MemoryStore(MutableMapping): - """Store class that uses a hierarchy of :class:`dict` objects, thus all data - will be held in main memory. - - Examples - -------- - This is the default class used when creating a group. E.g.:: - - >>> import zarr - >>> g = zarr.group() - >>> type(g.store) - - - Note that the default class when creating an array is the built-in - :class:`dict` class, i.e.:: - - >>> z = zarr.zeros(100) - >>> type(z.store) - - - Notes - ----- - Safe to write in multiple threads. - - """ - - def __init__(self, root=None, cls=dict, dimension_separator=None): - if root is None: - self.root = cls() - else: - self.root = root - self.cls = cls - self.write_mutex = Lock() - self._dimension_separator = dimension_separator - - def __getstate__(self): - return self.root, self.cls - - def __setstate__(self, state): - root, cls = state - self.__init__(root=root, cls=cls) - - def _get_parent(self, item: str): - parent = self.root - # split the item - segments = item.split('/') - # find the parent container - for k in segments[:-1]: - parent = parent[k] - if not isinstance(parent, self.cls): - raise KeyError(item) - return parent, segments[-1] - - def _require_parent(self, item): - parent = self.root - # split the item - segments = item.split('/') - # require the parent container - for k in segments[:-1]: - try: - parent = parent[k] - except KeyError: - parent[k] = self.cls() - parent = parent[k] - else: - if not isinstance(parent, self.cls): - raise KeyError(item) - return parent, segments[-1] - - def __getitem__(self, item: str): - parent, key = self._get_parent(item) - try: - value = parent[key] - except KeyError: - raise KeyError(item) - else: - if isinstance(value, self.cls): - raise KeyError(item) - else: - return value - - def __setitem__(self, item: str, value): - with self.write_mutex: - parent, key = self._require_parent(item) - value = ensure_bytes(value) - parent[key] = value - - def __delitem__(self, item: str): - with self.write_mutex: - parent, key = self._get_parent(item) - try: - del parent[key] - except KeyError: - raise KeyError(item) - - def __contains__(self, item: str): # type: ignore[override] - try: - parent, key = self._get_parent(item) - value = parent[key] - except KeyError: - return False - else: - return not isinstance(value, self.cls) - - def __eq__(self, other): - return ( - isinstance(other, MemoryStore) and - self.root == other.root and - self.cls == other.cls - ) - - def keys(self): - for k in _dict_store_keys(self.root, cls=self.cls): - yield k - - def __iter__(self): - return self.keys() - - def __len__(self) -> int: - return sum(1 for _ in self.keys()) - - def listdir(self, path: Path = None) -> List[str]: - path = normalize_storage_path(path) - if path: - try: - parent, key = self._get_parent(path) - value = parent[key] - except KeyError: - return [] - else: - value = self.root - if isinstance(value, self.cls): - return sorted(value.keys()) - else: - return [] - - def rename(self, src_path: Path, dst_path: Path): - src_path = normalize_storage_path(src_path) - dst_path = normalize_storage_path(dst_path) - - src_parent, src_key = self._get_parent(src_path) - dst_parent, dst_key = self._require_parent(dst_path) - - dst_parent[dst_key] = src_parent.pop(src_key) - - def rmdir(self, path: Path = None): - path = normalize_storage_path(path) - if path: - try: - parent, key = self._get_parent(path) - value = parent[key] - except KeyError: - return - else: - if isinstance(value, self.cls): - del parent[key] - else: - # clear out root - self.root = self.cls() - - def getsize(self, path: Path = None): - path = normalize_storage_path(path) - - # obtain value to return size of - value = None - if path: - try: - parent, key = self._get_parent(path) - value = parent[key] - except KeyError: - pass - else: - value = self.root - - # obtain size of value - if value is None: - return 0 - - elif isinstance(value, self.cls): - # total size for directory - size = 0 - for v in value.values(): - if not isinstance(v, self.cls): - size += buffer_size(v) - return size - - else: - return buffer_size(value) - - def clear(self): - with self.write_mutex: - self.root.clear() - - -class DictStore(MemoryStore): - - def __init__(self, *args, **kwargs): - warnings.warn("DictStore has been renamed to MemoryStore in 2.4.0 and " - "will be removed in the future. Please use MemoryStore.", - DeprecationWarning, - stacklevel=2) - super().__init__(*args, **kwargs) - - -class DirectoryStore(MutableMapping): - """Storage class using directories and files on a standard file system. - - Parameters - ---------- - path : string - Location of directory to use as the root of the storage hierarchy. - normalize_keys : bool, optional - If True, all store keys will be normalized to use lower case characters - (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be - useful to avoid potential discrepancies between case-senstive and - case-insensitive file system. Default value is False. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/array.zarr') - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - - Each chunk of the array is stored as a separate file on the file system, - i.e.:: - - >>> import os - >>> sorted(os.listdir('data/array.zarr')) - ['.zarray', '0.0', '0.1', '1.0', '1.1'] - - Store a group:: - - >>> store = zarr.DirectoryStore('data/group.zarr') - >>> root = zarr.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - - When storing a group, levels in the group hierarchy will correspond to - directories on the file system, i.e.:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup', 'bar'] - >>> sorted(os.listdir('data/group.zarr/foo/bar')) - ['.zarray', '0.0', '0.1', '1.0', '1.1'] - - Notes - ----- - Atomic writes are used, which means that data are first written to a - temporary file, then moved into place when the write is successfully - completed. Files are only held open while they are being read or written and are - closed immediately afterwards, so there is no need to manually close any files. - - Safe to write in multiple threads or processes. - - """ - - def __init__(self, path, normalize_keys=False, dimension_separator=None): - - # guard conditions - path = os.path.abspath(path) - if os.path.exists(path) and not os.path.isdir(path): - raise FSPathExistNotDir(path) - - self.path = path - self.normalize_keys = normalize_keys - self._dimension_separator = dimension_separator - - def _normalize_key(self, key): - return key.lower() if self.normalize_keys else key - - def _fromfile(self, fn): - """ Read data from a file - - Parameters - ---------- - fn : str - Filepath to open and read from. - - Notes - ----- - Subclasses should overload this method to specify any custom - file reading logic. - """ - with open(fn, 'rb') as f: - return f.read() - - def _tofile(self, a, fn): - """ Write data to a file - - Parameters - ---------- - a : array-like - Data to write into the file. - fn : str - Filepath to open and write to. - - Notes - ----- - Subclasses should overload this method to specify any custom - file writing logic. - """ - with open(fn, mode='wb') as f: - f.write(a) - - def __getitem__(self, key): - key = self._normalize_key(key) - filepath = os.path.join(self.path, key) - if os.path.isfile(filepath): - return self._fromfile(filepath) - else: - raise KeyError(key) - - def __setitem__(self, key, value): - key = self._normalize_key(key) - - # coerce to flat, contiguous array (ideally without copying) - value = ensure_contiguous_ndarray(value) - - # destination path for key - file_path = os.path.join(self.path, key) - - # ensure there is no directory in the way - if os.path.isdir(file_path): - shutil.rmtree(file_path) - - # ensure containing directory exists - dir_path, file_name = os.path.split(file_path) - if os.path.isfile(dir_path): - raise KeyError(key) - if not os.path.exists(dir_path): - try: - os.makedirs(dir_path) - except OSError as e: - if e.errno != errno.EEXIST: - raise KeyError(key) - - # write to temporary file - # note we're not using tempfile.NamedTemporaryFile to avoid restrictive file permissions - temp_name = file_name + '.' + uuid.uuid4().hex + '.partial' - temp_path = os.path.join(dir_path, temp_name) - try: - self._tofile(value, temp_path) - - # move temporary file into place; - # make several attempts at writing the temporary file to get past - # potential antivirus file locking issues - retry_call(os.replace, (temp_path, file_path), exceptions=(PermissionError,)) - - finally: - # clean up if temp file still exists for whatever reason - if os.path.exists(temp_path): # pragma: no cover - os.remove(temp_path) - - def __delitem__(self, key): - key = self._normalize_key(key) - path = os.path.join(self.path, key) - if os.path.isfile(path): - os.remove(path) - elif os.path.isdir(path): - # include support for deleting directories, even though strictly - # speaking these do not exist as keys in the store - shutil.rmtree(path) - else: - raise KeyError(key) - - def __contains__(self, key): - key = self._normalize_key(key) - file_path = os.path.join(self.path, key) - return os.path.isfile(file_path) - - def __eq__(self, other): - return ( - isinstance(other, DirectoryStore) and - self.path == other.path - ) - - def keys(self): - if os.path.exists(self.path): - yield from self._keys_fast(self.path) - - @staticmethod - def _keys_fast(path, walker=os.walk): - """ - - Faster logic on platform where the separator is `/` and using - `os.walk()` to decrease the number of stats.call. - - """ - it = iter(walker(path)) - d0, dirnames, filenames = next(it) - if d0.endswith('/'): - root_len = len(d0) - else: - root_len = len(d0)+1 - for f in filenames: - yield f - for dirpath, _, filenames in it: - for f in filenames: - yield dirpath[root_len:].replace('\\', '/')+'/'+f - - def __iter__(self): - return self.keys() - - def __len__(self): - return sum(1 for _ in self.keys()) - - def dir_path(self, path=None): - store_path = normalize_storage_path(path) - dir_path = self.path - if store_path: - dir_path = os.path.join(dir_path, store_path) - return dir_path - - def listdir(self, path=None): - return self._dimension_separator == "/" and \ - self._nested_listdir(path) or self._flat_listdir(path) - - def _flat_listdir(self, path=None): - dir_path = self.dir_path(path) - if os.path.isdir(dir_path): - return sorted(os.listdir(dir_path)) - else: - return [] - - def _nested_listdir(self, path=None): - children = self._flat_listdir(path=path) - if array_meta_key in children: - # special handling of directories containing an array to map nested chunk - # keys back to standard chunk keys - new_children = [] - root_path = self.dir_path(path) - for entry in children: - entry_path = os.path.join(root_path, entry) - if _prog_number.match(entry) and os.path.isdir(entry_path): - for dir_path, _, file_names in os.walk(entry_path): - for file_name in file_names: - file_path = os.path.join(dir_path, file_name) - rel_path = file_path.split(root_path + os.path.sep)[1] - new_children.append(rel_path.replace(os.path.sep, '.')) - else: - new_children.append(entry) - return sorted(new_children) - else: - return children - - def rename(self, src_path, dst_path): - store_src_path = normalize_storage_path(src_path) - store_dst_path = normalize_storage_path(dst_path) - - dir_path = self.path - - src_path = os.path.join(dir_path, store_src_path) - dst_path = os.path.join(dir_path, store_dst_path) - - os.renames(src_path, dst_path) - - def rmdir(self, path=None): - store_path = normalize_storage_path(path) - dir_path = self.path - if store_path: - dir_path = os.path.join(dir_path, store_path) - if os.path.isdir(dir_path): - shutil.rmtree(dir_path) - - def getsize(self, path=None): - store_path = normalize_storage_path(path) - fs_path = self.path - if store_path: - fs_path = os.path.join(fs_path, store_path) - if os.path.isfile(fs_path): - return os.path.getsize(fs_path) - elif os.path.isdir(fs_path): - size = 0 - for child in scandir(fs_path): - if child.is_file(): - size += child.stat().st_size - return size - else: - return 0 - - def clear(self): - shutil.rmtree(self.path) - - -def atexit_rmtree(path, - isdir=os.path.isdir, - rmtree=shutil.rmtree): # pragma: no cover - """Ensure directory removal at interpreter exit.""" - if isdir(path): - rmtree(path) - - -# noinspection PyShadowingNames -def atexit_rmglob(path, - glob=glob.glob, - isdir=os.path.isdir, - isfile=os.path.isfile, - remove=os.remove, - rmtree=shutil.rmtree): # pragma: no cover - """Ensure removal of multiple files at interpreter exit.""" - for p in glob(path): - if isfile(p): - remove(p) - elif isdir(p): - rmtree(p) - - -class FSStore(MutableMapping): - """Wraps an fsspec.FSMap to give access to arbitrary filesystems - - Requires that ``fsspec`` is installed, as well as any additional - requirements for the protocol chosen. - - Parameters - ---------- - url : str - The destination to map. Should include protocol and path, - like "s3://bucket/root" - normalize_keys : bool - key_separator : str - public API for accessing dimension_separator. Never `None` - See dimension_separator for more information. - mode : str - "w" for writable, "r" for read-only - exceptions : list of Exception subclasses - When accessing data, any of these exceptions will be treated - as a missing key - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - storage_options : passed to the fsspec implementation - """ - - _META_KEYS = (attrs_key, group_meta_key, array_meta_key) - - def __init__(self, url, normalize_keys=False, key_separator=None, - mode='w', - exceptions=(KeyError, PermissionError, IOError), - dimension_separator=None, - **storage_options): - import fsspec - self.normalize_keys = normalize_keys - self.map = fsspec.get_mapper(url, **storage_options) - self.fs = self.map.fs # for direct operations - self.path = self.fs._strip_protocol(url) - self.mode = mode - self.exceptions = exceptions - - # For backwards compatibility. Guaranteed to be non-None - if key_separator is not None: - dimension_separator = key_separator - - self.key_separator = dimension_separator - if self.key_separator is None: - self.key_separator = "." - - # Pass attributes to array creation - self._dimension_separator = dimension_separator - - if self.fs.exists(self.path) and not self.fs.isdir(self.path): - raise FSPathExistNotDir(url) - - def _normalize_key(self, key): - key = normalize_storage_path(key).lstrip('/') - if key: - *bits, end = key.split('/') - - if end not in FSStore._META_KEYS: - end = end.replace('.', self.key_separator) - key = '/'.join(bits + [end]) - - return key.lower() if self.normalize_keys else key - - def getitems(self, keys, **kwargs): - keys_transformed = [self._normalize_key(key) for key in keys] - results = self.map.getitems(keys_transformed, on_error="omit") - # The function calling this method may not recognize the transformed keys - # So we send the values returned by self.map.getitems back into the original key space. - return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()} - - def __getitem__(self, key): - key = self._normalize_key(key) - try: - return self.map[key] - except self.exceptions as e: - raise KeyError(key) from e - - def setitems(self, values): - if self.mode == 'r': - raise ReadOnlyError() - values = {self._normalize_key(key): val for key, val in values.items()} - self.map.setitems(values) - - def __setitem__(self, key, value): - if self.mode == 'r': - raise ReadOnlyError() - key = self._normalize_key(key) - path = self.dir_path(key) - try: - if self.fs.isdir(path): - self.fs.rm(path, recursive=True) - self.map[key] = value - self.fs.invalidate_cache(self.fs._parent(path)) - except self.exceptions as e: - raise KeyError(key) from e - - def __delitem__(self, key): - if self.mode == 'r': - raise ReadOnlyError() - key = self._normalize_key(key) - path = self.dir_path(key) - if self.fs.isdir(path): - self.fs.rm(path, recursive=True) - else: - del self.map[key] - - def __contains__(self, key): - key = self._normalize_key(key) - return key in self.map - - def __eq__(self, other): - return (type(self) == type(other) and self.map == other.map - and self.mode == other.mode) - - def keys(self): - return iter(self.map) - - def __iter__(self): - return self.keys() - - def __len__(self): - return len(list(self.keys())) - - def dir_path(self, path=None): - store_path = normalize_storage_path(path) - return self.map._key_to_str(store_path) - - def listdir(self, path=None): - dir_path = self.dir_path(path) - try: - children = sorted(p.rstrip('/').rsplit('/', 1)[-1] - for p in self.fs.ls(dir_path, detail=False)) - if self.key_separator != "/": - return children - else: - if array_meta_key in children: - # special handling of directories containing an array to map nested chunk - # keys back to standard chunk keys - new_children = [] - root_path = self.dir_path(path) - for entry in children: - entry_path = os.path.join(root_path, entry) - if _prog_number.match(entry) and self.fs.isdir(entry_path): - for file_name in self.fs.find(entry_path): - file_path = os.path.join(dir_path, file_name) - rel_path = file_path.split(root_path)[1] - rel_path = rel_path.lstrip('/') - new_children.append(rel_path.replace('/', '.')) - else: - new_children.append(entry) - return sorted(new_children) - else: - return children - except IOError: - return [] - - def rmdir(self, path=None): - if self.mode == 'r': - raise ReadOnlyError() - store_path = self.dir_path(path) - if self.fs.isdir(store_path): - self.fs.rm(store_path, recursive=True) - - def getsize(self, path=None): - store_path = self.dir_path(path) - return self.fs.du(store_path, True, True) - - def clear(self): - if self.mode == 'r': - raise ReadOnlyError() - self.map.clear() - - -class TempStore(DirectoryStore): - """Directory store using a temporary directory for storage. - - Parameters - ---------- - suffix : string, optional - Suffix for the temporary directory name. - prefix : string, optional - Prefix for the temporary directory name. - dir : string, optional - Path to parent directory in which to create temporary directory. - normalize_keys : bool, optional - If True, all store keys will be normalized to use lower case characters - (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be - useful to avoid potential discrepancies between case-senstive and - case-insensitive file system. Default value is False. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - """ - - # noinspection PyShadowingBuiltins - def __init__(self, suffix='', prefix='zarr', dir=None, normalize_keys=False, - dimension_separator=None): - path = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir) - atexit.register(atexit_rmtree, path) - super().__init__(path, normalize_keys=normalize_keys) - - -_prog_ckey = re.compile(r'^(\d+)(\.\d+)+$') -_prog_number = re.compile(r'^\d+$') - - -class NestedDirectoryStore(DirectoryStore): - """Storage class using directories and files on a standard file system, with - special handling for chunk keys so that chunk files for multidimensional - arrays are stored in a nested directory tree. - - Parameters - ---------- - path : string - Location of directory to use as the root of the storage hierarchy. - normalize_keys : bool, optional - If True, all store keys will be normalized to use lower case characters - (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be - useful to avoid potential discrepancies between case-senstive and - case-insensitive file system. Default value is False. - dimension_separator : {'/'}, optional - Separator placed between the dimensions of a chunk. - Only supports "/" unlike other implementations. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.NestedDirectoryStore('data/array.zarr') - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - - Each chunk of the array is stored as a separate file on the file system, - note the multiple directory levels used for the chunk files:: - - >>> import os - >>> sorted(os.listdir('data/array.zarr')) - ['.zarray', '0', '1'] - >>> sorted(os.listdir('data/array.zarr/0')) - ['0', '1'] - >>> sorted(os.listdir('data/array.zarr/1')) - ['0', '1'] - - Store a group:: - - >>> store = zarr.NestedDirectoryStore('data/group.zarr') - >>> root = zarr.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - - When storing a group, levels in the group hierarchy will correspond to - directories on the file system, i.e.:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup', 'bar'] - >>> sorted(os.listdir('data/group.zarr/foo/bar')) - ['.zarray', '0', '1'] - >>> sorted(os.listdir('data/group.zarr/foo/bar/0')) - ['0', '1'] - >>> sorted(os.listdir('data/group.zarr/foo/bar/1')) - ['0', '1'] - - Notes - ----- - The :class:`DirectoryStore` class stores all chunk files for an array - together in a single directory. On some file systems, the potentially large - number of files in a single directory can cause performance issues. The - :class:`NestedDirectoryStore` class provides an alternative where chunk - files for multidimensional arrays will be organised into a directory - hierarchy, thus reducing the number of files in any one directory. - - Safe to write in multiple threads or processes. - - """ - - def __init__(self, path, normalize_keys=False, dimension_separator="/"): - super().__init__(path, normalize_keys=normalize_keys) - if dimension_separator is None: - dimension_separator = "/" - elif dimension_separator != "/": - raise ValueError( - "NestedDirectoryStore only supports '/' as dimension_separator") - self._dimension_separator = dimension_separator - - def __eq__(self, other): - return ( - isinstance(other, NestedDirectoryStore) and - self.path == other.path - ) - - -# noinspection PyPep8Naming -class ZipStore(MutableMapping): - """Storage class using a Zip file. - - Parameters - ---------- - path : string - Location of file. - compression : integer, optional - Compression method to use when writing to the archive. - allowZip64 : bool, optional - If True (the default) will create ZIP files that use the ZIP64 - extensions when the zipfile is larger than 2 GiB. If False - will raise an exception when the ZIP file would require ZIP64 - extensions. - mode : string, optional - One of 'r' to read an existing file, 'w' to truncate and write a new - file, 'a' to append to an existing file, or 'x' to exclusively create - and write a new file. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.ZipStore('data/array.zip', mode='w') - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store) - >>> z[...] = 42 - >>> store.close() # don't forget to call this when you're done - - Store a group:: - - >>> store = zarr.ZipStore('data/group.zip', mode='w') - >>> root = zarr.group(store=store) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - >>> store.close() # don't forget to call this when you're done - - After modifying a ZipStore, the ``close()`` method must be called, otherwise - essential data will not be written to the underlying Zip file. The ZipStore - class also supports the context manager protocol, which ensures the ``close()`` - method is called on leaving the context, e.g.:: - - >>> with zarr.ZipStore('data/array.zip', mode='w') as store: - ... z = zarr.zeros((10, 10), chunks=(5, 5), store=store) - ... z[...] = 42 - ... # no need to call store.close() - - Notes - ----- - Each chunk of an array is stored as a separate entry in the Zip file. Note - that Zip files do not provide any way to remove or replace existing entries. - If an attempt is made to replace an entry, then a warning is generated by - the Python standard library about a duplicate Zip file entry. This can be - triggered if you attempt to write data to a Zarr array more than once, - e.g.:: - - >>> store = zarr.ZipStore('data/example.zip', mode='w') - >>> z = zarr.zeros(100, chunks=10, store=store) - >>> # first write OK - ... z[...] = 42 - >>> # second write generates warnings - ... z[...] = 42 # doctest: +SKIP - >>> store.close() - - This can also happen in a more subtle situation, where data are written only - once to a Zarr array, but the write operations are not aligned with chunk - boundaries, e.g.:: - - >>> store = zarr.ZipStore('data/example.zip', mode='w') - >>> z = zarr.zeros(100, chunks=10, store=store) - >>> z[5:15] = 42 - >>> # write overlaps chunk previously written, generates warnings - ... z[15:25] = 42 # doctest: +SKIP - - To avoid creating duplicate entries, only write data once, and align writes - with chunk boundaries. This alignment is done automatically if you call - ``z[...] = ...`` or create an array from existing data via :func:`zarr.array`. - - Alternatively, use a :class:`DirectoryStore` when writing the data, then - manually Zip the directory and use the Zip file for subsequent reads. - Take note that the files in the Zip file must be relative to the root of the - Zarr archive. You may find it easier to create such a Zip file with ``7z``, e.g.:: - - 7z a -tzip archive.zarr.zip archive.zarr/. - - Safe to write in multiple threads but not in multiple processes. - - """ - - def __init__(self, path, compression=zipfile.ZIP_STORED, allowZip64=True, mode='a', - dimension_separator=None): - - # store properties - path = os.path.abspath(path) - self.path = path - self.compression = compression - self.allowZip64 = allowZip64 - self.mode = mode - self._dimension_separator = dimension_separator - - # Current understanding is that zipfile module in stdlib is not thread-safe, - # and so locking is required for both read and write. However, this has not - # been investigated in detail, perhaps no lock is needed if mode='r'. - self.mutex = RLock() - - # open zip file - self.zf = zipfile.ZipFile(path, mode=mode, compression=compression, - allowZip64=allowZip64) - - def __getstate__(self): - self.flush() - return self.path, self.compression, self.allowZip64, self.mode - - def __setstate__(self, state): - path, compression, allowZip64, mode = state - # if initially opened with mode 'w' or 'x', re-open in mode 'a' so file doesn't - # get clobbered - if mode in 'wx': - mode = 'a' - self.__init__(path=path, compression=compression, allowZip64=allowZip64, - mode=mode) - - def close(self): - """Closes the underlying zip file, ensuring all records are written.""" - with self.mutex: - self.zf.close() - - def flush(self): - """Closes the underlying zip file, ensuring all records are written, - then re-opens the file for further modifications.""" - if self.mode != 'r': - with self.mutex: - self.zf.close() - # N.B., re-open with mode 'a' regardless of initial mode so we don't wipe - # what's been written - self.zf = zipfile.ZipFile(self.path, mode='a', - compression=self.compression, - allowZip64=self.allowZip64) - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def __getitem__(self, key): - with self.mutex: - with self.zf.open(key) as f: # will raise KeyError - return f.read() - - def __setitem__(self, key, value): - if self.mode == 'r': - raise ReadOnlyError() - value = ensure_contiguous_ndarray(value) - with self.mutex: - # writestr(key, value) writes with default permissions from - # zipfile (600) that are too restrictive, build ZipInfo for - # the key to work around limitation - keyinfo = zipfile.ZipInfo(filename=key, - date_time=time.localtime(time.time())[:6]) - keyinfo.compress_type = self.compression - if keyinfo.filename[-1] == os.sep: - keyinfo.external_attr = 0o40775 << 16 # drwxrwxr-x - keyinfo.external_attr |= 0x10 # MS-DOS directory flag - else: - keyinfo.external_attr = 0o644 << 16 # ?rw-r--r-- - - self.zf.writestr(keyinfo, value) - - def __delitem__(self, key): - raise NotImplementedError - - def __eq__(self, other): - return ( - isinstance(other, ZipStore) and - self.path == other.path and - self.compression == other.compression and - self.allowZip64 == other.allowZip64 - ) - - def keylist(self): - with self.mutex: - return sorted(self.zf.namelist()) - - def keys(self): - for key in self.keylist(): - yield key - - def __iter__(self): - return self.keys() - - def __len__(self): - return sum(1 for _ in self.keys()) - - def __contains__(self, key): - try: - with self.mutex: - self.zf.getinfo(key) - except KeyError: - return False - else: - return True - - def listdir(self, path=None): - path = normalize_storage_path(path) - return _listdir_from_keys(self, path) - - def getsize(self, path=None): - path = normalize_storage_path(path) - with self.mutex: - children = self.listdir(path) - if children: - size = 0 - for child in children: - if path: - name = path + '/' + child - else: - name = child - try: - info = self.zf.getinfo(name) - except KeyError: - pass - else: - size += info.compress_size - return size - elif path: - try: - info = self.zf.getinfo(path) - return info.compress_size - except KeyError: - return 0 - else: - return 0 - - def clear(self): - if self.mode == 'r': - raise ReadOnlyError() - with self.mutex: - self.close() - os.remove(self.path) - self.zf = zipfile.ZipFile(self.path, mode=self.mode, - compression=self.compression, - allowZip64=self.allowZip64) - - -def migrate_1to2(store): - """Migrate array metadata in `store` from Zarr format version 1 to - version 2. - - Parameters - ---------- - store : MutableMapping - Store to be migrated. - - Notes - ----- - Version 1 did not support hierarchies, so this migration function will - look for a single array in `store` and migrate the array metadata to - version 2. - - """ - - # migrate metadata - from zarr import meta_v1 - meta = meta_v1.decode_metadata(store['meta']) - del store['meta'] - - # add empty filters - meta['filters'] = None - - # migration compression metadata - compression = meta['compression'] - if compression is None or compression == 'none': - compressor_config = None - else: - compression_opts = meta['compression_opts'] - codec_cls = codec_registry[compression] - if isinstance(compression_opts, dict): - compressor = codec_cls(**compression_opts) - else: - compressor = codec_cls(compression_opts) - compressor_config = compressor.get_config() - meta['compressor'] = compressor_config - del meta['compression'] - del meta['compression_opts'] - - # store migrated metadata - store[array_meta_key] = encode_array_metadata(meta) - - # migrate user attributes - store[attrs_key] = store['attrs'] - del store['attrs'] - - -# noinspection PyShadowingBuiltins -class DBMStore(MutableMapping): - """Storage class using a DBM-style database. - - Parameters - ---------- - path : string - Location of database file. - flag : string, optional - Flags for opening the database file. - mode : int - File mode used if a new file is created. - open : function, optional - Function to open the database file. If not provided, :func:`dbm.open` will be - used on Python 3, and :func:`anydbm.open` will be used on Python 2. - write_lock: bool, optional - Use a lock to prevent concurrent writes from multiple threads (True by default). - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk.e - **open_kwargs - Keyword arguments to pass the `open` function. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.DBMStore('data/array.db') - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - >>> store.close() # don't forget to call this when you're done - - Store a group:: - - >>> store = zarr.DBMStore('data/group.db') - >>> root = zarr.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - >>> store.close() # don't forget to call this when you're done - - After modifying a DBMStore, the ``close()`` method must be called, otherwise - essential data may not be written to the underlying database file. The - DBMStore class also supports the context manager protocol, which ensures the - ``close()`` method is called on leaving the context, e.g.:: - - >>> with zarr.DBMStore('data/array.db') as store: - ... z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - ... z[...] = 42 - ... # no need to call store.close() - - A different database library can be used by passing a different function to - the `open` parameter. For example, if the `bsddb3 - `_ package is installed, a - Berkeley DB database can be used:: - - >>> import bsddb3 - >>> store = zarr.DBMStore('data/array.bdb', open=bsddb3.btopen) - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - >>> store.close() - - Notes - ----- - Please note that, by default, this class will use the Python standard - library `dbm.open` function to open the database file (or `anydbm.open` on - Python 2). There are up to three different implementations of DBM-style - databases available in any Python installation, and which one is used may - vary from one system to another. Database file formats are not compatible - between these different implementations. Also, some implementations are - more efficient than others. In particular, the "dumb" implementation will be - the fall-back on many systems, and has very poor performance for some usage - scenarios. If you want to ensure a specific implementation is used, pass the - corresponding open function, e.g., `dbm.gnu.open` to use the GNU DBM - library. - - Safe to write in multiple threads. May be safe to write in multiple processes, - depending on which DBM implementation is being used, although this has not been - tested. - - """ - - def __init__(self, path, flag='c', mode=0o666, open=None, write_lock=True, - dimension_separator=None, - **open_kwargs): - if open is None: - import dbm - open = dbm.open - path = os.path.abspath(path) - # noinspection PyArgumentList - self.db = open(path, flag, mode, **open_kwargs) - self.path = path - self.flag = flag - self.mode = mode - self.open = open - self.write_lock = write_lock - if write_lock: - # This may not be required as some dbm implementations manage their own - # locks, but err on the side of caution. - self.write_mutex = Lock() - else: - self.write_mutex = nolock - self.open_kwargs = open_kwargs - self._dimension_separator = dimension_separator - - def __getstate__(self): - try: - self.flush() # needed for ndbm - except Exception: - # flush may fail if db has already been closed - pass - return (self.path, self.flag, self.mode, self.open, self.write_lock, - self.open_kwargs) - - def __setstate__(self, state): - path, flag, mode, open, write_lock, open_kws = state - if flag[0] == 'n': - flag = 'c' + flag[1:] # don't clobber an existing database - self.__init__(path=path, flag=flag, mode=mode, open=open, - write_lock=write_lock, **open_kws) - - def close(self): - """Closes the underlying database file.""" - if hasattr(self.db, 'close'): - with self.write_mutex: - self.db.close() - - def flush(self): - """Synchronizes data to the underlying database file.""" - if self.flag[0] != 'r': - with self.write_mutex: - if hasattr(self.db, 'sync'): - self.db.sync() - else: # pragma: no cover - # we don't cover this branch anymore as ndbm (oracle) is not packaged - # by conda-forge on non-mac OS: - # https://github.com/conda-forge/staged-recipes/issues/4476 - # fall-back, close and re-open, needed for ndbm - flag = self.flag - if flag[0] == 'n': - flag = 'c' + flag[1:] # don't clobber an existing database - self.db.close() - # noinspection PyArgumentList - self.db = self.open(self.path, flag, self.mode, **self.open_kwargs) - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def __getitem__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - return self.db[key] - - def __setitem__(self, key, value): - if isinstance(key, str): - key = key.encode("ascii") - value = ensure_bytes(value) - with self.write_mutex: - self.db[key] = value - - def __delitem__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - with self.write_mutex: - del self.db[key] - - def __eq__(self, other): - return ( - isinstance(other, DBMStore) and - self.path == other.path and - # allow flag and mode to differ - self.open == other.open and - self.open_kwargs == other.open_kwargs - ) - - def keys(self): - return (ensure_text(k, "ascii") for k in iter(self.db.keys())) - - def __iter__(self): - return self.keys() - - def __len__(self): - return sum(1 for _ in self.keys()) - - def __contains__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - return key in self.db - - -class LMDBStore(MutableMapping): - """Storage class using LMDB. Requires the `lmdb `_ - package to be installed. - - - Parameters - ---------- - path : string - Location of database file. - buffers : bool, optional - If True (default) use support for buffers, which should increase performance by - reducing memory copies. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - **kwargs - Keyword arguments passed through to the `lmdb.open` function. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.LMDBStore('data/array.mdb') - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - >>> store.close() # don't forget to call this when you're done - - Store a group:: - - >>> store = zarr.LMDBStore('data/group.mdb') - >>> root = zarr.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - >>> store.close() # don't forget to call this when you're done - - After modifying a DBMStore, the ``close()`` method must be called, otherwise - essential data may not be written to the underlying database file. The - DBMStore class also supports the context manager protocol, which ensures the - ``close()`` method is called on leaving the context, e.g.:: - - >>> with zarr.LMDBStore('data/array.mdb') as store: - ... z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - ... z[...] = 42 - ... # no need to call store.close() - - Notes - ----- - By default writes are not immediately flushed to disk to increase performance. You - can ensure data are flushed to disk by calling the ``flush()`` or ``close()`` methods. - - Should be safe to write in multiple threads or processes due to the synchronization - support within LMDB, although writing from multiple processes has not been tested. - - """ - - def __init__(self, path, buffers=True, dimension_separator=None, **kwargs): - import lmdb - - # set default memory map size to something larger than the lmdb default, which is - # very likely to be too small for any moderate array (logic copied from zict) - map_size = (2**40 if sys.maxsize >= 2**32 else 2**28) - kwargs.setdefault('map_size', map_size) - - # don't initialize buffers to zero by default, shouldn't be necessary - kwargs.setdefault('meminit', False) - - # decide whether to use the writemap option based on the operating system's - # support for sparse files - writemap requires sparse file support otherwise - # the whole# `map_size` may be reserved up front on disk (logic copied from zict) - writemap = sys.platform.startswith('linux') - kwargs.setdefault('writemap', writemap) - - # decide options for when data are flushed to disk - choose to delay syncing - # data to filesystem, otherwise pay a large performance penalty (zict also does - # this) - kwargs.setdefault('metasync', False) - kwargs.setdefault('sync', False) - kwargs.setdefault('map_async', False) - - # set default option for number of cached transactions - max_spare_txns = multiprocessing.cpu_count() - kwargs.setdefault('max_spare_txns', max_spare_txns) - - # normalize path - path = os.path.abspath(path) - - # open database - self.db = lmdb.open(path, **kwargs) - - # store properties - self.buffers = buffers - self.path = path - self.kwargs = kwargs - self._dimension_separator = dimension_separator - - def __getstate__(self): - try: - self.flush() # just in case - except Exception: - # flush may fail if db has already been closed - pass - return self.path, self.buffers, self.kwargs - - def __setstate__(self, state): - path, buffers, kwargs = state - self.__init__(path=path, buffers=buffers, **kwargs) - - def close(self): - """Closes the underlying database.""" - self.db.close() - - def flush(self): - """Synchronizes data to the file system.""" - self.db.sync() - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def __getitem__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - # use the buffers option, should avoid a memory copy - with self.db.begin(buffers=self.buffers) as txn: - value = txn.get(key) - if value is None: - raise KeyError(key) - return value - - def __setitem__(self, key, value): - if isinstance(key, str): - key = key.encode("ascii") - with self.db.begin(write=True, buffers=self.buffers) as txn: - txn.put(key, value) - - def __delitem__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - with self.db.begin(write=True) as txn: - if not txn.delete(key): - raise KeyError(key) - - def __contains__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - with self.db.begin(buffers=self.buffers) as txn: - with txn.cursor() as cursor: - return cursor.set_key(key) - - def items(self): - with self.db.begin(buffers=self.buffers) as txn: - with txn.cursor() as cursor: - for k, v in cursor.iternext(keys=True, values=True): - yield ensure_text(k, "ascii"), v - - def keys(self): - with self.db.begin(buffers=self.buffers) as txn: - with txn.cursor() as cursor: - for k in cursor.iternext(keys=True, values=False): - yield ensure_text(k, "ascii") - - def values(self): - with self.db.begin(buffers=self.buffers) as txn: - with txn.cursor() as cursor: - for v in cursor.iternext(keys=False, values=True): - yield v - - def __iter__(self): - return self.keys() - - def __len__(self): - return self.db.stat()['entries'] - - -class LRUStoreCache(MutableMapping): - """Storage class that implements a least-recently-used (LRU) cache layer over - some other store. Intended primarily for use with stores that can be slow to - access, e.g., remote stores that require network communication to store and - retrieve data. - - Parameters - ---------- - store : MutableMapping - The store containing the actual data to be cached. - max_size : int - The maximum size that the cache may grow to, in number of bytes. Provide `None` - if you would like the cache to have unlimited size. - - Examples - -------- - The example below wraps an S3 store with an LRU cache:: - - >>> import s3fs - >>> import zarr - >>> s3 = s3fs.S3FileSystem(anon=True, client_kwargs=dict(region_name='eu-west-2')) - >>> store = s3fs.S3Map(root='zarr-demo/store', s3=s3, check=False) - >>> cache = zarr.LRUStoreCache(store, max_size=2**28) - >>> root = zarr.group(store=cache) # doctest: +REMOTE_DATA - >>> z = root['foo/bar/baz'] # doctest: +REMOTE_DATA - >>> from timeit import timeit - >>> # first data access is relatively slow, retrieved from store - ... timeit('print(z[:].tostring())', number=1, globals=globals()) # doctest: +SKIP - b'Hello from the cloud!' - 0.1081731989979744 - >>> # second data access is faster, uses cache - ... timeit('print(z[:].tostring())', number=1, globals=globals()) # doctest: +SKIP - b'Hello from the cloud!' - 0.0009490990014455747 - - """ - - def __init__(self, store, max_size): - self._store = store - self._max_size = max_size - self._current_size = 0 - self._keys_cache = None - self._contains_cache = None - self._listdir_cache = dict() - self._values_cache = OrderedDict() - self._mutex = Lock() - self.hits = self.misses = 0 - - def __getstate__(self): - return (self._store, self._max_size, self._current_size, self._keys_cache, - self._contains_cache, self._listdir_cache, self._values_cache, self.hits, - self.misses) - - def __setstate__(self, state): - (self._store, self._max_size, self._current_size, self._keys_cache, - self._contains_cache, self._listdir_cache, self._values_cache, self.hits, - self.misses) = state - self._mutex = Lock() - - def __len__(self): - return len(self._keys()) - - def __iter__(self): - return self.keys() - - def __contains__(self, key): - with self._mutex: - if self._contains_cache is None: - self._contains_cache = set(self._keys()) - return key in self._contains_cache - - def clear(self): - self._store.clear() - self.invalidate() - - def keys(self): - with self._mutex: - return iter(self._keys()) - - def _keys(self): - if self._keys_cache is None: - self._keys_cache = list(self._store.keys()) - return self._keys_cache - - def listdir(self, path=None): - with self._mutex: - try: - return self._listdir_cache[path] - except KeyError: - listing = listdir(self._store, path) - self._listdir_cache[path] = listing - return listing - - def getsize(self, path=None): - return getsize(self._store, path=path) - - def _pop_value(self): - # remove the first value from the cache, as this will be the least recently - # used value - _, v = self._values_cache.popitem(last=False) - return v - - def _accommodate_value(self, value_size): - if self._max_size is None: - return - # ensure there is enough space in the cache for a new value - while self._current_size + value_size > self._max_size: - v = self._pop_value() - self._current_size -= buffer_size(v) - - def _cache_value(self, key, value): - # cache a value - value_size = buffer_size(value) - # check size of the value against max size, as if the value itself exceeds max - # size then we are never going to cache it - if self._max_size is None or value_size <= self._max_size: - self._accommodate_value(value_size) - self._values_cache[key] = value - self._current_size += value_size - - def invalidate(self): - """Completely clear the cache.""" - with self._mutex: - self._values_cache.clear() - self._invalidate_keys() - - def invalidate_values(self): - """Clear the values cache.""" - with self._mutex: - self._values_cache.clear() - - def invalidate_keys(self): - """Clear the keys cache.""" - with self._mutex: - self._invalidate_keys() - - def _invalidate_keys(self): - self._keys_cache = None - self._contains_cache = None - self._listdir_cache.clear() - - def _invalidate_value(self, key): - if key in self._values_cache: - value = self._values_cache.pop(key) - self._current_size -= buffer_size(value) - - def __getitem__(self, key): - try: - # first try to obtain the value from the cache - with self._mutex: - value = self._values_cache[key] - # cache hit if no KeyError is raised - self.hits += 1 - # treat the end as most recently used - self._values_cache.move_to_end(key) - - except KeyError: - # cache miss, retrieve value from the store - value = self._store[key] - with self._mutex: - self.misses += 1 - # need to check if key is not in the cache, as it may have been cached - # while we were retrieving the value from the store - if key not in self._values_cache: - self._cache_value(key, value) - - return value - - def __setitem__(self, key, value): - self._store[key] = value - with self._mutex: - self._invalidate_keys() - self._invalidate_value(key) - self._cache_value(key, value) - - def __delitem__(self, key): - del self._store[key] - with self._mutex: - self._invalidate_keys() - self._invalidate_value(key) - - -class SQLiteStore(MutableMapping): - """Storage class using SQLite. - - Parameters - ---------- - path : string - Location of database file. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - **kwargs - Keyword arguments passed through to the `sqlite3.connect` function. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.SQLiteStore('data/array.sqldb') - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - >>> store.close() # don't forget to call this when you're done - - Store a group:: - - >>> store = zarr.SQLiteStore('data/group.sqldb') - >>> root = zarr.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - >>> store.close() # don't forget to call this when you're done - """ - - def __init__(self, path, dimension_separator=None, **kwargs): - import sqlite3 - - self._dimension_separator = dimension_separator - - # normalize path - if path != ':memory:': - path = os.path.abspath(path) - - # store properties - self.path = path - self.kwargs = kwargs - - # allow threading if SQLite connections are thread-safe - # - # ref: https://www.sqlite.org/releaselog/3_3_1.html - # ref: https://bugs.python.org/issue27190 - check_same_thread = True - if sqlite3.sqlite_version_info >= (3, 3, 1): - check_same_thread = False - - # keep a lock for serializing mutable operations - self.lock = Lock() - - # open database - self.db = sqlite3.connect( - self.path, - detect_types=0, - isolation_level=None, - check_same_thread=check_same_thread, - **self.kwargs - ) - - # handle keys as `str`s - self.db.text_factory = str - - # get a cursor to read/write to the database - self.cursor = self.db.cursor() - - # initialize database with our table if missing - with self.lock: - self.cursor.execute( - 'CREATE TABLE IF NOT EXISTS zarr(k TEXT PRIMARY KEY, v BLOB)' - ) - - def __getstate__(self): - if self.path == ':memory:': - raise PicklingError('Cannot pickle in-memory SQLite databases') - return self.path, self.kwargs - - def __setstate__(self, state): - path, kwargs = state - self.__init__(path=path, **kwargs) - - def close(self): - """Closes the underlying database.""" - - # close cursor and db objects - self.cursor.close() - self.db.close() - - def __getitem__(self, key): - value = self.cursor.execute('SELECT v FROM zarr WHERE (k = ?)', (key,)) - for v, in value: - return v - raise KeyError(key) - - def __setitem__(self, key, value): - self.update({key: value}) - - def __delitem__(self, key): - with self.lock: - self.cursor.execute('DELETE FROM zarr WHERE (k = ?)', (key,)) - if self.cursor.rowcount < 1: - raise KeyError(key) - - def __contains__(self, key): - cs = self.cursor.execute( - 'SELECT COUNT(*) FROM zarr WHERE (k = ?)', (key,) - ) - for has, in cs: - has = bool(has) - return has - - def items(self): - kvs = self.cursor.execute('SELECT k, v FROM zarr') - for k, v in kvs: - yield k, v - - def keys(self): - ks = self.cursor.execute('SELECT k FROM zarr') - for k, in ks: - yield k - - def values(self): - vs = self.cursor.execute('SELECT v FROM zarr') - for v, in vs: - yield v - - def __iter__(self): - return self.keys() - - def __len__(self): - cs = self.cursor.execute('SELECT COUNT(*) FROM zarr') - for c, in cs: - return c - - def update(self, *args, **kwargs): - args += (kwargs,) - - kv_list = [] - for dct in args: - for k, v in dct.items(): - v = ensure_contiguous_ndarray(v) - - # Accumulate key-value pairs for storage - kv_list.append((k, v)) - - with self.lock: - self.cursor.executemany('REPLACE INTO zarr VALUES (?, ?)', kv_list) - - def listdir(self, path=None): - path = normalize_storage_path(path) - sep = '_' if path == '' else '/' - keys = self.cursor.execute( - ''' - SELECT DISTINCT SUBSTR(m, 0, INSTR(m, "/")) AS l FROM ( - SELECT LTRIM(SUBSTR(k, LENGTH(?) + 1), "/") || "/" AS m - FROM zarr WHERE k LIKE (? || "{sep}%") - ) ORDER BY l ASC - '''.format(sep=sep), - (path, path) - ) - keys = list(map(operator.itemgetter(0), keys)) - return keys - - def getsize(self, path=None): - path = normalize_storage_path(path) - size = self.cursor.execute( - ''' - SELECT COALESCE(SUM(LENGTH(v)), 0) FROM zarr - WHERE k LIKE (? || "%") AND - 0 == INSTR(LTRIM(SUBSTR(k, LENGTH(?) + 1), "/"), "/") - ''', - (path, path) - ) - for s, in size: - return s - - def rmdir(self, path=None): - path = normalize_storage_path(path) - if path: - with self.lock: - self.cursor.execute( - 'DELETE FROM zarr WHERE k LIKE (? || "/%")', (path,) - ) - else: - self.clear() - - def clear(self): - with self.lock: - self.cursor.executescript( - ''' - BEGIN TRANSACTION; - DROP TABLE zarr; - CREATE TABLE zarr(k TEXT PRIMARY KEY, v BLOB); - COMMIT TRANSACTION; - ''' - ) - - -class MongoDBStore(MutableMapping): - """Storage class using MongoDB. - - .. note:: This is an experimental feature. - - Requires the `pymongo `_ - package to be installed. - - Parameters - ---------- - database : string - Name of database - collection : string - Name of collection - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - **kwargs - Keyword arguments passed through to the `pymongo.MongoClient` function. - - Notes - ----- - The maximum chunksize in MongoDB documents is 16 MB. - - """ - - _key = 'key' - _value = 'value' - - def __init__(self, database='mongodb_zarr', collection='zarr_collection', - dimension_separator=None, **kwargs): - import pymongo - - self._database = database - self._collection = collection - self._dimension_separator = dimension_separator - self._kwargs = kwargs - - self.client = pymongo.MongoClient(**self._kwargs) - self.db = self.client.get_database(self._database) - self.collection = self.db.get_collection(self._collection) - - def __getitem__(self, key): - doc = self.collection.find_one({self._key: key}) - - if doc is None: - raise KeyError(key) - else: - return doc[self._value] - - def __setitem__(self, key, value): - value = ensure_bytes(value) - self.collection.replace_one({self._key: key}, - {self._key: key, self._value: value}, - upsert=True) - - def __delitem__(self, key): - result = self.collection.delete_many({self._key: key}) - if not result.deleted_count == 1: - raise KeyError(key) - - def __iter__(self): - for f in self.collection.find({}): - yield f[self._key] - - def __len__(self): - return self.collection.count_documents({}) - - def __getstate__(self): - return self._database, self._collection, self._kwargs - - def __setstate__(self, state): - database, collection, kwargs = state - self.__init__(database=database, collection=collection, **kwargs) - - def close(self): - """Cleanup client resources and disconnect from MongoDB.""" - self.client.close() - - def clear(self): - """Remove all items from store.""" - self.collection.delete_many({}) - - -class RedisStore(MutableMapping): - """Storage class using Redis. - - .. note:: This is an experimental feature. - - Requires the `redis `_ - package to be installed. - - Parameters - ---------- - prefix : string - Name of prefix for Redis keys - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - **kwargs - Keyword arguments passed through to the `redis.Redis` function. - - """ - def __init__(self, prefix='zarr', dimension_separator=None, **kwargs): - import redis - self._prefix = prefix - self._kwargs = kwargs - self._dimension_separator = dimension_separator - - self.client = redis.Redis(**kwargs) - - def _key(self, key): - return '{prefix}:{key}'.format(prefix=self._prefix, key=key) - - def __getitem__(self, key): - return self.client[self._key(key)] - - def __setitem__(self, key, value): - value = ensure_bytes(value) - self.client[self._key(key)] = value - - def __delitem__(self, key): - count = self.client.delete(self._key(key)) - if not count: - raise KeyError(key) - - def keylist(self): - offset = len(self._key('')) # length of prefix - return [key[offset:].decode('utf-8') - for key in self.client.keys(self._key('*'))] - - def keys(self): - for key in self.keylist(): - yield key - - def __iter__(self): - for key in self.keys(): - yield key - - def __len__(self): - return len(self.keylist()) - - def __getstate__(self): - return self._prefix, self._kwargs - - def __setstate__(self, state): - prefix, kwargs = state - self.__init__(prefix=prefix, **kwargs) - - def clear(self): - for key in self.keys(): - del self[key] - - -class ConsolidatedMetadataStore(MutableMapping): - """A layer over other storage, where the metadata has been consolidated into - a single key. - - The purpose of this class, is to be able to get all of the metadata for - a given array in a single read operation from the underlying storage. - See :func:`zarr.convenience.consolidate_metadata` for how to create this - single metadata key. - - This class loads from the one key, and stores the data in a dict, so that - accessing the keys no longer requires operations on the backend store. - - This class is read-only, and attempts to change the array metadata will - fail, but changing the data is possible. If the backend storage is changed - directly, then the metadata stored here could become obsolete, and - :func:`zarr.convenience.consolidate_metadata` should be called again and the class - re-invoked. The use case is for write once, read many times. - - .. versionadded:: 2.3 - - .. note:: This is an experimental feature. - - Parameters - ---------- - store: MutableMapping - Containing the zarr array. - metadata_key: str - The target in the store where all of the metadata are stored. We - assume JSON encoding. - - See Also - -------- - zarr.convenience.consolidate_metadata, zarr.convenience.open_consolidated - - """ - - def __init__(self, store, metadata_key='.zmetadata'): - self.store = store - - # retrieve consolidated metadata - meta = json_loads(store[metadata_key]) - - # check format of consolidated metadata - consolidated_format = meta.get('zarr_consolidated_format', None) - if consolidated_format != 1: - raise MetadataError('unsupported zarr consolidated metadata format: %s' % - consolidated_format) - - # decode metadata - self.meta_store = meta['metadata'] - - def __getitem__(self, key): - return self.meta_store[key] - - def __contains__(self, item): - return item in self.meta_store - - def __iter__(self): - return iter(self.meta_store) - - def __len__(self): - return len(self.meta_store) - - def __delitem__(self, key): - raise ReadOnlyError() - - def __setitem__(self, key, value): - raise ReadOnlyError() - - def getsize(self, path): - return getsize(self.meta_store, path) - - def listdir(self, path): - return listdir(self.meta_store, path) diff --git a/zarr/sync.py b/zarr/sync.py deleted file mode 100644 index fa5d811a1e..0000000000 --- a/zarr/sync.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -from collections import defaultdict -from threading import Lock - -import fasteners - - -class ThreadSynchronizer(object): - """Provides synchronization using thread locks.""" - - def __init__(self): - self.mutex = Lock() - self.locks = defaultdict(Lock) - - def __getitem__(self, item): - with self.mutex: - return self.locks[item] - - def __getstate__(self): - return True - - def __setstate__(self, *args): - # reinitialize from scratch - self.__init__() - - -class ProcessSynchronizer(object): - """Provides synchronization using file locks via the - `fasteners `_ - package. - - Parameters - ---------- - path : string - Path to a directory on a file system that is shared by all processes. - N.B., this should be a *different* path to where you store the array. - - """ - - def __init__(self, path): - self.path = path - - def __getitem__(self, item): - path = os.path.join(self.path, item) - lock = fasteners.InterProcessLock(path) - return lock - - # pickling and unpickling should be handled automatically diff --git a/zarr/tests/conftest.py b/zarr/tests/conftest.py deleted file mode 100644 index aa73b8691e..0000000000 --- a/zarr/tests/conftest.py +++ /dev/null @@ -1,8 +0,0 @@ -import pathlib - -import pytest - - -@pytest.fixture(params=[str, pathlib.Path]) -def path_type(request): - return request.param diff --git a/zarr/tests/data/store.zip b/zarr/tests/data/store.zip deleted file mode 100644 index a36fd675b3..0000000000 Binary files a/zarr/tests/data/store.zip and /dev/null differ diff --git a/zarr/tests/data/store/foo b/zarr/tests/data/store/foo deleted file mode 100644 index ba0e162e1c..0000000000 --- a/zarr/tests/data/store/foo +++ /dev/null @@ -1 +0,0 @@ -bar \ No newline at end of file diff --git a/zarr/tests/test_attrs.py b/zarr/tests/test_attrs.py deleted file mode 100644 index 2aced3abaa..0000000000 --- a/zarr/tests/test_attrs.py +++ /dev/null @@ -1,219 +0,0 @@ -import json -import unittest - -import pytest - -from zarr.attrs import Attributes -from zarr.tests.util import CountingDict - - -class TestAttributes(unittest.TestCase): - - def init_attributes(self, store, read_only=False, cache=True): - return Attributes(store, key='attrs', read_only=read_only, cache=cache) - - def test_storage(self): - - store = dict() - a = Attributes(store=store, key='attrs') - assert 'foo' not in a - assert 'bar' not in a - assert dict() == a.asdict() - - a['foo'] = 'bar' - a['baz'] = 42 - assert 'attrs' in store - assert isinstance(store['attrs'], bytes) - d = json.loads(str(store['attrs'], 'ascii')) - assert dict(foo='bar', baz=42) == d - - def test_get_set_del_contains(self): - - a = self.init_attributes(dict()) - assert 'foo' not in a - a['foo'] = 'bar' - a['baz'] = 42 - assert 'foo' in a - assert 'baz' in a - assert 'bar' == a['foo'] - assert 42 == a['baz'] - del a['foo'] - assert 'foo' not in a - with pytest.raises(KeyError): - # noinspection PyStatementEffect - a['foo'] - - def test_update_put(self): - - a = self.init_attributes(dict()) - assert 'foo' not in a - assert 'bar' not in a - assert 'baz' not in a - - a.update(foo='spam', bar=42, baz=4.2) - assert a['foo'] == 'spam' - assert a['bar'] == 42 - assert a['baz'] == 4.2 - - a.put(dict(foo='eggs', bar=84)) - assert a['foo'] == 'eggs' - assert a['bar'] == 84 - assert 'baz' not in a - - def test_iterators(self): - - a = self.init_attributes(dict()) - assert 0 == len(a) - assert set() == set(a) - assert set() == set(a.keys()) - assert set() == set(a.values()) - assert set() == set(a.items()) - - a['foo'] = 'bar' - a['baz'] = 42 - - assert 2 == len(a) - assert {'foo', 'baz'} == set(a) - assert {'foo', 'baz'} == set(a.keys()) - assert {'bar', 42} == set(a.values()) - assert {('foo', 'bar'), ('baz', 42)} == set(a.items()) - - def test_read_only(self): - store = dict() - a = self.init_attributes(store, read_only=True) - store['attrs'] = json.dumps(dict(foo='bar', baz=42)).encode('ascii') - assert a['foo'] == 'bar' - assert a['baz'] == 42 - with pytest.raises(PermissionError): - a['foo'] = 'quux' - with pytest.raises(PermissionError): - del a['foo'] - with pytest.raises(PermissionError): - a.update(foo='quux') - - def test_key_completions(self): - a = self.init_attributes(dict()) - d = a._ipython_key_completions_() - assert 'foo' not in d - assert '123' not in d - assert 'baz' not in d - assert 'asdf;' not in d - a['foo'] = 42 - a['123'] = 4.2 - a['asdf;'] = 'ghjkl;' - d = a._ipython_key_completions_() - assert 'foo' in d - assert '123' in d - assert 'asdf;' in d - assert 'baz' not in d - - def test_caching_on(self): - # caching is turned on by default - - # setup store - store = CountingDict() - assert 0 == store.counter['__getitem__', 'attrs'] - assert 0 == store.counter['__setitem__', 'attrs'] - store['attrs'] = json.dumps(dict(foo='xxx', bar=42)).encode('ascii') - assert 0 == store.counter['__getitem__', 'attrs'] - assert 1 == store.counter['__setitem__', 'attrs'] - - # setup attributes - a = self.init_attributes(store) - - # test __getitem__ causes all attributes to be cached - assert a['foo'] == 'xxx' - assert 1 == store.counter['__getitem__', 'attrs'] - assert a['bar'] == 42 - assert 1 == store.counter['__getitem__', 'attrs'] - assert a['foo'] == 'xxx' - assert 1 == store.counter['__getitem__', 'attrs'] - - # test __setitem__ updates the cache - a['foo'] = 'yyy' - assert 2 == store.counter['__getitem__', 'attrs'] - assert 2 == store.counter['__setitem__', 'attrs'] - assert a['foo'] == 'yyy' - assert 2 == store.counter['__getitem__', 'attrs'] - assert 2 == store.counter['__setitem__', 'attrs'] - - # test update() updates the cache - a.update(foo='zzz', bar=84) - assert 3 == store.counter['__getitem__', 'attrs'] - assert 3 == store.counter['__setitem__', 'attrs'] - assert a['foo'] == 'zzz' - assert a['bar'] == 84 - assert 3 == store.counter['__getitem__', 'attrs'] - assert 3 == store.counter['__setitem__', 'attrs'] - - # test __contains__ uses the cache - assert 'foo' in a - assert 3 == store.counter['__getitem__', 'attrs'] - assert 3 == store.counter['__setitem__', 'attrs'] - assert 'spam' not in a - assert 3 == store.counter['__getitem__', 'attrs'] - assert 3 == store.counter['__setitem__', 'attrs'] - - # test __delitem__ updates the cache - del a['bar'] - assert 4 == store.counter['__getitem__', 'attrs'] - assert 4 == store.counter['__setitem__', 'attrs'] - assert 'bar' not in a - assert 4 == store.counter['__getitem__', 'attrs'] - assert 4 == store.counter['__setitem__', 'attrs'] - - # test refresh() - store['attrs'] = json.dumps(dict(foo='xxx', bar=42)).encode('ascii') - assert 4 == store.counter['__getitem__', 'attrs'] - a.refresh() - assert 5 == store.counter['__getitem__', 'attrs'] - assert a['foo'] == 'xxx' - assert 5 == store.counter['__getitem__', 'attrs'] - assert a['bar'] == 42 - assert 5 == store.counter['__getitem__', 'attrs'] - - def test_caching_off(self): - - # setup store - store = CountingDict() - assert 0 == store.counter['__getitem__', 'attrs'] - assert 0 == store.counter['__setitem__', 'attrs'] - store['attrs'] = json.dumps(dict(foo='xxx', bar=42)).encode('ascii') - assert 0 == store.counter['__getitem__', 'attrs'] - assert 1 == store.counter['__setitem__', 'attrs'] - - # setup attributes - a = self.init_attributes(store, cache=False) - - # test __getitem__ - assert a['foo'] == 'xxx' - assert 1 == store.counter['__getitem__', 'attrs'] - assert a['bar'] == 42 - assert 2 == store.counter['__getitem__', 'attrs'] - assert a['foo'] == 'xxx' - assert 3 == store.counter['__getitem__', 'attrs'] - - # test __setitem__ - a['foo'] = 'yyy' - assert 4 == store.counter['__getitem__', 'attrs'] - assert 2 == store.counter['__setitem__', 'attrs'] - assert a['foo'] == 'yyy' - assert 5 == store.counter['__getitem__', 'attrs'] - assert 2 == store.counter['__setitem__', 'attrs'] - - # test update() - a.update(foo='zzz', bar=84) - assert 6 == store.counter['__getitem__', 'attrs'] - assert 3 == store.counter['__setitem__', 'attrs'] - assert a['foo'] == 'zzz' - assert a['bar'] == 84 - assert 8 == store.counter['__getitem__', 'attrs'] - assert 3 == store.counter['__setitem__', 'attrs'] - - # test __contains__ - assert 'foo' in a - assert 9 == store.counter['__getitem__', 'attrs'] - assert 3 == store.counter['__setitem__', 'attrs'] - assert 'spam' not in a - assert 10 == store.counter['__getitem__', 'attrs'] - assert 3 == store.counter['__setitem__', 'attrs'] diff --git a/zarr/tests/test_convenience.py b/zarr/tests/test_convenience.py deleted file mode 100644 index a5ac40e371..0000000000 --- a/zarr/tests/test_convenience.py +++ /dev/null @@ -1,701 +0,0 @@ -import atexit -import tempfile -import unittest -from numbers import Integral - -import numpy as np -import pytest -from numcodecs import Adler32, Zlib -from numpy.testing import assert_array_equal - -import zarr -from zarr.convenience import ( - consolidate_metadata, - copy, - copy_store, - load, - open, - open_consolidated, - save, - save_group, - copy_all, -) -from zarr.core import Array -from zarr.errors import CopyError -from zarr.hierarchy import Group, group -from zarr.storage import (ConsolidatedMetadataStore, MemoryStore, - atexit_rmtree, getsize) - - -def test_open_array(path_type): - - store = tempfile.mkdtemp() - atexit.register(atexit_rmtree, store) - store = path_type(store) - - # open array, create if doesn't exist - z = open(store, mode='a', shape=100) - assert isinstance(z, Array) - assert z.shape == (100,) - - # open array, overwrite - z = open(store, mode='w', shape=200) - assert isinstance(z, Array) - assert z.shape == (200,) - - # open array, read-only - z = open(store, mode='r') - assert isinstance(z, Array) - assert z.shape == (200,) - assert z.read_only - - # path not found - with pytest.raises(ValueError): - open('doesnotexist', mode='r') - - -def test_open_group(path_type): - - store = tempfile.mkdtemp() - atexit.register(atexit_rmtree, store) - store = path_type(store) - - # open group, create if doesn't exist - g = open(store, mode='a') - g.create_group('foo') - assert isinstance(g, Group) - assert 'foo' in g - - # open group, overwrite - g = open(store, mode='w') - assert isinstance(g, Group) - assert 'foo' not in g - - # open group, read-only - g = open(store, mode='r') - assert isinstance(g, Group) - assert g.read_only - - -def test_save_errors(): - with pytest.raises(ValueError): - # no arrays provided - save_group('data/group.zarr') - with pytest.raises(ValueError): - # no arrays provided - save('data/group.zarr') - - -def test_lazy_loader(): - foo = np.arange(100) - bar = np.arange(100, 0, -1) - store = 'data/group.zarr' - save(store, foo=foo, bar=bar) - loader = load(store) - assert 'foo' in loader - assert 'bar' in loader - assert 'baz' not in loader - assert len(loader) == 2 - assert sorted(loader) == ['bar', 'foo'] - assert_array_equal(foo, loader['foo']) - assert_array_equal(bar, loader['bar']) - - -def test_consolidate_metadata(): - - # setup initial data - store = MemoryStore() - z = group(store) - z.create_group('g1') - g2 = z.create_group('g2') - g2.attrs['hello'] = 'world' - arr = g2.create_dataset('arr', shape=(20, 20), chunks=(5, 5), dtype='f8') - assert 16 == arr.nchunks - assert 0 == arr.nchunks_initialized - arr.attrs['data'] = 1 - arr[:] = 1.0 - assert 16 == arr.nchunks_initialized - - # perform consolidation - out = consolidate_metadata(store) - assert isinstance(out, Group) - assert '.zmetadata' in store - for key in ['.zgroup', - 'g1/.zgroup', - 'g2/.zgroup', - 'g2/.zattrs', - 'g2/arr/.zarray', - 'g2/arr/.zattrs']: - del store[key] - - # open consolidated - z2 = open_consolidated(store) - assert ['g1', 'g2'] == list(z2) - assert 'world' == z2.g2.attrs['hello'] - assert 1 == z2.g2.arr.attrs['data'] - assert (z2.g2.arr[:] == 1.0).all() - assert 16 == z2.g2.arr.nchunks - assert 16 == z2.g2.arr.nchunks_initialized - - # tests del/write on the store - cmd = ConsolidatedMetadataStore(store) - with pytest.raises(PermissionError): - del cmd['.zgroup'] - with pytest.raises(PermissionError): - cmd['.zgroup'] = None - - # test getsize on the store - assert isinstance(getsize(cmd), Integral) - - # test new metadata are not writeable - with pytest.raises(PermissionError): - z2.create_group('g3') - with pytest.raises(PermissionError): - z2.create_dataset('spam', shape=42, chunks=7, dtype='i4') - with pytest.raises(PermissionError): - del z2['g2'] - - # test consolidated metadata are not writeable - with pytest.raises(PermissionError): - z2.g2.attrs['hello'] = 'universe' - with pytest.raises(PermissionError): - z2.g2.arr.attrs['foo'] = 'bar' - - # test the data are writeable - z2.g2.arr[:] = 2 - assert (z2.g2.arr[:] == 2).all() - - # test invalid modes - with pytest.raises(ValueError): - open_consolidated(store, mode='a') - with pytest.raises(ValueError): - open_consolidated(store, mode='w') - - # make sure keyword arguments are passed through without error - open_consolidated(store, cache_attrs=True, synchronizer=None) - - -def test_consolidated_with_chunk_store(): - # setup initial data - store = MemoryStore() - chunk_store = MemoryStore() - z = group(store, chunk_store=chunk_store) - z.create_group('g1') - g2 = z.create_group('g2') - g2.attrs['hello'] = 'world' - arr = g2.create_dataset('arr', shape=(20, 20), chunks=(5, 5), dtype='f8') - assert 16 == arr.nchunks - assert 0 == arr.nchunks_initialized - arr.attrs['data'] = 1 - arr[:] = 1.0 - assert 16 == arr.nchunks_initialized - - # perform consolidation - out = consolidate_metadata(store) - assert isinstance(out, Group) - assert '.zmetadata' in store - for key in ['.zgroup', - 'g1/.zgroup', - 'g2/.zgroup', - 'g2/.zattrs', - 'g2/arr/.zarray', - 'g2/arr/.zattrs']: - del store[key] - # open consolidated - z2 = open_consolidated(store, chunk_store=chunk_store) - assert ['g1', 'g2'] == list(z2) - assert 'world' == z2.g2.attrs['hello'] - assert 1 == z2.g2.arr.attrs['data'] - assert (z2.g2.arr[:] == 1.0).all() - assert 16 == z2.g2.arr.nchunks - assert 16 == z2.g2.arr.nchunks_initialized - - # test the data are writeable - z2.g2.arr[:] = 2 - assert (z2.g2.arr[:] == 2).all() - - # test invalid modes - with pytest.raises(ValueError): - open_consolidated(store, mode='a', chunk_store=chunk_store) - with pytest.raises(ValueError): - open_consolidated(store, mode='w', chunk_store=chunk_store) - - # make sure keyword arguments are passed through without error - open_consolidated(store, cache_attrs=True, synchronizer=None, - chunk_store=chunk_store) - - -class TestCopyStore(unittest.TestCase): - - def setUp(self): - source = dict() - source['foo'] = b'xxx' - source['bar/baz'] = b'yyy' - source['bar/qux'] = b'zzz' - self.source = source - - def test_no_paths(self): - source = self.source - dest = dict() - copy_store(source, dest) - assert len(source) == len(dest) - for key in source: - assert source[key] == dest[key] - - def test_source_path(self): - source = self.source - # paths should be normalized - for source_path in 'bar', 'bar/', '/bar', '/bar/': - dest = dict() - copy_store(source, dest, source_path=source_path) - assert 2 == len(dest) - for key in source: - if key.startswith('bar/'): - dest_key = key.split('bar/')[1] - assert source[key] == dest[dest_key] - else: - assert key not in dest - - def test_dest_path(self): - source = self.source - # paths should be normalized - for dest_path in 'new', 'new/', '/new', '/new/': - dest = dict() - copy_store(source, dest, dest_path=dest_path) - assert len(source) == len(dest) - for key in source: - dest_key = 'new/' + key - assert source[key] == dest[dest_key] - - def test_source_dest_path(self): - source = self.source - # paths should be normalized - for source_path in 'bar', 'bar/', '/bar', '/bar/': - for dest_path in 'new', 'new/', '/new', '/new/': - dest = dict() - copy_store(source, dest, source_path=source_path, - dest_path=dest_path) - assert 2 == len(dest) - for key in source: - if key.startswith('bar/'): - dest_key = 'new/' + key.split('bar/')[1] - assert source[key] == dest[dest_key] - else: - assert key not in dest - assert ('new/' + key) not in dest - - def test_excludes_includes(self): - source = self.source - - # single excludes - dest = dict() - excludes = 'f.*' - copy_store(source, dest, excludes=excludes) - assert len(dest) == 2 - assert 'foo' not in dest - - # multiple excludes - dest = dict() - excludes = 'b.z', '.*x' - copy_store(source, dest, excludes=excludes) - assert len(dest) == 1 - assert 'foo' in dest - assert 'bar/baz' not in dest - assert 'bar/qux' not in dest - - # excludes and includes - dest = dict() - excludes = 'b.*' - includes = '.*x' - copy_store(source, dest, excludes=excludes, includes=includes) - assert len(dest) == 2 - assert 'foo' in dest - assert 'bar/baz' not in dest - assert 'bar/qux' in dest - - def test_dry_run(self): - source = self.source - dest = dict() - copy_store(source, dest, dry_run=True) - assert 0 == len(dest) - - def test_if_exists(self): - source = self.source - dest = dict() - dest['bar/baz'] = b'mmm' - - # default ('raise') - with pytest.raises(CopyError): - copy_store(source, dest) - - # explicit 'raise' - with pytest.raises(CopyError): - copy_store(source, dest, if_exists='raise') - - # skip - copy_store(source, dest, if_exists='skip') - assert 3 == len(dest) - assert dest['foo'] == b'xxx' - assert dest['bar/baz'] == b'mmm' - assert dest['bar/qux'] == b'zzz' - - # replace - copy_store(source, dest, if_exists='replace') - assert 3 == len(dest) - assert dest['foo'] == b'xxx' - assert dest['bar/baz'] == b'yyy' - assert dest['bar/qux'] == b'zzz' - - # invalid option - with pytest.raises(ValueError): - copy_store(source, dest, if_exists='foobar') - - -def check_copied_array(original, copied, without_attrs=False, - expect_props=None): - - # setup - source_h5py = original.__module__.startswith('h5py.') - dest_h5py = copied.__module__.startswith('h5py.') - zarr_to_zarr = not (source_h5py or dest_h5py) - h5py_to_h5py = source_h5py and dest_h5py - zarr_to_h5py = not source_h5py and dest_h5py - h5py_to_zarr = source_h5py and not dest_h5py - if expect_props is None: - expect_props = dict() - else: - expect_props = expect_props.copy() - - # common properties in zarr and h5py - for p in 'dtype', 'shape', 'chunks': - expect_props.setdefault(p, getattr(original, p)) - - # zarr-specific properties - if zarr_to_zarr: - for p in 'compressor', 'filters', 'order', 'fill_value': - expect_props.setdefault(p, getattr(original, p)) - - # h5py-specific properties - if h5py_to_h5py: - for p in ('maxshape', 'compression', 'compression_opts', 'shuffle', - 'scaleoffset', 'fletcher32', 'fillvalue'): - expect_props.setdefault(p, getattr(original, p)) - - # common properties with some name differences - if h5py_to_zarr: - expect_props.setdefault('fill_value', original.fillvalue) - if zarr_to_h5py: - expect_props.setdefault('fillvalue', original.fill_value) - - # compare properties - for k, v in expect_props.items(): - assert v == getattr(copied, k) - - # compare data - assert_array_equal(original[:], copied[:]) - - # compare attrs - if without_attrs: - for k in original.attrs.keys(): - assert k not in copied.attrs - else: - assert sorted(original.attrs.items()) == sorted(copied.attrs.items()) - - -def check_copied_group(original, copied, without_attrs=False, expect_props=None, - shallow=False): - - # setup - if expect_props is None: - expect_props = dict() - else: - expect_props = expect_props.copy() - - # compare children - for k, v in original.items(): - if hasattr(v, 'shape'): - assert k in copied - check_copied_array(v, copied[k], without_attrs=without_attrs, - expect_props=expect_props) - elif shallow: - assert k not in copied - else: - assert k in copied - check_copied_group(v, copied[k], without_attrs=without_attrs, - shallow=shallow, expect_props=expect_props) - - # compare attrs - if without_attrs: - for k in original.attrs.keys(): - assert k not in copied.attrs - else: - assert sorted(original.attrs.items()) == sorted(copied.attrs.items()) - - -def test_copy_all(): - """ - https://github.com/zarr-developers/zarr-python/issues/269 - - copy_all used to not copy attributes as `.keys()` does not return hidden `.zattrs`. - - """ - original_group = zarr.group(store=MemoryStore(), overwrite=True) - original_group.attrs["info"] = "group attrs" - original_subgroup = original_group.create_group("subgroup") - original_subgroup.attrs["info"] = "sub attrs" - - destination_group = zarr.group(store=MemoryStore(), overwrite=True) - - # copy from memory to directory store - copy_all( - original_group, - destination_group, - dry_run=False, - ) - - assert destination_group.attrs["info"] == "group attrs" - assert destination_group.subgroup.attrs["info"] == "sub attrs" - - -class TestCopy: - @pytest.fixture(params=[False, True], ids=['zarr', 'hdf5']) - def source(self, request, tmpdir): - def prep_source(source): - foo = source.create_group('foo') - foo.attrs['experiment'] = 'weird science' - baz = foo.create_dataset('bar/baz', data=np.arange(100), chunks=(50,)) - baz.attrs['units'] = 'metres' - if request.param: - extra_kws = dict(compression='gzip', compression_opts=3, fillvalue=84, - shuffle=True, fletcher32=True) - else: - extra_kws = dict(compressor=Zlib(3), order='F', fill_value=42, filters=[Adler32()]) - source.create_dataset('spam', data=np.arange(100, 200).reshape(20, 5), - chunks=(10, 2), dtype='i2', **extra_kws) - return source - - if request.param: - h5py = pytest.importorskip('h5py') - fn = tmpdir.join('source.h5') - with h5py.File(str(fn), mode='w') as h5f: - yield prep_source(h5f) - else: - yield prep_source(group()) - - @pytest.fixture(params=[False, True], ids=['zarr', 'hdf5']) - def dest(self, request, tmpdir): - if request.param: - h5py = pytest.importorskip('h5py') - fn = tmpdir.join('dest.h5') - with h5py.File(str(fn), mode='w') as h5f: - yield h5f - else: - yield group() - - def test_copy_array(self, source, dest): - # copy array with default options - copy(source['foo/bar/baz'], dest) - check_copied_array(source['foo/bar/baz'], dest['baz']) - copy(source['spam'], dest) - check_copied_array(source['spam'], dest['spam']) - - def test_copy_bad_dest(self, source, dest): - # try to copy to an array, dest must be a group - dest = dest.create_dataset('eggs', shape=(100,)) - with pytest.raises(ValueError): - copy(source['foo/bar/baz'], dest) - - def test_copy_array_name(self, source, dest): - # copy array with name - copy(source['foo/bar/baz'], dest, name='qux') - assert 'baz' not in dest - check_copied_array(source['foo/bar/baz'], dest['qux']) - - def test_copy_array_create_options(self, source, dest): - dest_h5py = dest.__module__.startswith('h5py.') - - # copy array, provide creation options - compressor = Zlib(9) - create_kws = dict(chunks=(10,)) - if dest_h5py: - create_kws.update(compression='gzip', compression_opts=9, - shuffle=True, fletcher32=True, fillvalue=42) - else: - create_kws.update(compressor=compressor, fill_value=42, order='F', - filters=[Adler32()]) - copy(source['foo/bar/baz'], dest, without_attrs=True, **create_kws) - check_copied_array(source['foo/bar/baz'], dest['baz'], - without_attrs=True, expect_props=create_kws) - - def test_copy_array_exists_array(self, source, dest): - # copy array, dest array in the way - dest.create_dataset('baz', shape=(10,)) - - # raise - with pytest.raises(CopyError): - # should raise by default - copy(source['foo/bar/baz'], dest) - assert (10,) == dest['baz'].shape - with pytest.raises(CopyError): - copy(source['foo/bar/baz'], dest, if_exists='raise') - assert (10,) == dest['baz'].shape - - # skip - copy(source['foo/bar/baz'], dest, if_exists='skip') - assert (10,) == dest['baz'].shape - - # replace - copy(source['foo/bar/baz'], dest, if_exists='replace') - check_copied_array(source['foo/bar/baz'], dest['baz']) - - # invalid option - with pytest.raises(ValueError): - copy(source['foo/bar/baz'], dest, if_exists='foobar') - - def test_copy_array_exists_group(self, source, dest): - # copy array, dest group in the way - dest.create_group('baz') - - # raise - with pytest.raises(CopyError): - copy(source['foo/bar/baz'], dest) - assert not hasattr(dest['baz'], 'shape') - with pytest.raises(CopyError): - copy(source['foo/bar/baz'], dest, if_exists='raise') - assert not hasattr(dest['baz'], 'shape') - - # skip - copy(source['foo/bar/baz'], dest, if_exists='skip') - assert not hasattr(dest['baz'], 'shape') - - # replace - copy(source['foo/bar/baz'], dest, if_exists='replace') - check_copied_array(source['foo/bar/baz'], dest['baz']) - - def test_copy_array_skip_initialized(self, source, dest): - dest_h5py = dest.__module__.startswith('h5py.') - - dest.create_dataset('baz', shape=(100,), chunks=(10,), dtype='i8') - assert not np.all(source['foo/bar/baz'][:] == dest['baz'][:]) - - if dest_h5py: - with pytest.raises(ValueError): - # not available with copy to h5py - copy(source['foo/bar/baz'], dest, if_exists='skip_initialized') - - else: - # copy array, dest array exists but not yet initialized - copy(source['foo/bar/baz'], dest, if_exists='skip_initialized') - check_copied_array(source['foo/bar/baz'], dest['baz']) - - # copy array, dest array exists and initialized, will be skipped - dest['baz'][:] = np.arange(100, 200) - copy(source['foo/bar/baz'], dest, if_exists='skip_initialized') - assert_array_equal(np.arange(100, 200), dest['baz'][:]) - assert not np.all(source['foo/bar/baz'][:] == dest['baz'][:]) - - def test_copy_group(self, source, dest): - # copy group, default options - copy(source['foo'], dest) - check_copied_group(source['foo'], dest['foo']) - - def test_copy_group_no_name(self, source, dest): - with pytest.raises(TypeError): - # need a name if copy root - copy(source, dest) - - copy(source, dest, name='root') - check_copied_group(source, dest['root']) - - def test_copy_group_options(self, source, dest): - # copy group, non-default options - copy(source['foo'], dest, name='qux', without_attrs=True) - assert 'foo' not in dest - check_copied_group(source['foo'], dest['qux'], without_attrs=True) - - def test_copy_group_shallow(self, source, dest): - # copy group, shallow - copy(source, dest, name='eggs', shallow=True) - check_copied_group(source, dest['eggs'], shallow=True) - - def test_copy_group_exists_group(self, source, dest): - # copy group, dest groups exist - dest.create_group('foo/bar') - copy(source['foo'], dest) - check_copied_group(source['foo'], dest['foo']) - - def test_copy_group_exists_array(self, source, dest): - # copy group, dest array in the way - dest.create_dataset('foo/bar', shape=(10,)) - - # raise - with pytest.raises(CopyError): - copy(source['foo'], dest) - assert dest['foo/bar'].shape == (10,) - with pytest.raises(CopyError): - copy(source['foo'], dest, if_exists='raise') - assert dest['foo/bar'].shape == (10,) - - # skip - copy(source['foo'], dest, if_exists='skip') - assert dest['foo/bar'].shape == (10,) - - # replace - copy(source['foo'], dest, if_exists='replace') - check_copied_group(source['foo'], dest['foo']) - - def test_copy_group_dry_run(self, source, dest): - # dry run, empty destination - n_copied, n_skipped, n_bytes_copied = \ - copy(source['foo'], dest, dry_run=True, return_stats=True) - assert 0 == len(dest) - assert 3 == n_copied - assert 0 == n_skipped - assert 0 == n_bytes_copied - - # dry run, array exists in destination - baz = np.arange(100, 200) - dest.create_dataset('foo/bar/baz', data=baz) - assert not np.all(source['foo/bar/baz'][:] == dest['foo/bar/baz'][:]) - assert 1 == len(dest) - - # raise - with pytest.raises(CopyError): - copy(source['foo'], dest, dry_run=True) - assert 1 == len(dest) - - # skip - n_copied, n_skipped, n_bytes_copied = \ - copy(source['foo'], dest, dry_run=True, if_exists='skip', - return_stats=True) - assert 1 == len(dest) - assert 2 == n_copied - assert 1 == n_skipped - assert 0 == n_bytes_copied - assert_array_equal(baz, dest['foo/bar/baz']) - - # replace - n_copied, n_skipped, n_bytes_copied = \ - copy(source['foo'], dest, dry_run=True, if_exists='replace', - return_stats=True) - assert 1 == len(dest) - assert 3 == n_copied - assert 0 == n_skipped - assert 0 == n_bytes_copied - assert_array_equal(baz, dest['foo/bar/baz']) - - def test_logging(self, source, dest, tmpdir): - # callable log - copy(source['foo'], dest, dry_run=True, log=print) - - # file name - fn = str(tmpdir.join('log_name')) - copy(source['foo'], dest, dry_run=True, log=fn) - - # file - with tmpdir.join('log_file').open(mode='w') as f: - copy(source['foo'], dest, dry_run=True, log=f) - - # bad option - with pytest.raises(TypeError): - copy(source['foo'], dest, dry_run=True, log=True) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py deleted file mode 100644 index 9043a32a51..0000000000 --- a/zarr/tests/test_core.py +++ /dev/null @@ -1,2792 +0,0 @@ -import atexit -import os -import sys -import pickle -import shutil -import unittest -from itertools import zip_longest -from tempfile import mkdtemp, mktemp - -import numpy as np -import pytest -from numcodecs import (BZ2, JSON, LZ4, Blosc, Categorize, Delta, - FixedScaleOffset, GZip, MsgPack, Pickle, VLenArray, - VLenBytes, VLenUTF8, Zlib) -from numcodecs.compat import ensure_bytes, ensure_ndarray -from numcodecs.tests.common import greetings -from numpy.testing import assert_array_almost_equal, assert_array_equal -from pkg_resources import parse_version - -from zarr.core import Array -from zarr.meta import json_loads -from zarr.n5 import N5Store, n5_keywords -from zarr.storage import ( - ABSStore, - DBMStore, - DirectoryStore, - LMDBStore, - LRUStoreCache, - NestedDirectoryStore, - SQLiteStore, - FSStore, - atexit_rmglob, - atexit_rmtree, - init_array, - init_group, -) -from zarr.util import buffer_size -from zarr.tests.util import abs_container, skip_test_env_var, have_fsspec - -# noinspection PyMethodMayBeStatic - - -class TestArray(unittest.TestCase): - - def test_array_init(self): - - # normal initialization - store = dict() - init_array(store, shape=100, chunks=10, dtype=' end - assert [] == list(z.islice(6, 5)) - - if hasattr(z.store, 'close'): - z.store.close() - - def test_iter(self): - params = ( - ((1,), (1,)), - ((2,), (1,)), - ((1,), (2,)), - ((3,), (3,)), - ((1000,), (100,)), - ((100,), (1000,)), - ((1, 100), (1, 1)), - ((1, 0), (1, 1)), - ((0, 1), (1, 1)), - ((0, 1), (2, 1)), - ((100, 1), (3, 1)), - ((100, 100), (10, 10)), - ((10, 10, 10), (3, 3, 3)), - ) - for shape, chunks in params: - z = self.create_array(shape=shape, chunks=chunks, dtype=int) - a = np.arange(np.product(shape)).reshape(shape) - z[:] = a - for expect, actual in zip_longest(a, z): - assert_array_equal(expect, actual) - if hasattr(z.store, 'close'): - z.store.close() - - def test_islice(self): - params = ( - ((1,), (1,), 0, 1), - ((2,), (1,), 0, 1), - ((1,), (2,), 0, 1), - ((3,), (3,), 1, 2), - ((1000,), (100,), 150, 1050), - ((100,), (1000,), 25, 75), - ((1, 100), (1, 1), 0, 1), - ((100, 1), (3, 1), 56, 100), - ((100, 100), (10, 10), 13, 99), - ((10, 10, 10), (3, 3, 3), 2, 4), - ) - for shape, chunks, start, end in params: - z = self.create_array(shape=shape, chunks=chunks, dtype=int) - a = np.arange(np.product(shape)).reshape(shape) - z[:] = a - end_array = min(end, a.shape[0]) - for expect, actual in zip_longest(a[start:end_array], - z.islice(start, end)): - assert_array_equal(expect, actual) - if hasattr(z.store, 'close'): - z.store.close() - - def test_compressors(self): - compressors = [ - None, BZ2(), Blosc(), LZ4(), Zlib(), GZip() - ] - if LZMA: - compressors.append(LZMA()) - for compressor in compressors: - a = self.create_array(shape=1000, chunks=100, compressor=compressor) - a[0:100] = 1 - assert np.all(a[0:100] == 1) - a[:] = 1 - assert np.all(a[:] == 1) - if hasattr(a.store, 'close'): - a.store.close() - - def test_endian(self): - dtype = np.dtype('float32') - a1 = self.create_array(shape=1000, chunks=100, dtype=dtype.newbyteorder('<')) - a1[:] = 1 - x1 = a1[:] - a2 = self.create_array(shape=1000, chunks=100, dtype=dtype.newbyteorder('>')) - a2[:] = 1 - x2 = a2[:] - assert_array_equal(x1, x2) - if hasattr(a1.store, 'close'): - a1.store.close() - if hasattr(a2.store, 'close'): - a2.store.close() - - def test_attributes(self): - a = self.create_array(shape=10, chunks=10, dtype='i8') - a.attrs['foo'] = 'bar' - assert a.attrs.key in a.store - attrs = json_loads(a.store[a.attrs.key]) - assert 'foo' in attrs and attrs['foo'] == 'bar' - - a.attrs['bar'] = 'foo' - assert a.attrs.key in a.store - attrs = json_loads(a.store[a.attrs.key]) - assert 'foo' in attrs and attrs['foo'] == 'bar' - assert 'bar' in attrs and attrs['bar'] == 'foo' - if hasattr(a.store, 'close'): - a.store.close() - - def test_structured_with_object(self): - a = self.create_array(fill_value=(0.0, None), - shape=10, - chunks=10, - dtype=[('x', float), ('y', object)], - object_codec=Pickle()) - assert tuple(a[0]) == (0.0, None) - - -class TestArrayWithPath(TestArray): - - @staticmethod - def create_array(read_only=False, **kwargs): - store = dict() - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - init_array(store, path='foo/bar', **kwargs) - return Array(store, path='foo/bar', read_only=read_only, - cache_metadata=cache_metadata, cache_attrs=cache_attrs) - - def test_hexdigest(self): - # Check basic 1-D array - z = self.create_array(shape=(1050,), chunks=100, dtype=' len(array_keys) - assert sorted(array_keys_recurse) == ['baz', 'zab'] - - # test recursive arrays - arrays = list(g1['foo'].arrays(recurse=False)) - arrays_recurse = list(g1['foo'].arrays(recurse=True)) - assert len(arrays_recurse) > len(arrays) - assert 'zab' == arrays_recurse[0][0] - assert g1['foo']['bar']['zab'] == arrays_recurse[0][1] - - if hasattr(g1.store, 'close'): - g1.store.close() - - def test_getattr(self): - # setup - g1 = self.create_group() - g2 = g1.create_group('foo') - g2.create_dataset('bar', shape=100) - - # test - assert g1['foo'] == g1.foo - assert g2['bar'] == g2.bar - # test that hasattr returns False instead of an exception (issue #88) - assert not hasattr(g1, 'unexistingattribute') - - if hasattr(g1.store, 'close'): - g1.store.close() - - def test_setitem(self): - g = self.create_group() - try: - data = np.arange(100) - g['foo'] = data - assert_array_equal(data, g['foo']) - data = np.arange(200) - g['foo'] = data - assert_array_equal(data, g['foo']) - # 0d array - g['foo'] = 42 - assert () == g['foo'].shape - assert 42 == g['foo'][()] - except NotImplementedError: - pass - if hasattr(g.store, 'close'): - g.store.close() - - def test_delitem(self): - g = self.create_group() - g.create_group('foo') - g.create_dataset('bar/baz', shape=100, chunks=10) - assert 'foo' in g - assert 'bar' in g - assert 'bar/baz' in g - try: - del g['bar'] - with pytest.raises(KeyError): - del g['xxx'] - except NotImplementedError: - pass - else: - assert 'foo' in g - assert 'bar' not in g - assert 'bar/baz' not in g - if hasattr(g.store, 'close'): - g.store.close() - - def test_move(self): - g = self.create_group() - - data = np.arange(100) - g['boo'] = data - - data = np.arange(100) - g['foo'] = data - - try: - g.move('foo', 'bar') - assert 'foo' not in g - assert 'bar' in g - assert_array_equal(data, g['bar']) - - g.move('bar', 'foo/bar') - assert 'bar' not in g - assert 'foo' in g - assert 'foo/bar' in g - assert isinstance(g['foo'], Group) - assert_array_equal(data, g['foo/bar']) - - g.move('foo', 'foo2') - assert 'foo' not in g - assert 'foo/bar' not in g - assert 'foo2' in g - assert 'foo2/bar' in g - assert isinstance(g['foo2'], Group) - assert_array_equal(data, g['foo2/bar']) - - g2 = g['foo2'] - g2.move('bar', '/bar') - assert 'foo2' in g - assert 'foo2/bar' not in g - assert 'bar' in g - assert isinstance(g['foo2'], Group) - assert_array_equal(data, g['bar']) - - with pytest.raises(ValueError): - g2.move('bar', 'bar2') - - with pytest.raises(ValueError): - g.move('bar', 'boo') - except NotImplementedError: - pass - - if hasattr(g.store, 'close'): - g.store.close() - - def test_array_creation(self): - grp = self.create_group() - - a = grp.create('a', shape=100, chunks=10) - assert isinstance(a, Array) - b = grp.empty('b', shape=100, chunks=10) - assert isinstance(b, Array) - assert b.fill_value is None - c = grp.zeros('c', shape=100, chunks=10) - assert isinstance(c, Array) - assert 0 == c.fill_value - d = grp.ones('d', shape=100, chunks=10) - assert isinstance(d, Array) - assert 1 == d.fill_value - e = grp.full('e', shape=100, chunks=10, fill_value=42) - assert isinstance(e, Array) - assert 42 == e.fill_value - - f = grp.empty_like('f', a) - assert isinstance(f, Array) - assert f.fill_value is None - g = grp.zeros_like('g', a) - assert isinstance(g, Array) - assert 0 == g.fill_value - h = grp.ones_like('h', a) - assert isinstance(h, Array) - assert 1 == h.fill_value - i = grp.full_like('i', e) - assert isinstance(i, Array) - assert 42 == i.fill_value - - j = grp.array('j', data=np.arange(100), chunks=10) - assert isinstance(j, Array) - assert_array_equal(np.arange(100), j[:]) - - if hasattr(grp.store, 'close'): - grp.store.close() - - grp = self.create_group(read_only=True) - with pytest.raises(PermissionError): - grp.create('aa', shape=100, chunks=10) - with pytest.raises(PermissionError): - grp.empty('aa', shape=100, chunks=10) - with pytest.raises(PermissionError): - grp.zeros('aa', shape=100, chunks=10) - with pytest.raises(PermissionError): - grp.ones('aa', shape=100, chunks=10) - with pytest.raises(PermissionError): - grp.full('aa', shape=100, chunks=10, fill_value=42) - with pytest.raises(PermissionError): - grp.array('aa', data=np.arange(100), chunks=10) - with pytest.raises(PermissionError): - grp.create('aa', shape=100, chunks=10) - with pytest.raises(PermissionError): - grp.empty_like('aa', a) - with pytest.raises(PermissionError): - grp.zeros_like('aa', a) - with pytest.raises(PermissionError): - grp.ones_like('aa', a) - with pytest.raises(PermissionError): - grp.full_like('aa', a) - - if hasattr(grp.store, 'close'): - grp.store.close() - - def test_paths(self): - g1 = self.create_group() - g2 = g1.create_group('foo/bar') - - assert g1 == g1['/'] - assert g1 == g1['//'] - assert g1 == g1['///'] - assert g1 == g2['/'] - assert g1 == g2['//'] - assert g1 == g2['///'] - assert g2 == g1['foo/bar'] - assert g2 == g1['/foo/bar'] - assert g2 == g1['foo/bar/'] - assert g2 == g1['//foo/bar'] - assert g2 == g1['//foo//bar//'] - assert g2 == g1['///foo///bar///'] - assert g2 == g2['/foo/bar'] - - with pytest.raises(ValueError): - g1['.'] - with pytest.raises(ValueError): - g1['..'] - with pytest.raises(ValueError): - g1['foo/.'] - with pytest.raises(ValueError): - g1['foo/..'] - with pytest.raises(ValueError): - g1['foo/./bar'] - with pytest.raises(ValueError): - g1['foo/../bar'] - - if hasattr(g1.store, 'close'): - g1.store.close() - - def test_pickle(self): - - # setup group - g = self.create_group() - d = g.create_dataset('foo/bar', shape=100, chunks=10) - d[:] = np.arange(100) - path = g.path - name = g.name - n = len(g) - keys = list(g) - - # round-trip through pickle - dump = pickle.dumps(g) - # some stores cannot be opened twice at the same time, need to close - # store before can round-trip through pickle - if hasattr(g.store, 'close'): - g.store.close() - g2 = pickle.loads(dump) - - # verify - assert path == g2.path - assert name == g2.name - assert n == len(g2) - assert keys == list(g2) - assert isinstance(g2['foo'], Group) - assert isinstance(g2['foo/bar'], Array) - - if hasattr(g2.store, 'close'): - g2.store.close() - - def test_context_manager(self): - - with self.create_group() as g: - d = g.create_dataset('foo/bar', shape=100, chunks=10) - d[:] = np.arange(100) - - -class TestGroupWithMemoryStore(TestGroup): - - @staticmethod - def create_store(): - return MemoryStore(), None - - -class TestGroupWithDirectoryStore(TestGroup): - - @staticmethod - def create_store(): - path = tempfile.mkdtemp() - atexit.register(atexit_rmtree, path) - store = DirectoryStore(path) - return store, None - - -@skip_test_env_var("ZARR_TEST_ABS") -class TestGroupWithABSStore(TestGroup): - - @staticmethod - def create_store(): - container_client = abs_container() - store = ABSStore(client=container_client) - store.rmdir() - return store, None - - @pytest.mark.skipif(sys.version_info < (3, 7), reason="attr not serializable in py36") - def test_pickle(self): - # internal attribute on ContainerClient isn't serializable for py36 and earlier - super().test_pickle() - - -class TestGroupWithNestedDirectoryStore(TestGroup): - - @staticmethod - def create_store(): - path = tempfile.mkdtemp() - atexit.register(atexit_rmtree, path) - store = NestedDirectoryStore(path) - return store, None - - -@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") -class TestGroupWithFSStore(TestGroup): - - @staticmethod - def create_store(): - path = tempfile.mkdtemp() - atexit.register(atexit_rmtree, path) - store = FSStore(path) - return store, None - - def test_round_trip_nd(self): - data = np.arange(1000).reshape(10, 10, 10) - name = 'raw' - - store, _ = self.create_store() - f = open_group(store, mode='w') - f.create_dataset(name, data=data, chunks=(5, 5, 5), - compressor=None) - h = open_group(store, mode='r') - np.testing.assert_array_equal(h[name][:], data) - - -@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") -class TestGroupWithNestedFSStore(TestGroupWithFSStore): - - @staticmethod - def create_store(): - path = tempfile.mkdtemp() - atexit.register(atexit_rmtree, path) - store = FSStore(path, key_separator='/', auto_mkdir=True) - return store, None - - def test_inconsistent_dimension_separator(self): - data = np.arange(1000).reshape(10, 10, 10) - name = 'raw' - - store, _ = self.create_store() - f = open_group(store, mode='w') - - # cannot specify dimension_separator that conflicts with the store - with pytest.raises(ValueError): - f.create_dataset(name, data=data, chunks=(5, 5, 5), - compressor=None, dimension_separator='.') - - -class TestGroupWithZipStore(TestGroup): - - @staticmethod - def create_store(): - path = tempfile.mktemp(suffix='.zip') - atexit.register(os.remove, path) - store = ZipStore(path) - return store, None - - def test_context_manager(self): - - with self.create_group() as g: - store = g.store - d = g.create_dataset('foo/bar', shape=100, chunks=10) - d[:] = np.arange(100) - - # Check that exiting the context manager closes the store, - # and therefore the underlying ZipFile. - with pytest.raises(ValueError): - store.zf.extractall() - - -class TestGroupWithDBMStore(TestGroup): - - @staticmethod - def create_store(): - path = tempfile.mktemp(suffix='.anydbm') - atexit.register(atexit_rmglob, path + '*') - store = DBMStore(path, flag='n') - return store, None - - -class TestGroupWithDBMStoreBerkeleyDB(TestGroup): - - @staticmethod - def create_store(): - bsddb3 = pytest.importorskip("bsddb3") - path = tempfile.mktemp(suffix='.dbm') - atexit.register(os.remove, path) - store = DBMStore(path, flag='n', open=bsddb3.btopen) - return store, None - - -class TestGroupWithLMDBStore(TestGroup): - - @staticmethod - def create_store(): - pytest.importorskip("lmdb") - path = tempfile.mktemp(suffix='.lmdb') - atexit.register(atexit_rmtree, path) - store = LMDBStore(path) - return store, None - - -class TestGroupWithSQLiteStore(TestGroup): - - def create_store(self): - pytest.importorskip("sqlite3") - path = tempfile.mktemp(suffix='.db') - atexit.register(atexit_rmtree, path) - store = SQLiteStore(path) - return store, None - - -class TestGroupWithChunkStore(TestGroup): - - @staticmethod - def create_store(): - return dict(), dict() - - def test_chunk_store(self): - # setup - store, chunk_store = self.create_store() - g = self.create_group(store, chunk_store=chunk_store) - - # check attributes - assert store is g.store - assert chunk_store is g.chunk_store - - # create array - a = g.zeros('foo', shape=100, chunks=10) - assert store is a.store - assert chunk_store is a.chunk_store - a[:] = np.arange(100) - assert_array_equal(np.arange(100), a[:]) - - # check store keys - expect = sorted([group_meta_key, 'foo/' + array_meta_key]) - actual = sorted(store.keys()) - assert expect == actual - expect = ['foo/' + str(i) for i in range(10)] - actual = sorted(chunk_store.keys()) - assert expect == actual - - -class TestGroupWithStoreCache(TestGroup): - - @staticmethod - def create_store(): - store = LRUStoreCache(dict(), max_size=None) - return store, None - - -def test_group(): - # test the group() convenience function - - # basic usage - g = group() - assert isinstance(g, Group) - assert '' == g.path - assert '/' == g.name - - # usage with custom store - store = dict() - g = group(store=store) - assert isinstance(g, Group) - assert store is g.store - - # overwrite behaviour - store = dict() - init_array(store, shape=100, chunks=10) - with pytest.raises(ValueError): - group(store) - g = group(store, overwrite=True) - assert isinstance(g, Group) - assert store is g.store - - -def test_open_group(): - # test the open_group() convenience function - - store = 'data/group.zarr' - - # mode == 'w' - g = open_group(store, mode='w') - assert isinstance(g, Group) - assert isinstance(g.store, DirectoryStore) - assert 0 == len(g) - g.create_groups('foo', 'bar') - assert 2 == len(g) - - # mode in 'r', 'r+' - open_array('data/array.zarr', shape=100, chunks=10, mode='w') - for mode in 'r', 'r+': - with pytest.raises(ValueError): - open_group('doesnotexist', mode=mode) - with pytest.raises(ValueError): - open_group('data/array.zarr', mode=mode) - g = open_group(store, mode='r') - assert isinstance(g, Group) - assert 2 == len(g) - with pytest.raises(PermissionError): - g.create_group('baz') - g = open_group(store, mode='r+') - assert isinstance(g, Group) - assert 2 == len(g) - g.create_groups('baz', 'quux') - assert 4 == len(g) - - # mode == 'a' - shutil.rmtree(store) - g = open_group(store, mode='a') - assert isinstance(g, Group) - assert isinstance(g.store, DirectoryStore) - assert 0 == len(g) - g.create_groups('foo', 'bar') - assert 2 == len(g) - with pytest.raises(ValueError): - open_group('data/array.zarr', mode='a') - - # mode in 'w-', 'x' - for mode in 'w-', 'x': - shutil.rmtree(store) - g = open_group(store, mode=mode) - assert isinstance(g, Group) - assert isinstance(g.store, DirectoryStore) - assert 0 == len(g) - g.create_groups('foo', 'bar') - assert 2 == len(g) - with pytest.raises(ValueError): - open_group(store, mode=mode) - with pytest.raises(ValueError): - open_group('data/array.zarr', mode=mode) - - # open with path - g = open_group(store, path='foo/bar') - assert isinstance(g, Group) - assert 'foo/bar' == g.path - - -def test_group_completions(): - g = group() - d = dir(g) - assert 'foo' not in d - assert 'bar' not in d - assert 'baz' not in d - assert 'qux' not in d - assert 'xxx' not in d - assert 'yyy' not in d - assert 'zzz' not in d - assert '123' not in d - assert '456' not in d - g.create_groups('foo', 'bar', 'baz/qux', '123') - g.zeros('xxx', shape=100) - g.zeros('yyy', shape=100) - g.zeros('zzz', shape=100) - g.zeros('456', shape=100) - d = dir(g) - assert 'foo' in d - assert 'bar' in d - assert 'baz' in d - assert 'qux' not in d - assert 'xxx' in d - assert 'yyy' in d - assert 'zzz' in d - assert '123' not in d # not valid identifier - assert '456' not in d # not valid identifier - - -def test_group_key_completions(): - g = group() - d = dir(g) - # noinspection PyProtectedMember - k = g._ipython_key_completions_() - - # none of these names should be an attribute - assert 'foo' not in d - assert 'bar' not in d - assert 'baz' not in d - assert 'qux' not in d - assert 'xxx' not in d - assert 'yyy' not in d - assert 'zzz' not in d - assert '123' not in d - assert '456' not in d - assert 'asdf;' not in d - - # none of these names should be an item - assert 'foo' not in k - assert 'bar' not in k - assert 'baz' not in k - assert 'qux' not in k - assert 'xxx' not in k - assert 'yyy' not in k - assert 'zzz' not in k - assert '123' not in k - assert '456' not in k - assert 'asdf;' not in k - - g.create_groups('foo', 'bar', 'baz/qux', '123') - g.zeros('xxx', shape=100) - g.zeros('yyy', shape=100) - g.zeros('zzz', shape=100) - g.zeros('456', shape=100) - g.zeros('asdf;', shape=100) - - d = dir(g) - # noinspection PyProtectedMember - k = g._ipython_key_completions_() - - assert 'foo' in d - assert 'bar' in d - assert 'baz' in d - assert 'qux' not in d - assert 'xxx' in d - assert 'yyy' in d - assert 'zzz' in d - assert '123' not in d # not valid identifier - assert '456' not in d # not valid identifier - assert 'asdf;' not in d # not valid identifier - - assert 'foo' in k - assert 'bar' in k - assert 'baz' in k - assert 'qux' not in k - assert 'xxx' in k - assert 'yyy' in k - assert 'zzz' in k - assert '123' in k - assert '456' in k - assert 'asdf;' in k - - -def _check_tree(g, expect_bytes, expect_text): - assert expect_bytes == bytes(g.tree()) - assert expect_text == str(g.tree()) - expect_repr = expect_text - assert expect_repr == repr(g.tree()) - if ipytree: - # noinspection PyProtectedMember - widget = g.tree()._ipython_display_() - isinstance(widget, ipytree.Tree) - - -def test_tree(): - # setup - g1 = group() - g2 = g1.create_group('foo') - g3 = g1.create_group('bar') - g3.create_group('baz') - g5 = g3.create_group('quux') - g5.create_dataset('baz', shape=100, chunks=10) - - # test root group - expect_bytes = textwrap.dedent("""\ - / - +-- bar - | +-- baz - | +-- quux - | +-- baz (100,) float64 - +-- foo""").encode() - expect_text = textwrap.dedent("""\ - / - ├── bar - │ ├── baz - │ └── quux - │ └── baz (100,) float64 - └── foo""") - _check_tree(g1, expect_bytes, expect_text) - - # test different group - expect_bytes = textwrap.dedent("""\ - foo""").encode() - expect_text = textwrap.dedent("""\ - foo""") - _check_tree(g2, expect_bytes, expect_text) - - # test different group - expect_bytes = textwrap.dedent("""\ - bar - +-- baz - +-- quux - +-- baz (100,) float64""").encode() - expect_text = textwrap.dedent("""\ - bar - ├── baz - └── quux - └── baz (100,) float64""") - _check_tree(g3, expect_bytes, expect_text) diff --git a/zarr/tests/test_indexing.py b/zarr/tests/test_indexing.py deleted file mode 100644 index 8c534f8e4a..0000000000 --- a/zarr/tests/test_indexing.py +++ /dev/null @@ -1,1375 +0,0 @@ -import numpy as np -import pytest -from numpy.testing import assert_array_equal - -import zarr -from zarr.indexing import ( - normalize_integer_selection, - oindex, - oindex_set, - replace_ellipsis, - PartialChunkIterator, -) - - -def test_normalize_integer_selection(): - - assert 1 == normalize_integer_selection(1, 100) - assert 99 == normalize_integer_selection(-1, 100) - with pytest.raises(IndexError): - normalize_integer_selection(100, 100) - with pytest.raises(IndexError): - normalize_integer_selection(1000, 100) - with pytest.raises(IndexError): - normalize_integer_selection(-1000, 100) - - -def test_replace_ellipsis(): - - # 1D, single item - assert (0,) == replace_ellipsis(0, (100,)) - - # 1D - assert (slice(None),) == replace_ellipsis(Ellipsis, (100,)) - assert (slice(None),) == replace_ellipsis(slice(None), (100,)) - assert (slice(None, 100),) == replace_ellipsis(slice(None, 100), (100,)) - assert (slice(0, None),) == replace_ellipsis(slice(0, None), (100,)) - assert (slice(None),) == replace_ellipsis((slice(None), Ellipsis), (100,)) - assert (slice(None),) == replace_ellipsis((Ellipsis, slice(None)), (100,)) - - # 2D, single item - assert (0, 0) == replace_ellipsis((0, 0), (100, 100)) - assert (-1, 1) == replace_ellipsis((-1, 1), (100, 100)) - - # 2D, single col/row - assert (0, slice(None)) == replace_ellipsis((0, slice(None)), (100, 100)) - assert (0, slice(None)) == replace_ellipsis((0,), (100, 100)) - assert (slice(None), 0) == replace_ellipsis((slice(None), 0), (100, 100)) - - # 2D slice - assert ((slice(None), slice(None)) == - replace_ellipsis(Ellipsis, (100, 100))) - assert ((slice(None), slice(None)) == - replace_ellipsis(slice(None), (100, 100))) - assert ((slice(None), slice(None)) == - replace_ellipsis((slice(None), slice(None)), (100, 100))) - assert ((slice(None), slice(None)) == - replace_ellipsis((Ellipsis, slice(None)), (100, 100))) - assert ((slice(None), slice(None)) == - replace_ellipsis((slice(None), Ellipsis), (100, 100))) - assert ((slice(None), slice(None)) == - replace_ellipsis((slice(None), Ellipsis, slice(None)), (100, 100))) - assert ((slice(None), slice(None)) == - replace_ellipsis((Ellipsis, slice(None), slice(None)), (100, 100))) - assert ((slice(None), slice(None)) == - replace_ellipsis((slice(None), slice(None), Ellipsis), (100, 100))) - - -def test_get_basic_selection_0d(): - - # setup - a = np.array(42) - z = zarr.create(shape=a.shape, dtype=a.dtype, fill_value=None) - z[...] = a - - assert_array_equal(a, z.get_basic_selection(Ellipsis)) - assert_array_equal(a, z[...]) - assert 42 == z.get_basic_selection(()) - assert 42 == z[()] - - # test out param - b = np.zeros_like(a) - z.get_basic_selection(Ellipsis, out=b) - assert_array_equal(a, b) - - # test structured array - value = (b'aaa', 1, 4.2) - a = np.array(value, dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - z = zarr.create(shape=a.shape, dtype=a.dtype, fill_value=None) - z[()] = value - assert_array_equal(a, z.get_basic_selection(Ellipsis)) - assert_array_equal(a, z[...]) - assert a[()] == z.get_basic_selection(()) - assert a[()] == z[()] - assert b'aaa' == z.get_basic_selection((), fields='foo') - assert b'aaa' == z['foo'] - assert a[['foo', 'bar']] == z.get_basic_selection((), fields=['foo', 'bar']) - assert a[['foo', 'bar']] == z['foo', 'bar'] - # test out param - b = np.zeros_like(a) - z.get_basic_selection(Ellipsis, out=b) - assert_array_equal(a, b) - c = np.zeros_like(a[['foo', 'bar']]) - z.get_basic_selection(Ellipsis, out=c, fields=['foo', 'bar']) - assert_array_equal(a[['foo', 'bar']], c) - - -basic_selections_1d = [ - # single value - 42, - -1, - # slices - slice(0, 1050), - slice(50, 150), - slice(0, 2000), - slice(-150, -50), - slice(-2000, 2000), - slice(0, 0), # empty result - slice(-1, 0), # empty result - # total selections - slice(None), - Ellipsis, - (), - (Ellipsis, slice(None)), - # slice with step - slice(None), - slice(None, None), - slice(None, None, 1), - slice(None, None, 10), - slice(None, None, 100), - slice(None, None, 1000), - slice(None, None, 10000), - slice(0, 1050), - slice(0, 1050, 1), - slice(0, 1050, 10), - slice(0, 1050, 100), - slice(0, 1050, 1000), - slice(0, 1050, 10000), - slice(1, 31, 3), - slice(1, 31, 30), - slice(1, 31, 300), - slice(81, 121, 3), - slice(81, 121, 30), - slice(81, 121, 300), - slice(50, 150), - slice(50, 150, 1), - slice(50, 150, 10), -] - - -basic_selections_1d_bad = [ - # only positive step supported - slice(None, None, -1), - slice(None, None, -10), - slice(None, None, -100), - slice(None, None, -1000), - slice(None, None, -10000), - slice(1050, -1, -1), - slice(1050, -1, -10), - slice(1050, -1, -100), - slice(1050, -1, -1000), - slice(1050, -1, -10000), - slice(1050, 0, -1), - slice(1050, 0, -10), - slice(1050, 0, -100), - slice(1050, 0, -1000), - slice(1050, 0, -10000), - slice(150, 50, -1), - slice(150, 50, -10), - slice(31, 1, -3), - slice(121, 81, -3), - slice(-1, 0, -1), - # bad stuff - 2.3, - 'foo', - b'xxx', - None, - (0, 0), - (slice(None), slice(None)), -] - - -def _test_get_basic_selection(a, z, selection): - expect = a[selection] - actual = z.get_basic_selection(selection) - assert_array_equal(expect, actual) - actual = z[selection] - assert_array_equal(expect, actual) - - -# noinspection PyStatementEffect -def test_get_basic_selection_1d(): - - # setup - a = np.arange(1050, dtype=int) - z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) - z[:] = a - - for selection in basic_selections_1d: - _test_get_basic_selection(a, z, selection) - - bad_selections = basic_selections_1d_bad + [ - [0, 1], # fancy indexing - ] - for selection in bad_selections: - with pytest.raises(IndexError): - z.get_basic_selection(selection) - with pytest.raises(IndexError): - z[selection] - - -basic_selections_2d = [ - # single row - 42, - -1, - (42, slice(None)), - (-1, slice(None)), - # single col - (slice(None), 4), - (slice(None), -1), - # row slices - slice(None), - slice(0, 1000), - slice(250, 350), - slice(0, 2000), - slice(-350, -250), - slice(0, 0), # empty result - slice(-1, 0), # empty result - slice(-2000, 0), - slice(-2000, 2000), - # 2D slices - (slice(None), slice(1, 5)), - (slice(250, 350), slice(None)), - (slice(250, 350), slice(1, 5)), - (slice(250, 350), slice(-5, -1)), - (slice(250, 350), slice(-50, 50)), - (slice(250, 350, 10), slice(1, 5)), - (slice(250, 350), slice(1, 5, 2)), - (slice(250, 350, 33), slice(1, 5, 3)), - # total selections - (slice(None), slice(None)), - Ellipsis, - (), - (Ellipsis, slice(None)), - (Ellipsis, slice(None), slice(None)), -] - - -basic_selections_2d_bad = [ - # bad stuff - 2.3, - 'foo', - b'xxx', - None, - (2.3, slice(None)), - # only positive step supported - slice(None, None, -1), - (slice(None, None, -1), slice(None)), - (0, 0, 0), - (slice(None), slice(None), slice(None)), -] - - -# noinspection PyStatementEffect -def test_get_basic_selection_2d(): - - # setup - a = np.arange(10000, dtype=int).reshape(1000, 10) - z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) - z[:] = a - - for selection in basic_selections_2d: - _test_get_basic_selection(a, z, selection) - - bad_selections = basic_selections_2d_bad + [ - # integer arrays - [0, 1], - ([0, 1], [0, 1]), - (slice(None), [0, 1]), - ] - for selection in bad_selections: - with pytest.raises(IndexError): - z.get_basic_selection(selection) - with pytest.raises(IndexError): - z[selection] - - -def test_set_basic_selection_0d(): - - # setup - v = np.array(42) - a = np.zeros_like(v) - z = zarr.zeros_like(v) - assert_array_equal(a, z) - - # tests - z.set_basic_selection(Ellipsis, v) - assert_array_equal(v, z) - z[...] = 0 - assert_array_equal(a, z) - z[...] = v - assert_array_equal(v, z) - - # test structured array - value = (b'aaa', 1, 4.2) - v = np.array(value, dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - a = np.zeros_like(v) - z = zarr.create(shape=a.shape, dtype=a.dtype, fill_value=None) - - # tests - z.set_basic_selection(Ellipsis, v) - assert_array_equal(v, z) - z.set_basic_selection(Ellipsis, a) - assert_array_equal(a, z) - z[...] = v - assert_array_equal(v, z) - z[...] = a - assert_array_equal(a, z) - # with fields - z.set_basic_selection(Ellipsis, v['foo'], fields='foo') - assert v['foo'] == z['foo'] - assert a['bar'] == z['bar'] - assert a['baz'] == z['baz'] - z['bar'] = v['bar'] - assert v['foo'] == z['foo'] - assert v['bar'] == z['bar'] - assert a['baz'] == z['baz'] - # multiple field assignment not supported - with pytest.raises(IndexError): - z.set_basic_selection(Ellipsis, v[['foo', 'bar']], fields=['foo', 'bar']) - with pytest.raises(IndexError): - z[..., 'foo', 'bar'] = v[['foo', 'bar']] - - -def _test_get_orthogonal_selection(a, z, selection): - expect = oindex(a, selection) - actual = z.get_orthogonal_selection(selection) - assert_array_equal(expect, actual) - actual = z.oindex[selection] - assert_array_equal(expect, actual) - - -# noinspection PyStatementEffect -def test_get_orthogonal_selection_1d_bool(): - - # setup - a = np.arange(1050, dtype=int) - z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) - z[:] = a - - np.random.seed(42) - # test with different degrees of sparseness - for p in 0.5, 0.1, 0.01: - ix = np.random.binomial(1, p, size=a.shape[0]).astype(bool) - _test_get_orthogonal_selection(a, z, ix) - - # test errors - with pytest.raises(IndexError): - z.oindex[np.zeros(50, dtype=bool)] # too short - with pytest.raises(IndexError): - z.oindex[np.zeros(2000, dtype=bool)] # too long - with pytest.raises(IndexError): - z.oindex[[[True, False], [False, True]]] # too many dimensions - - -# noinspection PyStatementEffect -def test_get_orthogonal_selection_1d_int(): - - # setup - a = np.arange(1050, dtype=int) - z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) - z[:] = a - - np.random.seed(42) - # test with different degrees of sparseness - for p in 2, 0.5, 0.1, 0.01: - # unordered - ix = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) - _test_get_orthogonal_selection(a, z, ix) - # increasing - ix.sort() - _test_get_orthogonal_selection(a, z, ix) - # decreasing - ix = ix[::-1] - _test_get_orthogonal_selection(a, z, ix) - - selections = basic_selections_1d + [ - # test wraparound - [0, 3, 10, -23, -12, -1], - # explicit test not sorted - [3, 105, 23, 127], - - ] - for selection in selections: - _test_get_orthogonal_selection(a, z, selection) - - bad_selections = basic_selections_1d_bad + [ - [a.shape[0] + 1], # out of bounds - [-(a.shape[0] + 1)], # out of bounds - [[2, 4], [6, 8]], # too many dimensions - ] - for selection in bad_selections: - with pytest.raises(IndexError): - z.get_orthogonal_selection(selection) - with pytest.raises(IndexError): - z.oindex[selection] - - -def _test_get_orthogonal_selection_2d(a, z, ix0, ix1): - selections = [ - # index both axes with array - (ix0, ix1), - # mixed indexing with array / slice - (ix0, slice(1, 5)), - (ix0, slice(1, 5, 2)), - (slice(250, 350), ix1), - (slice(250, 350, 10), ix1), - # mixed indexing with array / int - (ix0, 4), - (42, ix1), - ] - for selection in selections: - _test_get_orthogonal_selection(a, z, selection) - - -# noinspection PyStatementEffect -def test_get_orthogonal_selection_2d(): - - # setup - a = np.arange(10000, dtype=int).reshape(1000, 10) - z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) - z[:] = a - - np.random.seed(42) - # test with different degrees of sparseness - for p in 0.5, 0.1, 0.01: - - # boolean arrays - ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) - ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) - _test_get_orthogonal_selection_2d(a, z, ix0, ix1) - - # mixed int array / bool array - selections = ( - (ix0, np.nonzero(ix1)[0]), - (np.nonzero(ix0)[0], ix1), - ) - for selection in selections: - _test_get_orthogonal_selection(a, z, selection) - - # integer arrays - ix0 = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) - ix1 = np.random.choice(a.shape[1], size=int(a.shape[1] * .5), replace=True) - _test_get_orthogonal_selection_2d(a, z, ix0, ix1) - ix0.sort() - ix1.sort() - _test_get_orthogonal_selection_2d(a, z, ix0, ix1) - ix0 = ix0[::-1] - ix1 = ix1[::-1] - _test_get_orthogonal_selection_2d(a, z, ix0, ix1) - - for selection in basic_selections_2d: - _test_get_orthogonal_selection(a, z, selection) - - for selection in basic_selections_2d_bad: - with pytest.raises(IndexError): - z.get_orthogonal_selection(selection) - with pytest.raises(IndexError): - z.oindex[selection] - - -def _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2): - selections = [ - # single value - (84, 42, 4), - (-1, -1, -1), - # index all axes with array - (ix0, ix1, ix2), - # mixed indexing with single array / slices - (ix0, slice(15, 25), slice(1, 5)), - (slice(50, 70), ix1, slice(1, 5)), - (slice(50, 70), slice(15, 25), ix2), - (ix0, slice(15, 25, 5), slice(1, 5, 2)), - (slice(50, 70, 3), ix1, slice(1, 5, 2)), - (slice(50, 70, 3), slice(15, 25, 5), ix2), - # mixed indexing with single array / ints - (ix0, 42, 4), - (84, ix1, 4), - (84, 42, ix2), - # mixed indexing with single array / slice / int - (ix0, slice(15, 25), 4), - (42, ix1, slice(1, 5)), - (slice(50, 70), 42, ix2), - # mixed indexing with two array / slice - (ix0, ix1, slice(1, 5)), - (slice(50, 70), ix1, ix2), - (ix0, slice(15, 25), ix2), - # mixed indexing with two array / integer - (ix0, ix1, 4), - (42, ix1, ix2), - (ix0, 42, ix2), - ] - for selection in selections: - _test_get_orthogonal_selection(a, z, selection) - - -def test_get_orthogonal_selection_3d(): - - # setup - a = np.arange(100000, dtype=int).reshape(200, 50, 10) - z = zarr.create(shape=a.shape, chunks=(60, 20, 3), dtype=a.dtype) - z[:] = a - - np.random.seed(42) - # test with different degrees of sparseness - for p in 0.5, 0.1, 0.01: - - # boolean arrays - ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) - ix1 = np.random.binomial(1, .5, size=a.shape[1]).astype(bool) - ix2 = np.random.binomial(1, .5, size=a.shape[2]).astype(bool) - _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2) - - # integer arrays - ix0 = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) - ix1 = np.random.choice(a.shape[1], size=int(a.shape[1] * .5), replace=True) - ix2 = np.random.choice(a.shape[2], size=int(a.shape[2] * .5), replace=True) - _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2) - ix0.sort() - ix1.sort() - ix2.sort() - _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2) - ix0 = ix0[::-1] - ix1 = ix1[::-1] - ix2 = ix2[::-1] - _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2) - - -def test_orthogonal_indexing_edge_cases(): - - a = np.arange(6).reshape(1, 2, 3) - z = zarr.create(shape=a.shape, chunks=(1, 2, 3), dtype=a.dtype) - z[:] = a - - expect = oindex(a, (0, slice(None), [0, 1, 2])) - actual = z.oindex[0, :, [0, 1, 2]] - assert_array_equal(expect, actual) - - expect = oindex(a, (0, slice(None), [True, True, True])) - actual = z.oindex[0, :, [True, True, True]] - assert_array_equal(expect, actual) - - -def _test_set_orthogonal_selection(v, a, z, selection): - for value in 42, oindex(v, selection), oindex(v, selection).tolist(): - if isinstance(value, list) and value == []: - # skip these cases as cannot preserve all dimensions - continue - # setup expectation - a[:] = 0 - oindex_set(a, selection, value) - # long-form API - z[:] = 0 - z.set_orthogonal_selection(selection, value) - assert_array_equal(a, z[:]) - # short-form API - z[:] = 0 - z.oindex[selection] = value - assert_array_equal(a, z[:]) - - -def test_set_orthogonal_selection_1d(): - - # setup - v = np.arange(1050, dtype=int) - a = np.empty(v.shape, dtype=int) - z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) - - # test with different degrees of sparseness - np.random.seed(42) - for p in 0.5, 0.1, 0.01: - - # boolean arrays - ix = np.random.binomial(1, p, size=a.shape[0]).astype(bool) - _test_set_orthogonal_selection(v, a, z, ix) - - # integer arrays - ix = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) - _test_set_orthogonal_selection(v, a, z, ix) - ix.sort() - _test_set_orthogonal_selection(v, a, z, ix) - ix = ix[::-1] - _test_set_orthogonal_selection(v, a, z, ix) - - # basic selections - for selection in basic_selections_1d: - _test_set_orthogonal_selection(v, a, z, selection) - - -def _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1): - - selections = [ - # index both axes with array - (ix0, ix1), - # mixed indexing with array / slice or int - (ix0, slice(1, 5)), - (slice(250, 350), ix1), - (ix0, 4), - (42, ix1), - ] - for selection in selections: - _test_set_orthogonal_selection(v, a, z, selection) - - -def test_set_orthogonal_selection_2d(): - - # setup - v = np.arange(10000, dtype=int).reshape(1000, 10) - a = np.empty_like(v) - z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) - - np.random.seed(42) - # test with different degrees of sparseness - for p in 0.5, 0.1, 0.01: - - # boolean arrays - ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) - ix1 = np.random.binomial(1, .5, size=a.shape[1]).astype(bool) - _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1) - - # integer arrays - ix0 = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) - ix1 = np.random.choice(a.shape[1], size=int(a.shape[1] * .5), replace=True) - _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1) - ix0.sort() - ix1.sort() - _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1) - ix0 = ix0[::-1] - ix1 = ix1[::-1] - _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1) - - for selection in basic_selections_2d: - _test_set_orthogonal_selection(v, a, z, selection) - - -def _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2): - - selections = ( - # single value - (84, 42, 4), - (-1, -1, -1), - # index all axes with bool array - (ix0, ix1, ix2), - # mixed indexing with single bool array / slice or int - (ix0, slice(15, 25), slice(1, 5)), - (slice(50, 70), ix1, slice(1, 5)), - (slice(50, 70), slice(15, 25), ix2), - (ix0, 42, 4), - (84, ix1, 4), - (84, 42, ix2), - (ix0, slice(15, 25), 4), - (slice(50, 70), ix1, 4), - (slice(50, 70), 42, ix2), - # indexing with two arrays / slice - (ix0, ix1, slice(1, 5)), - # indexing with two arrays / integer - (ix0, ix1, 4), - ) - for selection in selections: - _test_set_orthogonal_selection(v, a, z, selection) - - -def test_set_orthogonal_selection_3d(): - - # setup - v = np.arange(100000, dtype=int).reshape(200, 50, 10) - a = np.empty_like(v) - z = zarr.create(shape=a.shape, chunks=(60, 20, 3), dtype=a.dtype) - - np.random.seed(42) - # test with different degrees of sparseness - for p in 0.5, 0.1, 0.01: - - # boolean arrays - ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) - ix1 = np.random.binomial(1, .5, size=a.shape[1]).astype(bool) - ix2 = np.random.binomial(1, .5, size=a.shape[2]).astype(bool) - _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2) - - # integer arrays - ix0 = np.random.choice(a.shape[0], size=int(a.shape[0] * p), replace=True) - ix1 = np.random.choice(a.shape[1], size=int(a.shape[1] * .5), replace=True) - ix2 = np.random.choice(a.shape[2], size=int(a.shape[2] * .5), replace=True) - _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2) - - # sorted increasing - ix0.sort() - ix1.sort() - ix2.sort() - _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2) - - # sorted decreasing - ix0 = ix0[::-1] - ix1 = ix1[::-1] - ix2 = ix2[::-1] - _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2) - - -def _test_get_coordinate_selection(a, z, selection): - expect = a[selection] - actual = z.get_coordinate_selection(selection) - assert_array_equal(expect, actual) - actual = z.vindex[selection] - assert_array_equal(expect, actual) - - -coordinate_selections_1d_bad = [ - # slice not supported - slice(5, 15), - slice(None), - Ellipsis, - # bad stuff - 2.3, - 'foo', - b'xxx', - None, - (0, 0), - (slice(None), slice(None)), -] - - -# noinspection PyStatementEffect -def test_get_coordinate_selection_1d(): - - # setup - a = np.arange(1050, dtype=int) - z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) - z[:] = a - - np.random.seed(42) - # test with different degrees of sparseness - for p in 2, 0.5, 0.1, 0.01: - n = int(a.size * p) - ix = np.random.choice(a.shape[0], size=n, replace=True) - _test_get_coordinate_selection(a, z, ix) - ix.sort() - _test_get_coordinate_selection(a, z, ix) - ix = ix[::-1] - _test_get_coordinate_selection(a, z, ix) - - selections = [ - # test single item - 42, - -1, - # test wraparound - [0, 3, 10, -23, -12, -1], - # test out of order - [3, 105, 23, 127], # not monotonically increasing - # test multi-dimensional selection - np.array([[2, 4], [6, 8]]), - ] - for selection in selections: - _test_get_coordinate_selection(a, z, selection) - - # test errors - bad_selections = coordinate_selections_1d_bad + [ - [a.shape[0] + 1], # out of bounds - [-(a.shape[0] + 1)], # out of bounds - ] - for selection in bad_selections: - with pytest.raises(IndexError): - z.get_coordinate_selection(selection) - with pytest.raises(IndexError): - z.vindex[selection] - - -def test_get_coordinate_selection_2d(): - - # setup - a = np.arange(10000, dtype=int).reshape(1000, 10) - z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) - z[:] = a - - np.random.seed(42) - # test with different degrees of sparseness - for p in 2, 0.5, 0.1, 0.01: - n = int(a.size * p) - ix0 = np.random.choice(a.shape[0], size=n, replace=True) - ix1 = np.random.choice(a.shape[1], size=n, replace=True) - selections = [ - # single value - (42, 4), - (-1, -1), - # index both axes with array - (ix0, ix1), - # mixed indexing with array / int - (ix0, 4), - (42, ix1), - (42, 4), - ] - for selection in selections: - _test_get_coordinate_selection(a, z, selection) - - # not monotonically increasing (first dim) - ix0 = [3, 3, 4, 2, 5] - ix1 = [1, 3, 5, 7, 9] - _test_get_coordinate_selection(a, z, (ix0, ix1)) - - # not monotonically increasing (second dim) - ix0 = [1, 1, 2, 2, 5] - ix1 = [1, 3, 2, 1, 0] - _test_get_coordinate_selection(a, z, (ix0, ix1)) - - # multi-dimensional selection - ix0 = np.array([[1, 1, 2], - [2, 2, 5]]) - ix1 = np.array([[1, 3, 2], - [1, 0, 0]]) - _test_get_coordinate_selection(a, z, (ix0, ix1)) - - with pytest.raises(IndexError): - selection = slice(5, 15), [1, 2, 3] - z.get_coordinate_selection(selection) - with pytest.raises(IndexError): - selection = [1, 2, 3], slice(5, 15) - z.get_coordinate_selection(selection) - with pytest.raises(IndexError): - selection = Ellipsis, [1, 2, 3] - z.get_coordinate_selection(selection) - with pytest.raises(IndexError): - selection = Ellipsis - z.get_coordinate_selection(selection) - - -def _test_set_coordinate_selection(v, a, z, selection): - for value in 42, v[selection], v[selection].tolist(): - # setup expectation - a[:] = 0 - a[selection] = value - # test long-form API - z[:] = 0 - z.set_coordinate_selection(selection, value) - assert_array_equal(a, z[:]) - # test short-form API - z[:] = 0 - z.vindex[selection] = value - assert_array_equal(a, z[:]) - - -def test_set_coordinate_selection_1d(): - - # setup - v = np.arange(1050, dtype=int) - a = np.empty(v.shape, dtype=v.dtype) - z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) - - np.random.seed(42) - # test with different degrees of sparseness - for p in 2, 0.5, 0.1, 0.01: - n = int(a.size * p) - ix = np.random.choice(a.shape[0], size=n, replace=True) - _test_set_coordinate_selection(v, a, z, ix) - - # multi-dimensional selection - ix = np.array([[2, 4], [6, 8]]) - _test_set_coordinate_selection(v, a, z, ix) - - for selection in coordinate_selections_1d_bad: - with pytest.raises(IndexError): - z.set_coordinate_selection(selection, 42) - with pytest.raises(IndexError): - z.vindex[selection] = 42 - - -def test_set_coordinate_selection_2d(): - - # setup - v = np.arange(10000, dtype=int).reshape(1000, 10) - a = np.empty_like(v) - z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) - - np.random.seed(42) - # test with different degrees of sparseness - for p in 2, 0.5, 0.1, 0.01: - n = int(a.size * p) - ix0 = np.random.choice(a.shape[0], size=n, replace=True) - ix1 = np.random.choice(a.shape[1], size=n, replace=True) - - selections = ( - (42, 4), - (-1, -1), - # index both axes with array - (ix0, ix1), - # mixed indexing with array / int - (ix0, 4), - (42, ix1), - ) - for selection in selections: - _test_set_coordinate_selection(v, a, z, selection) - - # multi-dimensional selection - ix0 = np.array([[1, 2, 3], - [4, 5, 6]]) - ix1 = np.array([[1, 3, 2], - [2, 0, 5]]) - _test_set_coordinate_selection(v, a, z, (ix0, ix1)) - - -def _test_get_mask_selection(a, z, selection): - expect = a[selection] - actual = z.get_mask_selection(selection) - assert_array_equal(expect, actual) - actual = z.vindex[selection] - assert_array_equal(expect, actual) - - -mask_selections_1d_bad = [ - # slice not supported - slice(5, 15), - slice(None), - Ellipsis, - # bad stuff - 2.3, - 'foo', - b'xxx', - None, - (0, 0), - (slice(None), slice(None)), -] - - -# noinspection PyStatementEffect -def test_get_mask_selection_1d(): - - # setup - a = np.arange(1050, dtype=int) - z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) - z[:] = a - - np.random.seed(42) - # test with different degrees of sparseness - for p in 0.5, 0.1, 0.01: - ix = np.random.binomial(1, p, size=a.shape[0]).astype(bool) - _test_get_mask_selection(a, z, ix) - - # test errors - bad_selections = mask_selections_1d_bad + [ - np.zeros(50, dtype=bool), # too short - np.zeros(2000, dtype=bool), # too long - [[True, False], [False, True]], # too many dimensions - ] - for selection in bad_selections: - with pytest.raises(IndexError): - z.get_mask_selection(selection) - with pytest.raises(IndexError): - z.vindex[selection] - - -# noinspection PyStatementEffect -def test_get_mask_selection_2d(): - - # setup - a = np.arange(10000, dtype=int).reshape(1000, 10) - z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) - z[:] = a - - np.random.seed(42) - # test with different degrees of sparseness - for p in 0.5, 0.1, 0.01: - ix = np.random.binomial(1, p, size=a.size).astype(bool).reshape(a.shape) - _test_get_mask_selection(a, z, ix) - - # test errors - with pytest.raises(IndexError): - z.vindex[np.zeros((1000, 5), dtype=bool)] # too short - with pytest.raises(IndexError): - z.vindex[np.zeros((2000, 10), dtype=bool)] # too long - with pytest.raises(IndexError): - z.vindex[[True, False]] # wrong no. dimensions - - -def _test_set_mask_selection(v, a, z, selection): - a[:] = 0 - z[:] = 0 - a[selection] = v[selection] - z.set_mask_selection(selection, v[selection]) - assert_array_equal(a, z[:]) - z[:] = 0 - z.vindex[selection] = v[selection] - assert_array_equal(a, z[:]) - - -def test_set_mask_selection_1d(): - - # setup - v = np.arange(1050, dtype=int) - a = np.empty_like(v) - z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) - - np.random.seed(42) - # test with different degrees of sparseness - for p in 0.5, 0.1, 0.01: - ix = np.random.binomial(1, p, size=a.shape[0]).astype(bool) - _test_set_mask_selection(v, a, z, ix) - - for selection in mask_selections_1d_bad: - with pytest.raises(IndexError): - z.set_mask_selection(selection, 42) - with pytest.raises(IndexError): - z.vindex[selection] = 42 - - -def test_set_mask_selection_2d(): - - # setup - v = np.arange(10000, dtype=int).reshape(1000, 10) - a = np.empty_like(v) - z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) - - np.random.seed(42) - # test with different degrees of sparseness - for p in 0.5, 0.1, 0.01: - ix = np.random.binomial(1, p, size=a.size).astype(bool).reshape(a.shape) - _test_set_mask_selection(v, a, z, ix) - - -def test_get_selection_out(): - - # basic selections - a = np.arange(1050) - z = zarr.create(shape=1050, chunks=100, dtype=a.dtype) - z[:] = a - selections = [ - slice(50, 150), - slice(0, 1050), - slice(1, 2), - ] - for selection in selections: - expect = a[selection] - out = zarr.create(shape=expect.shape, chunks=10, dtype=expect.dtype, fill_value=0) - z.get_basic_selection(selection, out=out) - assert_array_equal(expect, out[:]) - - with pytest.raises(TypeError): - z.get_basic_selection(Ellipsis, out=[]) - - # orthogonal selections - a = np.arange(10000, dtype=int).reshape(1000, 10) - z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) - z[:] = a - np.random.seed(42) - # test with different degrees of sparseness - for p in 0.5, 0.1, 0.01: - ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) - ix1 = np.random.binomial(1, .5, size=a.shape[1]).astype(bool) - selections = [ - # index both axes with array - (ix0, ix1), - # mixed indexing with array / slice - (ix0, slice(1, 5)), - (slice(250, 350), ix1), - # mixed indexing with array / int - (ix0, 4), - (42, ix1), - # mixed int array / bool array - (ix0, np.nonzero(ix1)[0]), - (np.nonzero(ix0)[0], ix1), - ] - for selection in selections: - expect = oindex(a, selection) - # out = zarr.create(shape=expect.shape, chunks=10, dtype=expect.dtype, - # fill_value=0) - out = np.zeros(expect.shape, dtype=expect.dtype) - z.get_orthogonal_selection(selection, out=out) - assert_array_equal(expect, out[:]) - - # coordinate selections - a = np.arange(10000, dtype=int).reshape(1000, 10) - z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) - z[:] = a - np.random.seed(42) - # test with different degrees of sparseness - for p in 0.5, 0.1, 0.01: - n = int(a.size * p) - ix0 = np.random.choice(a.shape[0], size=n, replace=True) - ix1 = np.random.choice(a.shape[1], size=n, replace=True) - selections = [ - # index both axes with array - (ix0, ix1), - # mixed indexing with array / int - (ix0, 4), - (42, ix1), - ] - for selection in selections: - expect = a[selection] - out = np.zeros(expect.shape, dtype=expect.dtype) - z.get_coordinate_selection(selection, out=out) - assert_array_equal(expect, out[:]) - - -def test_get_selections_with_fields(): - - a = [('aaa', 1, 4.2), - ('bbb', 2, 8.4), - ('ccc', 3, 12.6)] - a = np.array(a, dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - z = zarr.create(shape=a.shape, chunks=2, dtype=a.dtype, fill_value=None) - z[:] = a - - fields_fixture = [ - 'foo', - ['foo'], - ['foo', 'bar'], - ['foo', 'baz'], - ['bar', 'baz'], - ['foo', 'bar', 'baz'], - ['bar', 'foo'], - ['baz', 'bar', 'foo'], - ] - - for fields in fields_fixture: - - # total selection - expect = a[fields] - actual = z.get_basic_selection(Ellipsis, fields=fields) - assert_array_equal(expect, actual) - # alternative API - if isinstance(fields, str): - actual = z[fields] - assert_array_equal(expect, actual) - elif len(fields) == 2: - actual = z[fields[0], fields[1]] - assert_array_equal(expect, actual) - if isinstance(fields, str): - actual = z[..., fields] - assert_array_equal(expect, actual) - elif len(fields) == 2: - actual = z[..., fields[0], fields[1]] - assert_array_equal(expect, actual) - - # basic selection with slice - expect = a[fields][0:2] - actual = z.get_basic_selection(slice(0, 2), fields=fields) - assert_array_equal(expect, actual) - # alternative API - if isinstance(fields, str): - actual = z[0:2, fields] - assert_array_equal(expect, actual) - elif len(fields) == 2: - actual = z[0:2, fields[0], fields[1]] - assert_array_equal(expect, actual) - - # basic selection with single item - expect = a[fields][1] - actual = z.get_basic_selection(1, fields=fields) - assert_array_equal(expect, actual) - # alternative API - if isinstance(fields, str): - actual = z[1, fields] - assert_array_equal(expect, actual) - elif len(fields) == 2: - actual = z[1, fields[0], fields[1]] - assert_array_equal(expect, actual) - - # orthogonal selection - ix = [0, 2] - expect = a[fields][ix] - actual = z.get_orthogonal_selection(ix, fields=fields) - assert_array_equal(expect, actual) - # alternative API - if isinstance(fields, str): - actual = z.oindex[ix, fields] - assert_array_equal(expect, actual) - elif len(fields) == 2: - actual = z.oindex[ix, fields[0], fields[1]] - assert_array_equal(expect, actual) - - # coordinate selection - ix = [0, 2] - expect = a[fields][ix] - actual = z.get_coordinate_selection(ix, fields=fields) - assert_array_equal(expect, actual) - # alternative API - if isinstance(fields, str): - actual = z.vindex[ix, fields] - assert_array_equal(expect, actual) - elif len(fields) == 2: - actual = z.vindex[ix, fields[0], fields[1]] - assert_array_equal(expect, actual) - - # mask selection - ix = [True, False, True] - expect = a[fields][ix] - actual = z.get_mask_selection(ix, fields=fields) - assert_array_equal(expect, actual) - # alternative API - if isinstance(fields, str): - actual = z.vindex[ix, fields] - assert_array_equal(expect, actual) - elif len(fields) == 2: - actual = z.vindex[ix, fields[0], fields[1]] - assert_array_equal(expect, actual) - - # missing/bad fields - with pytest.raises(IndexError): - z.get_basic_selection(Ellipsis, fields=['notafield']) - with pytest.raises(IndexError): - z.get_basic_selection(Ellipsis, fields=slice(None)) - - -def test_set_selections_with_fields(): - - v = [('aaa', 1, 4.2), - ('bbb', 2, 8.4), - ('ccc', 3, 12.6)] - v = np.array(v, dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - a = np.empty_like(v) - z = zarr.empty_like(v, chunks=2) - - fields_fixture = [ - 'foo', - [], - ['foo'], - ['foo', 'bar'], - ['foo', 'baz'], - ['bar', 'baz'], - ['foo', 'bar', 'baz'], - ['bar', 'foo'], - ['baz', 'bar', 'foo'], - ] - - for fields in fields_fixture: - - # currently multi-field assignment is not supported in numpy, so we won't support - # it either - if isinstance(fields, list) and len(fields) > 1: - with pytest.raises(IndexError): - z.set_basic_selection(Ellipsis, v, fields=fields) - with pytest.raises(IndexError): - z.set_orthogonal_selection([0, 2], v, fields=fields) - with pytest.raises(IndexError): - z.set_coordinate_selection([0, 2], v, fields=fields) - with pytest.raises(IndexError): - z.set_mask_selection([True, False, True], v, fields=fields) - - else: - - if isinstance(fields, list) and len(fields) == 1: - # work around numpy does not support multi-field assignment even if there - # is only one field - key = fields[0] - elif isinstance(fields, list) and len(fields) == 0: - # work around numpy ambiguity about what is a field selection - key = Ellipsis - else: - key = fields - - # setup expectation - a[:] = ('', 0, 0) - z[:] = ('', 0, 0) - assert_array_equal(a, z[:]) - a[key] = v[key] - # total selection - z.set_basic_selection(Ellipsis, v[key], fields=fields) - assert_array_equal(a, z[:]) - - # basic selection with slice - a[:] = ('', 0, 0) - z[:] = ('', 0, 0) - a[key][0:2] = v[key][0:2] - z.set_basic_selection(slice(0, 2), v[key][0:2], fields=fields) - assert_array_equal(a, z[:]) - - # orthogonal selection - a[:] = ('', 0, 0) - z[:] = ('', 0, 0) - ix = [0, 2] - a[key][ix] = v[key][ix] - z.set_orthogonal_selection(ix, v[key][ix], fields=fields) - assert_array_equal(a, z[:]) - - # coordinate selection - a[:] = ('', 0, 0) - z[:] = ('', 0, 0) - ix = [0, 2] - a[key][ix] = v[key][ix] - z.set_coordinate_selection(ix, v[key][ix], fields=fields) - assert_array_equal(a, z[:]) - - # mask selection - a[:] = ('', 0, 0) - z[:] = ('', 0, 0) - ix = [True, False, True] - a[key][ix] = v[key][ix] - z.set_mask_selection(ix, v[key][ix], fields=fields) - assert_array_equal(a, z[:]) - - -@pytest.mark.parametrize( - "selection, arr, expected", - [ - ( - (slice(5, 8, 1), slice(2, 4, 1), slice(0, 100, 1)), - np.arange(2, 100_002).reshape((100, 10, 100)), - [ - (5200, 200, (slice(5, 6, 1), slice(2, 4, 1))), - (6200, 200, (slice(6, 7, 1), slice(2, 4, 1))), - (7200, 200, (slice(7, 8, 1), slice(2, 4, 1))), - ], - ), - ( - (slice(5, 8, 1), slice(2, 4, 1), slice(0, 5, 1)), - np.arange(2, 100_002).reshape((100, 10, 100)), - [ - (5200.0, 5.0, (slice(5, 6, 1), slice(2, 3, 1), slice(0, 5, 1))), - (5300.0, 5.0, (slice(5, 6, 1), slice(3, 4, 1), slice(0, 5, 1))), - (6200.0, 5.0, (slice(6, 7, 1), slice(2, 3, 1), slice(0, 5, 1))), - (6300.0, 5.0, (slice(6, 7, 1), slice(3, 4, 1), slice(0, 5, 1))), - (7200.0, 5.0, (slice(7, 8, 1), slice(2, 3, 1), slice(0, 5, 1))), - (7300.0, 5.0, (slice(7, 8, 1), slice(3, 4, 1), slice(0, 5, 1))), - ], - ), - ( - (slice(5, 8, 1), slice(2, 4, 1), slice(0, 5, 1)), - np.asfortranarray(np.arange(2, 100_002).reshape((100, 10, 100))), - [ - (5200.0, 5.0, (slice(5, 6, 1), slice(2, 3, 1), slice(0, 5, 1))), - (5300.0, 5.0, (slice(5, 6, 1), slice(3, 4, 1), slice(0, 5, 1))), - (6200.0, 5.0, (slice(6, 7, 1), slice(2, 3, 1), slice(0, 5, 1))), - (6300.0, 5.0, (slice(6, 7, 1), slice(3, 4, 1), slice(0, 5, 1))), - (7200.0, 5.0, (slice(7, 8, 1), slice(2, 3, 1), slice(0, 5, 1))), - (7300.0, 5.0, (slice(7, 8, 1), slice(3, 4, 1), slice(0, 5, 1))), - ], - ), - ( - (slice(5, 8, 1), slice(2, 4, 1)), - np.arange(2, 100_002).reshape((100, 10, 100)), - [ - (5200, 200, (slice(5, 6, 1), slice(2, 4, 1))), - (6200, 200, (slice(6, 7, 1), slice(2, 4, 1))), - (7200, 200, (slice(7, 8, 1), slice(2, 4, 1))), - ], - ), - ( - (slice(0, 10, 1),), - np.arange(0, 10).reshape((10)), - [(0, 10, (slice(0, 10, 1),))], - ), - ((0,), np.arange(0, 100).reshape((10, 10)), [(0, 10, (slice(0, 1, 1),))]), - ( - ( - 0, - 0, - ), - np.arange(0, 100).reshape((10, 10)), - [(0, 1, (slice(0, 1, 1), slice(0, 1, 1)))], - ), - ((0,), np.arange(0, 10).reshape((10)), [(0, 1, (slice(0, 1, 1),))]), - pytest.param( - (slice(5, 8, 1), slice(2, 4, 1), slice(0, 5, 1)), - np.arange(2, 100002).reshape((10, 1, 10000)), - None, - marks=[pytest.mark.xfail(reason="slice 2 is out of range")], - ), - pytest.param( - (slice(5, 8, 1), slice(2, 4, 1), slice(0, 5, 1)), - np.arange(2, 100_002).reshape((10, 10_000)), - None, - marks=[pytest.mark.xfail(reason="slice 2 is out of range")], - ), - ], -) -def test_PartialChunkIterator(selection, arr, expected): - PCI = PartialChunkIterator(selection, arr.shape) - results = list(PCI) - assert results == expected diff --git a/zarr/tests/test_info.py b/zarr/tests/test_info.py deleted file mode 100644 index 361490c0a8..0000000000 --- a/zarr/tests/test_info.py +++ /dev/null @@ -1,31 +0,0 @@ -import numcodecs - -import zarr - - -def test_info(): - - # setup - g = zarr.group(store=dict(), chunk_store=dict(), - synchronizer=zarr.ThreadSynchronizer()) - g.create_group('foo') - z = g.zeros('bar', shape=10, filters=[numcodecs.Adler32()]) - - # test group info - items = g.info_items() - keys = sorted([k for k, _ in items]) - expected_keys = sorted([ - 'Type', 'Read-only', 'Synchronizer type', 'Store type', 'Chunk store type', - 'No. members', 'No. arrays', 'No. groups', 'Arrays', 'Groups', 'Name' - ]) - assert expected_keys == keys - - # test array info - items = z.info_items() - keys = sorted([k for k, _ in items]) - expected_keys = sorted([ - 'Type', 'Data type', 'Shape', 'Chunk shape', 'Order', 'Read-only', 'Filter [0]', - 'Compressor', 'Synchronizer type', 'Store type', 'Chunk store type', 'No. bytes', - 'No. bytes stored', 'Storage ratio', 'Chunks initialized', 'Name' - ]) - assert expected_keys == keys diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py deleted file mode 100644 index 5469921110..0000000000 --- a/zarr/tests/test_meta.py +++ /dev/null @@ -1,504 +0,0 @@ -import base64 -import json - -import numpy as np -import pytest - -from zarr.codecs import Blosc, Delta, Pickle, Zlib -from zarr.errors import MetadataError -from zarr.meta import (ZARR_FORMAT, decode_array_metadata, decode_dtype, - decode_group_metadata, encode_array_metadata, - encode_dtype, encode_fill_value, decode_fill_value) -from zarr.util import normalize_dtype, normalize_fill_value - - -def assert_json_equal(expect, actual): - if isinstance(actual, bytes): - actual = str(actual, 'ascii') - ej = json.loads(expect) - aj = json.loads(actual) - assert ej == aj - - -def test_encode_decode_array_1(): - - meta = dict( - shape=(100,), - chunks=(10,), - dtype=np.dtype(' 0: - try: - r = requests.get(endpoint_uri) - if r.ok: - break - except Exception: # pragma: no cover - pass - timeout -= 0.1 # pragma: no cover - time.sleep(0.1) # pragma: no cover - s3so = dict(client_kwargs={'endpoint_url': endpoint_uri}, - use_listings_cache=False) - s3 = s3fs.S3FileSystem(anon=False, **s3so) - s3.mkdir("test") - request.cls.s3so = s3so - yield - proc.terminate() - proc.wait() - - -class TestNestedDirectoryStore(TestDirectoryStore): - - def create_store(self, normalize_keys=False, **kwargs): - path = tempfile.mkdtemp() - atexit.register(atexit_rmtree, path) - store = NestedDirectoryStore(path, normalize_keys=normalize_keys, **kwargs) - return store - - def test_init_array(self): - store = self.create_store() - assert store._dimension_separator == "/" - init_array(store, shape=1000, chunks=100) - - # check metadata - assert array_meta_key in store - meta = decode_array_metadata(store[array_meta_key]) - assert ZARR_FORMAT == meta['zarr_format'] - assert (1000,) == meta['shape'] - assert (100,) == meta['chunks'] - assert np.dtype(None) == meta['dtype'] - assert meta['dimension_separator'] == "/" - - def test_chunk_nesting(self): - store = self.create_store() - # any path where last segment looks like a chunk key gets special handling - store['0.0'] = b'xxx' - assert b'xxx' == store['0.0'] - # assert b'xxx' == store['0/0'] - store['foo/10.20.30'] = b'yyy' - assert b'yyy' == store['foo/10.20.30'] - # assert b'yyy' == store['foo/10/20/30'] - store['42'] = b'zzz' - assert b'zzz' == store['42'] - - -class TestNestedDirectoryStoreNone: - - def test_value_error(self): - path = tempfile.mkdtemp() - atexit.register(atexit_rmtree, path) - store = NestedDirectoryStore( - path, normalize_keys=True, - dimension_separator=None) - assert store._dimension_separator == "/" - - -class TestNestedDirectoryStoreWithWrongValue: - - def test_value_error(self): - path = tempfile.mkdtemp() - atexit.register(atexit_rmtree, path) - with pytest.raises(ValueError): - NestedDirectoryStore( - path, normalize_keys=True, - dimension_separator=".") - - -class TestN5Store(TestNestedDirectoryStore): - - def create_store(self, normalize_keys=False): - path = tempfile.mkdtemp(suffix='.n5') - atexit.register(atexit_rmtree, path) - store = N5Store(path, normalize_keys=normalize_keys) - return store - - def test_equal(self): - store_a = self.create_store() - store_b = N5Store(store_a.path) - assert store_a == store_b - - def test_chunk_nesting(self): - store = self.create_store() - store['0.0'] = b'xxx' - assert '0.0' in store - assert b'xxx' == store['0.0'] - # assert b'xxx' == store['0/0'] - store['foo/10.20.30'] = b'yyy' - assert 'foo/10.20.30' in store - assert b'yyy' == store['foo/10.20.30'] - # N5 reverses axis order - assert b'yyy' == store['foo/30/20/10'] - store['42'] = b'zzz' - assert '42' in store - assert b'zzz' == store['42'] - - def test_init_array(self): - store = self.create_store() - init_array(store, shape=1000, chunks=100) - - # check metadata - assert array_meta_key in store - meta = decode_array_metadata(store[array_meta_key]) - assert ZARR_FORMAT == meta['zarr_format'] - assert (1000,) == meta['shape'] - assert (100,) == meta['chunks'] - assert np.dtype(None) == meta['dtype'] - # N5Store wraps the actual compressor - compressor_config = meta['compressor']['compressor_config'] - assert default_compressor.get_config() == compressor_config - # N5Store always has a fill value of 0 - assert meta['fill_value'] == 0 - - def test_init_array_path(self): - path = 'foo/bar' - store = self.create_store() - init_array(store, shape=1000, chunks=100, path=path) - - # check metadata - key = path + '/' + array_meta_key - assert key in store - meta = decode_array_metadata(store[key]) - assert ZARR_FORMAT == meta['zarr_format'] - assert (1000,) == meta['shape'] - assert (100,) == meta['chunks'] - assert np.dtype(None) == meta['dtype'] - # N5Store wraps the actual compressor - compressor_config = meta['compressor']['compressor_config'] - assert default_compressor.get_config() == compressor_config - # N5Store always has a fill value of 0 - assert meta['fill_value'] == 0 - - def test_init_array_compat(self): - store = self.create_store() - init_array(store, shape=1000, chunks=100, compressor='none') - meta = decode_array_metadata(store[array_meta_key]) - # N5Store wraps the actual compressor - compressor_config = meta['compressor']['compressor_config'] - assert compressor_config is None - - def test_init_array_overwrite(self): - self._test_init_array_overwrite('C') - - def test_init_array_overwrite_path(self): - self._test_init_array_overwrite_path('C') - - def test_init_array_overwrite_chunk_store(self): - self._test_init_array_overwrite_chunk_store('C') - - def test_init_group_overwrite(self): - self._test_init_group_overwrite('C') - - def test_init_group_overwrite_path(self): - self._test_init_group_overwrite_path('C') - - def test_init_group_overwrite_chunk_store(self): - self._test_init_group_overwrite_chunk_store('C') - - def test_init_group(self): - store = self.create_store() - init_group(store) - - # check metadata - assert group_meta_key in store - assert group_meta_key in store.listdir() - assert group_meta_key in store.listdir('') - meta = decode_group_metadata(store[group_meta_key]) - assert ZARR_FORMAT == meta['zarr_format'] - - def test_filters(self): - all_filters, all_errors = zip(*[ - (None, does_not_raise()), - ([], does_not_raise()), - ([AsType('f4', 'f8')], pytest.raises(ValueError)), - ]) - for filters, error in zip(all_filters, all_errors): - store = self.create_store() - with error: - init_array(store, shape=1000, chunks=100, filters=filters) - - -@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") -class TestNestedFSStore(TestNestedDirectoryStore): - - def create_store(self, normalize_keys=False, path=None, **kwargs): - if path is None: - path = tempfile.mkdtemp() - atexit.register(atexit_rmtree, path) - store = FSStore(path, normalize_keys=normalize_keys, - dimension_separator='/', auto_mkdir=True, **kwargs) - return store - - def test_numbered_groups(self): - import zarr - - # Create an array - store = self.create_store() - group = zarr.group(store=store) - arr = group.create_dataset('0', shape=(10, 10)) - arr[1] = 1 - - # Read it back - store = self.create_store(path=store.path) - zarr.open_group(store.path)["0"] - - -class TestTempStore(StoreTests): - - def create_store(self, **kwargs): - skip_if_nested_chunks(**kwargs) - return TempStore(**kwargs) - - def test_setdel(self): - store = self.create_store() - setdel_hierarchy_checks(store) - - -class TestZipStore(StoreTests): - - def create_store(self, **kwargs): - path = tempfile.mktemp(suffix='.zip') - atexit.register(os.remove, path) - store = ZipStore(path, mode='w', **kwargs) - return store - - def test_mode(self): - with ZipStore('data/store.zip', mode='w') as store: - store['foo'] = b'bar' - store = ZipStore('data/store.zip', mode='r') - with pytest.raises(PermissionError): - store['foo'] = b'bar' - with pytest.raises(PermissionError): - store.clear() - - def test_flush(self): - store = ZipStore('data/store.zip', mode='w') - store['foo'] = b'bar' - store.flush() - assert store['foo'] == b'bar' - store.close() - - store = ZipStore('data/store.zip', mode='r') - store.flush() # no-op - - def test_context_manager(self): - with self.create_store() as store: - store['foo'] = b'bar' - store['baz'] = b'qux' - assert 2 == len(store) - - def test_pop(self): - # override because not implemented - store = self.create_store() - store['foo'] = b'bar' - with pytest.raises(NotImplementedError): - store.pop('foo') - - def test_popitem(self): - # override because not implemented - store = self.create_store() - store['foo'] = b'bar' - with pytest.raises(NotImplementedError): - store.popitem() - - def test_permissions(self): - store = ZipStore('data/store.zip', mode='w') - store['foo'] = b'bar' - store['baz/'] = b'' - store.flush() - store.close() - z = ZipFile('data/store.zip', 'r') - info = z.getinfo('foo') - perm = oct(info.external_attr >> 16) - assert perm == '0o644' - info = z.getinfo('baz/') - perm = oct(info.external_attr >> 16) - # only for posix platforms - if os.name == 'posix': - assert perm == '0o40775' - z.close() - - -class TestDBMStore(StoreTests): - - def create_store(self, dimension_separator=None): - path = tempfile.mktemp(suffix='.anydbm') - atexit.register(atexit_rmglob, path + '*') - # create store using default dbm implementation - store = DBMStore(path, flag='n', dimension_separator=dimension_separator) - return store - - def test_context_manager(self): - with self.create_store() as store: - store['foo'] = b'bar' - store['baz'] = b'qux' - assert 2 == len(store) - - -class TestDBMStoreDumb(TestDBMStore): - - def create_store(self, **kwargs): - path = tempfile.mktemp(suffix='.dumbdbm') - atexit.register(atexit_rmglob, path + '*') - - import dbm.dumb as dumbdbm - store = DBMStore(path, flag='n', open=dumbdbm.open, **kwargs) - return store - - -class TestDBMStoreGnu(TestDBMStore): - - def create_store(self, **kwargs): - gdbm = pytest.importorskip("dbm.gnu") - path = tempfile.mktemp(suffix=".gdbm") # pragma: no cover - atexit.register(os.remove, path) # pragma: no cover - store = DBMStore( - path, flag="n", open=gdbm.open, write_lock=False, **kwargs - ) # pragma: no cover - return store # pragma: no cover - - -class TestDBMStoreNDBM(TestDBMStore): - - def create_store(self, **kwargs): - ndbm = pytest.importorskip("dbm.ndbm") - path = tempfile.mktemp(suffix=".ndbm") # pragma: no cover - atexit.register(atexit_rmglob, path + "*") # pragma: no cover - store = DBMStore(path, flag="n", open=ndbm.open, **kwargs) # pragma: no cover - return store # pragma: no cover - - -class TestDBMStoreBerkeleyDB(TestDBMStore): - - def create_store(self, **kwargs): - bsddb3 = pytest.importorskip("bsddb3") - path = tempfile.mktemp(suffix='.dbm') - atexit.register(os.remove, path) - store = DBMStore(path, flag='n', open=bsddb3.btopen, write_lock=False, **kwargs) - return store - - -class TestLMDBStore(StoreTests): - - def create_store(self, **kwargs): - pytest.importorskip("lmdb") - path = tempfile.mktemp(suffix='.lmdb') - atexit.register(atexit_rmtree, path) - buffers = True - store = LMDBStore(path, buffers=buffers, **kwargs) - return store - - def test_context_manager(self): - with self.create_store() as store: - store['foo'] = b'bar' - store['baz'] = b'qux' - assert 2 == len(store) - - -class TestSQLiteStore(StoreTests): - - def create_store(self, **kwargs): - pytest.importorskip("sqlite3") - path = tempfile.mktemp(suffix='.db') - atexit.register(atexit_rmtree, path) - store = SQLiteStore(path, **kwargs) - return store - - def test_underscore_in_name(self): - path = tempfile.mktemp(suffix='.db') - atexit.register(atexit_rmtree, path) - store = SQLiteStore(path) - store['a'] = b'aaa' - store['a_b'] = b'aa_bb' - store.rmdir('a') - assert 'a_b' in store - - -class TestSQLiteStoreInMemory(TestSQLiteStore): - - def create_store(self, **kwargs): - pytest.importorskip("sqlite3") - store = SQLiteStore(':memory:', **kwargs) - return store - - def test_pickle(self): - - # setup store - store = self.create_store() - store['foo'] = b'bar' - store['baz'] = b'quux' - - # round-trip through pickle - with pytest.raises(PicklingError): - pickle.dumps(store) - - -@skip_test_env_var("ZARR_TEST_MONGO") -class TestMongoDBStore(StoreTests): - - def create_store(self, **kwargs): - pytest.importorskip("pymongo") - store = MongoDBStore(host='127.0.0.1', database='zarr_tests', - collection='zarr_tests', **kwargs) - # start with an empty store - store.clear() - return store - - -@skip_test_env_var("ZARR_TEST_REDIS") -class TestRedisStore(StoreTests): - - def create_store(self, **kwargs): - # TODO: this is the default host for Redis on Travis, - # we probably want to generalize this though - pytest.importorskip("redis") - store = RedisStore(host='localhost', port=6379, **kwargs) - # start with an empty store - store.clear() - return store - - -class TestLRUStoreCache(StoreTests): - - def create_store(self, **kwargs): - # wrapper therefore no dimension_separator argument - skip_if_nested_chunks(**kwargs) - return LRUStoreCache(dict(), max_size=2**27) - - def test_cache_values_no_max_size(self): - - # setup store - store = CountingDict() - store['foo'] = b'xxx' - store['bar'] = b'yyy' - assert 0 == store.counter['__getitem__', 'foo'] - assert 1 == store.counter['__setitem__', 'foo'] - assert 0 == store.counter['__getitem__', 'bar'] - assert 1 == store.counter['__setitem__', 'bar'] - - # setup cache - cache = LRUStoreCache(store, max_size=None) - assert 0 == cache.hits - assert 0 == cache.misses - - # test first __getitem__, cache miss - assert b'xxx' == cache['foo'] - assert 1 == store.counter['__getitem__', 'foo'] - assert 1 == store.counter['__setitem__', 'foo'] - assert 0 == cache.hits - assert 1 == cache.misses - - # test second __getitem__, cache hit - assert b'xxx' == cache['foo'] - assert 1 == store.counter['__getitem__', 'foo'] - assert 1 == store.counter['__setitem__', 'foo'] - assert 1 == cache.hits - assert 1 == cache.misses - - # test __setitem__, __getitem__ - cache['foo'] = b'zzz' - assert 1 == store.counter['__getitem__', 'foo'] - assert 2 == store.counter['__setitem__', 'foo'] - # should be a cache hit - assert b'zzz' == cache['foo'] - assert 1 == store.counter['__getitem__', 'foo'] - assert 2 == store.counter['__setitem__', 'foo'] - assert 2 == cache.hits - assert 1 == cache.misses - - # manually invalidate all cached values - cache.invalidate_values() - assert b'zzz' == cache['foo'] - assert 2 == store.counter['__getitem__', 'foo'] - assert 2 == store.counter['__setitem__', 'foo'] - cache.invalidate() - assert b'zzz' == cache['foo'] - assert 3 == store.counter['__getitem__', 'foo'] - assert 2 == store.counter['__setitem__', 'foo'] - - # test __delitem__ - del cache['foo'] - with pytest.raises(KeyError): - # noinspection PyStatementEffect - cache['foo'] - with pytest.raises(KeyError): - # noinspection PyStatementEffect - store['foo'] - - # verify other keys untouched - assert 0 == store.counter['__getitem__', 'bar'] - assert 1 == store.counter['__setitem__', 'bar'] - - def test_cache_values_with_max_size(self): - - # setup store - store = CountingDict() - store['foo'] = b'xxx' - store['bar'] = b'yyy' - assert 0 == store.counter['__getitem__', 'foo'] - assert 0 == store.counter['__getitem__', 'bar'] - # setup cache - can only hold one item - cache = LRUStoreCache(store, max_size=5) - assert 0 == cache.hits - assert 0 == cache.misses - - # test first 'foo' __getitem__, cache miss - assert b'xxx' == cache['foo'] - assert 1 == store.counter['__getitem__', 'foo'] - assert 0 == cache.hits - assert 1 == cache.misses - - # test second 'foo' __getitem__, cache hit - assert b'xxx' == cache['foo'] - assert 1 == store.counter['__getitem__', 'foo'] - assert 1 == cache.hits - assert 1 == cache.misses - - # test first 'bar' __getitem__, cache miss - assert b'yyy' == cache['bar'] - assert 1 == store.counter['__getitem__', 'bar'] - assert 1 == cache.hits - assert 2 == cache.misses - - # test second 'bar' __getitem__, cache hit - assert b'yyy' == cache['bar'] - assert 1 == store.counter['__getitem__', 'bar'] - assert 2 == cache.hits - assert 2 == cache.misses - - # test 'foo' __getitem__, should have been evicted, cache miss - assert b'xxx' == cache['foo'] - assert 2 == store.counter['__getitem__', 'foo'] - assert 2 == cache.hits - assert 3 == cache.misses - - # test 'bar' __getitem__, should have been evicted, cache miss - assert b'yyy' == cache['bar'] - assert 2 == store.counter['__getitem__', 'bar'] - assert 2 == cache.hits - assert 4 == cache.misses - - # setup store - store = CountingDict() - store['foo'] = b'xxx' - store['bar'] = b'yyy' - assert 0 == store.counter['__getitem__', 'foo'] - assert 0 == store.counter['__getitem__', 'bar'] - # setup cache - can hold two items - cache = LRUStoreCache(store, max_size=6) - assert 0 == cache.hits - assert 0 == cache.misses - - # test first 'foo' __getitem__, cache miss - assert b'xxx' == cache['foo'] - assert 1 == store.counter['__getitem__', 'foo'] - assert 0 == cache.hits - assert 1 == cache.misses - - # test second 'foo' __getitem__, cache hit - assert b'xxx' == cache['foo'] - assert 1 == store.counter['__getitem__', 'foo'] - assert 1 == cache.hits - assert 1 == cache.misses - - # test first 'bar' __getitem__, cache miss - assert b'yyy' == cache['bar'] - assert 1 == store.counter['__getitem__', 'bar'] - assert 1 == cache.hits - assert 2 == cache.misses - - # test second 'bar' __getitem__, cache hit - assert b'yyy' == cache['bar'] - assert 1 == store.counter['__getitem__', 'bar'] - assert 2 == cache.hits - assert 2 == cache.misses - - # test 'foo' __getitem__, should still be cached - assert b'xxx' == cache['foo'] - assert 1 == store.counter['__getitem__', 'foo'] - assert 3 == cache.hits - assert 2 == cache.misses - - # test 'bar' __getitem__, should still be cached - assert b'yyy' == cache['bar'] - assert 1 == store.counter['__getitem__', 'bar'] - assert 4 == cache.hits - assert 2 == cache.misses - - def test_cache_keys(self): - - # setup - store = CountingDict() - store['foo'] = b'xxx' - store['bar'] = b'yyy' - assert 0 == store.counter['__contains__', 'foo'] - assert 0 == store.counter['__iter__'] - assert 0 == store.counter['keys'] - cache = LRUStoreCache(store, max_size=None) - - # keys should be cached on first call - keys = sorted(cache.keys()) - assert keys == ['bar', 'foo'] - assert 1 == store.counter['keys'] - # keys should now be cached - assert keys == sorted(cache.keys()) - assert 1 == store.counter['keys'] - assert 'foo' in cache - assert 0 == store.counter['__contains__', 'foo'] - assert keys == sorted(cache) - assert 0 == store.counter['__iter__'] - assert 1 == store.counter['keys'] - - # cache should be cleared if store is modified - crude but simple for now - cache['baz'] = b'zzz' - keys = sorted(cache.keys()) - assert keys == ['bar', 'baz', 'foo'] - assert 2 == store.counter['keys'] - # keys should now be cached - assert keys == sorted(cache.keys()) - assert 2 == store.counter['keys'] - - # manually invalidate keys - cache.invalidate_keys() - keys = sorted(cache.keys()) - assert keys == ['bar', 'baz', 'foo'] - assert 3 == store.counter['keys'] - assert 0 == store.counter['__contains__', 'foo'] - assert 0 == store.counter['__iter__'] - cache.invalidate_keys() - keys = sorted(cache) - assert keys == ['bar', 'baz', 'foo'] - assert 4 == store.counter['keys'] - assert 0 == store.counter['__contains__', 'foo'] - assert 0 == store.counter['__iter__'] - cache.invalidate_keys() - assert 'foo' in cache - assert 5 == store.counter['keys'] - assert 0 == store.counter['__contains__', 'foo'] - assert 0 == store.counter['__iter__'] - - # check these would get counted if called directly - assert 'foo' in store - assert 1 == store.counter['__contains__', 'foo'] - assert keys == sorted(store) - assert 1 == store.counter['__iter__'] - - -def test_getsize(): - store = dict() - store['foo'] = b'aaa' - store['bar'] = b'bbbb' - store['baz/quux'] = b'ccccc' - assert 7 == getsize(store) - assert 5 == getsize(store, 'baz') - - store = dict() - store['boo'] = None - assert -1 == getsize(store) - - -def test_migrate_1to2(): - from zarr import meta_v1 - - # N.B., version 1 did not support hierarchies, so we only have to be - # concerned about migrating a single array at the root of the store - - # setup - store = dict() - meta = dict( - shape=(100,), - chunks=(10,), - dtype=np.dtype('f4'), - compression='zlib', - compression_opts=1, - fill_value=None, - order='C' - ) - meta_json = meta_v1.encode_metadata(meta) - store['meta'] = meta_json - store['attrs'] = json.dumps(dict()).encode('ascii') - - # run migration - migrate_1to2(store) - - # check results - assert 'meta' not in store - assert array_meta_key in store - assert 'attrs' not in store - assert attrs_key in store - meta_migrated = decode_array_metadata(store[array_meta_key]) - assert 2 == meta_migrated['zarr_format'] - - # preserved fields - for f in 'shape', 'chunks', 'dtype', 'fill_value', 'order': - assert meta[f] == meta_migrated[f] - - # migrate should have added empty filters field - assert meta_migrated['filters'] is None - - # check compression and compression_opts migrated to compressor - assert 'compression' not in meta_migrated - assert 'compression_opts' not in meta_migrated - assert meta_migrated['compressor'] == Zlib(1).get_config() - - # check dict compression_opts - store = dict() - meta['compression'] = 'blosc' - meta['compression_opts'] = dict(cname='lz4', clevel=5, shuffle=1) - meta_json = meta_v1.encode_metadata(meta) - store['meta'] = meta_json - store['attrs'] = json.dumps(dict()).encode('ascii') - migrate_1to2(store) - meta_migrated = decode_array_metadata(store[array_meta_key]) - assert 'compression' not in meta_migrated - assert 'compression_opts' not in meta_migrated - assert (meta_migrated['compressor'] == - Blosc(cname='lz4', clevel=5, shuffle=1).get_config()) - - # check 'none' compression is migrated to None (null in JSON) - store = dict() - meta['compression'] = 'none' - meta_json = meta_v1.encode_metadata(meta) - store['meta'] = meta_json - store['attrs'] = json.dumps(dict()).encode('ascii') - migrate_1to2(store) - meta_migrated = decode_array_metadata(store[array_meta_key]) - assert 'compression' not in meta_migrated - assert 'compression_opts' not in meta_migrated - assert meta_migrated['compressor'] is None - - -def test_format_compatibility(): - - # This test is intended to catch any unintended changes that break the ability to - # read data stored with a previous minor version (which should be format-compatible). - - # fixture data - fixture = group(store=DirectoryStore('fixture')) - - # set seed to get consistent random data - np.random.seed(42) - - arrays_chunks = [ - (np.arange(1111, dtype='' == actual[-8:] - - -def test_tree_get_icon(): - assert tree_get_icon("Array") == tree_array_icon - assert tree_get_icon("Group") == tree_group_icon - with pytest.raises(ValueError): - tree_get_icon("Baz") - - -@mock.patch.dict("sys.modules", {"ipytree": None}) -def test_tree_widget_missing_ipytree(): - pattern = ( - "Run `pip install zarr[jupyter]` or `conda install ipytree`" - "to get the required ipytree dependency for displaying the tree " - "widget. If using jupyterlab<3, you also need to run " - "`jupyter labextension install ipytree`" - ) - with pytest.raises(ImportError, match=re.escape(pattern)): - tree_widget(None, None, None) - - -def test_retry_call(): - - class Fixture: - - def __init__(self, pass_on=1): - self.c = 0 - self.pass_on = pass_on - - def __call__(self): - self.c += 1 - if self.c != self.pass_on: - raise PermissionError() - - for x in range(1, 11): - # Any number of failures less than 10 will be accepted. - fixture = Fixture(pass_on=x) - retry_call(fixture, exceptions=(PermissionError,), wait=0) - assert fixture.c == x - - def fail(x): - # Failures after 10 will cause an error to be raised. - retry_call(Fixture(pass_on=x), exceptions=(Exception,), wait=0) - - for x in range(11, 15): - pytest.raises(PermissionError, fail, x) diff --git a/zarr/tests/util.py b/zarr/tests/util.py deleted file mode 100644 index 1de014589b..0000000000 --- a/zarr/tests/util.py +++ /dev/null @@ -1,76 +0,0 @@ -import collections -from collections.abc import MutableMapping -import os - -import pytest - - -class CountingDict(MutableMapping): - - def __init__(self): - self.wrapped = dict() - self.counter = collections.Counter() - - def __len__(self): - self.counter['__len__'] += 1 - return len(self.wrapped) - - def keys(self): - self.counter['keys'] += 1 - return self.wrapped.keys() - - def __iter__(self): - self.counter['__iter__'] += 1 - return iter(self.wrapped) - - def __contains__(self, item): - self.counter['__contains__', item] += 1 - return item in self.wrapped - - def __getitem__(self, item): - self.counter['__getitem__', item] += 1 - return self.wrapped[item] - - def __setitem__(self, key, value): - self.counter['__setitem__', key] += 1 - self.wrapped[key] = value - - def __delitem__(self, key): - self.counter['__delitem__', key] += 1 - del self.wrapped[key] - - -def skip_test_env_var(name): - """ Checks for environment variables indicating whether tests requiring services should be run - """ - value = os.environ.get(name, '0') - return pytest.mark.skipif(value == '0', reason='Tests not enabled via environment variable') - - -try: - import fsspec # noqa: F401 - - have_fsspec = True -except ImportError: # pragma: no cover - have_fsspec = False - - -def abs_container(): - from azure.core.exceptions import ResourceExistsError - import azure.storage.blob as asb - - URL = "http://127.0.0.1:10000" - ACCOUNT_NAME = "devstoreaccount1" - KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" - CONN_STR = ( - f"DefaultEndpointsProtocol=http;AccountName={ACCOUNT_NAME};" - f"AccountKey={KEY};BlobEndpoint={URL}/{ACCOUNT_NAME};" - ) - - blob_service_client = asb.BlobServiceClient.from_connection_string(CONN_STR) - try: - container_client = blob_service_client.create_container("test") - except ResourceExistsError: - container_client = blob_service_client.get_container_client("test") - - return container_client diff --git a/zarr/util.py b/zarr/util.py deleted file mode 100644 index 2a2250433c..0000000000 --- a/zarr/util.py +++ /dev/null @@ -1,652 +0,0 @@ -import inspect -import json -import math -import numbers -from textwrap import TextWrapper -import mmap -import time - -import numpy as np -from asciitree import BoxStyle, LeftAligned -from asciitree.traversal import Traversal -from numcodecs.compat import ensure_ndarray, ensure_text -from numcodecs.registry import codec_registry -from numcodecs.blosc import cbuffer_sizes, cbuffer_metainfo - -from typing import Any, Callable, Dict, Optional, Tuple, Union - - -# codecs to use for object dtype convenience API -object_codecs = { - str.__name__: 'vlen-utf8', - bytes.__name__: 'vlen-bytes', - 'array': 'vlen-array', -} - - -def json_dumps(o: Any) -> bytes: - """Write JSON in a consistent, human-readable way.""" - return json.dumps(o, indent=4, sort_keys=True, ensure_ascii=True, - separators=(',', ': ')).encode('ascii') - - -def json_loads(s: str) -> Dict[str, Any]: - """Read JSON in a consistent way.""" - return json.loads(ensure_text(s, 'ascii')) - - -def normalize_shape(shape) -> Tuple[int]: - """Convenience function to normalize the `shape` argument.""" - - if shape is None: - raise TypeError('shape is None') - - # handle 1D convenience form - if isinstance(shape, numbers.Integral): - shape = (int(shape),) - - # normalize - shape = tuple(int(s) for s in shape) - return shape - - -# code to guess chunk shape, adapted from h5py - -CHUNK_BASE = 256*1024 # Multiplier by which chunks are adjusted -CHUNK_MIN = 128*1024 # Soft lower limit (128k) -CHUNK_MAX = 64*1024*1024 # Hard upper limit - - -def guess_chunks(shape: Tuple[int, ...], typesize: int) -> Tuple[int, ...]: - """ - Guess an appropriate chunk layout for an array, given its shape and - the size of each element in bytes. Will allocate chunks only as large - as MAX_SIZE. Chunks are generally close to some power-of-2 fraction of - each axis, slightly favoring bigger values for the last index. - Undocumented and subject to change without warning. - """ - - ndims = len(shape) - # require chunks to have non-zero length for all dimensions - chunks = np.maximum(np.array(shape, dtype='=f8'), 1) - - # Determine the optimal chunk size in bytes using a PyTables expression. - # This is kept as a float. - dset_size = np.product(chunks)*typesize - target_size = CHUNK_BASE * (2**np.log10(dset_size/(1024.*1024))) - - if target_size > CHUNK_MAX: - target_size = CHUNK_MAX - elif target_size < CHUNK_MIN: - target_size = CHUNK_MIN - - idx = 0 - while True: - # Repeatedly loop over the axes, dividing them by 2. Stop when: - # 1a. We're smaller than the target chunk size, OR - # 1b. We're within 50% of the target chunk size, AND - # 2. The chunk is smaller than the maximum chunk size - - chunk_bytes = np.product(chunks)*typesize - - if (chunk_bytes < target_size or - abs(chunk_bytes-target_size)/target_size < 0.5) and \ - chunk_bytes < CHUNK_MAX: - break - - if np.product(chunks) == 1: - break # Element size larger than CHUNK_MAX - - chunks[idx % ndims] = math.ceil(chunks[idx % ndims] / 2.0) - idx += 1 - - return tuple(int(x) for x in chunks) - - -def normalize_chunks( - chunks: Any, shape: Tuple[int, ...], typesize: int -) -> Tuple[int, ...]: - """Convenience function to normalize the `chunks` argument for an array - with the given `shape`.""" - - # N.B., expect shape already normalized - - # handle auto-chunking - if chunks is None or chunks is True: - return guess_chunks(shape, typesize) - - # handle no chunking - if chunks is False: - return shape - - # handle 1D convenience form - if isinstance(chunks, numbers.Integral): - chunks = tuple(int(chunks) for _ in shape) - - # handle bad dimensionality - if len(chunks) > len(shape): - raise ValueError('too many dimensions in chunks') - - # handle underspecified chunks - if len(chunks) < len(shape): - # assume chunks across remaining dimensions - chunks += shape[len(chunks):] - - # handle None or -1 in chunks - if -1 in chunks or None in chunks: - chunks = tuple(s if c == -1 or c is None else int(c) - for s, c in zip(shape, chunks)) - - return tuple(chunks) - - -def normalize_dtype(dtype: Union[str, np.dtype], object_codec) -> Tuple[np.dtype, Any]: - - # convenience API for object arrays - if inspect.isclass(dtype): - dtype = dtype.__name__ # type: ignore - if isinstance(dtype, str): - # allow ':' to delimit class from codec arguments - tokens = dtype.split(':') - key = tokens[0] - if key in object_codecs: - dtype = np.dtype(object) - if object_codec is None: - codec_id = object_codecs[key] - if len(tokens) > 1: - args = tokens[1].split(',') - else: - args = [] - try: - object_codec = codec_registry[codec_id](*args) - except KeyError: # pragma: no cover - raise ValueError('codec %r for object type %r is not ' - 'available; please provide an ' - 'object_codec manually' % (codec_id, key)) - return dtype, object_codec - - dtype = np.dtype(dtype) - - # don't allow generic datetime64 or timedelta64, require units to be specified - if dtype == np.dtype('M8') or dtype == np.dtype('m8'): - raise ValueError('datetime64 and timedelta64 dtypes with generic units ' - 'are not supported, please specify units (e.g., "M8[ns]")') - - return dtype, object_codec - - -# noinspection PyTypeChecker -def is_total_slice(item, shape: Tuple[int]) -> bool: - """Determine whether `item` specifies a complete slice of array with the - given `shape`. Used to optimize __setitem__ operations on the Chunk - class.""" - - # N.B., assume shape is normalized - - if item == Ellipsis: - return True - if item == slice(None): - return True - if isinstance(item, slice): - item = item, - if isinstance(item, tuple): - return all( - (isinstance(s, slice) and - ((s == slice(None)) or - ((s.stop - s.start == l) and (s.step in [1, None])))) - for s, l in zip(item, shape) - ) - else: - raise TypeError('expected slice or tuple of slices, found %r' % item) - - -def normalize_resize_args(old_shape, *args): - - # normalize new shape argument - if len(args) == 1: - new_shape = args[0] - else: - new_shape = args - if isinstance(new_shape, int): - new_shape = (new_shape,) - else: - new_shape = tuple(new_shape) - if len(new_shape) != len(old_shape): - raise ValueError('new shape must have same number of dimensions') - - # handle None in new_shape - new_shape = tuple(s if n is None else int(n) - for s, n in zip(old_shape, new_shape)) - - return new_shape - - -def human_readable_size(size) -> str: - if size < 2**10: - return '%s' % size - elif size < 2**20: - return '%.1fK' % (size / float(2**10)) - elif size < 2**30: - return '%.1fM' % (size / float(2**20)) - elif size < 2**40: - return '%.1fG' % (size / float(2**30)) - elif size < 2**50: - return '%.1fT' % (size / float(2**40)) - else: - return '%.1fP' % (size / float(2**50)) - - -def normalize_order(order: str) -> str: - order = str(order).upper() - if order not in ['C', 'F']: - raise ValueError("order must be either 'C' or 'F', found: %r" % order) - return order - - -def normalize_dimension_separator(sep: Optional[str]) -> Optional[str]: - if sep in (".", "/", None): - return sep - else: - raise ValueError( - "dimension_separator must be either '.' or '/', found: %r" % sep) - - -def normalize_fill_value(fill_value, dtype: np.dtype): - - if fill_value is None or dtype.hasobject: - # no fill value - pass - elif fill_value == 0: - # this should be compatible across numpy versions for any array type, including - # structured arrays - fill_value = np.zeros((), dtype=dtype)[()] - - elif dtype.kind == 'U': - # special case unicode because of encoding issues on Windows if passed through numpy - # https://github.com/alimanfoo/zarr/pull/172#issuecomment-343782713 - - if not isinstance(fill_value, str): - raise ValueError('fill_value {!r} is not valid for dtype {}; must be a ' - 'unicode string'.format(fill_value, dtype)) - - else: - try: - if isinstance(fill_value, bytes) and dtype.kind == 'V': - # special case for numpy 1.14 compatibility - fill_value = np.array(fill_value, dtype=dtype.str).view(dtype)[()] - else: - fill_value = np.array(fill_value, dtype=dtype)[()] - - except Exception as e: - # re-raise with our own error message to be helpful - raise ValueError('fill_value {!r} is not valid for dtype {}; nested ' - 'exception: {}'.format(fill_value, dtype, e)) - - return fill_value - - -def normalize_storage_path(path: Union[str, bytes, None]) -> str: - - # handle bytes - if isinstance(path, bytes): - path = str(path, 'ascii') - - # ensure str - if path is not None and not isinstance(path, str): - path = str(path) - - if path: - - # convert backslash to forward slash - path = path.replace('\\', '/') - - # ensure no leading slash - while len(path) > 0 and path[0] == '/': - path = path[1:] - - # ensure no trailing slash - while len(path) > 0 and path[-1] == '/': - path = path[:-1] - - # collapse any repeated slashes - previous_char = None - collapsed = '' - for char in path: - if char == '/' and previous_char == '/': - pass - else: - collapsed += char - previous_char = char - path = collapsed - - # don't allow path segments with just '.' or '..' - segments = path.split('/') - if any([s in {'.', '..'} for s in segments]): - raise ValueError("path containing '.' or '..' segment not allowed") - - else: - path = '' - - return path - - -def buffer_size(v) -> int: - return ensure_ndarray(v).nbytes - - -def info_text_report(items: Dict[Any, Any]) -> str: - keys = [k for k, v in items] - max_key_len = max(len(k) for k in keys) - report = '' - for k, v in items: - wrapper = TextWrapper(width=80, - initial_indent=k.ljust(max_key_len) + ' : ', - subsequent_indent=' '*max_key_len + ' : ') - text = wrapper.fill(str(v)) - report += text + '\n' - return report - - -def info_html_report(items) -> str: - report = '' - report += '' - for k, v in items: - report += '' \ - '' \ - '' \ - '' \ - % (k, v) - report += '' - report += '
%s%s
' - return report - - -class InfoReporter(object): - - def __init__(self, obj): - self.obj = obj - - def __repr__(self): - items = self.obj.info_items() - return info_text_report(items) - - def _repr_html_(self): - items = self.obj.info_items() - return info_html_report(items) - - -class TreeNode(object): - - def __init__(self, obj, depth=0, level=None): - self.obj = obj - self.depth = depth - self.level = level - - def get_children(self): - if hasattr(self.obj, 'values'): - if self.level is None or self.depth < self.level: - depth = self.depth + 1 - return [TreeNode(o, depth=depth, level=self.level) - for o in self.obj.values()] - return [] - - def get_text(self): - name = self.obj.name.split("/")[-1] or "/" - if hasattr(self.obj, 'shape'): - name += ' {} {}'.format(self.obj.shape, self.obj.dtype) - return name - - def get_type(self): - return type(self.obj).__name__ - - -class TreeTraversal(Traversal): - - def get_children(self, node): - return node.get_children() - - def get_root(self, tree): - return tree - - def get_text(self, node): - return node.get_text() - - -tree_group_icon = 'folder' -tree_array_icon = 'table' - - -def tree_get_icon(stype: str) -> str: - if stype == "Array": - return tree_array_icon - elif stype == "Group": - return tree_group_icon - else: - raise ValueError("Unknown type: %s" % stype) - - -def tree_widget_sublist(node, root=False, expand=False): - import ipytree - - result = ipytree.Node() - result.icon = tree_get_icon(node.get_type()) - if root or (expand is True) or (isinstance(expand, int) and node.depth < expand): - result.opened = True - else: - result.opened = False - result.name = node.get_text() - result.nodes = [tree_widget_sublist(c, expand=expand) for c in node.get_children()] - result.disabled = True - - return result - - -def tree_widget(group, expand, level): - try: - import ipytree - except ImportError as error: - raise ImportError( - "{}: Run `pip install zarr[jupyter]` or `conda install ipytree`" - "to get the required ipytree dependency for displaying the tree " - "widget. If using jupyterlab<3, you also need to run " - "`jupyter labextension install ipytree`".format(error) - ) - - result = ipytree.Tree() - root = TreeNode(group, level=level) - result.add_node(tree_widget_sublist(root, root=True, expand=expand)) - - return result - - -class TreeViewer(object): - - def __init__(self, group, expand=False, level=None): - - self.group = group - self.expand = expand - self.level = level - - self.text_kwargs = dict( - horiz_len=2, - label_space=1, - indent=1 - ) - - self.bytes_kwargs = dict( - UP_AND_RIGHT="+", - HORIZONTAL="-", - VERTICAL="|", - VERTICAL_AND_RIGHT="+" - ) - - self.unicode_kwargs = dict( - UP_AND_RIGHT="\u2514", - HORIZONTAL="\u2500", - VERTICAL="\u2502", - VERTICAL_AND_RIGHT="\u251C" - ) - - def __bytes__(self): - drawer = LeftAligned( - traverse=TreeTraversal(), - draw=BoxStyle(gfx=self.bytes_kwargs, **self.text_kwargs) - ) - root = TreeNode(self.group, level=self.level) - result = drawer(root) - - # Unicode characters slip in on Python 3. - # So we need to straighten that out first. - result = result.encode() - - return result - - def __unicode__(self): - drawer = LeftAligned( - traverse=TreeTraversal(), - draw=BoxStyle(gfx=self.unicode_kwargs, **self.text_kwargs) - ) - root = TreeNode(self.group, level=self.level) - return drawer(root) - - def __repr__(self): - return self.__unicode__() - - def _ipython_display_(self): - tree = tree_widget(self.group, expand=self.expand, level=self.level) - tree._ipython_display_() - return tree - - -def check_array_shape(param, array, shape): - if not hasattr(array, 'shape'): - raise TypeError('parameter {!r}: expected an array-like object, got {!r}' - .format(param, type(array))) - if array.shape != shape: - raise ValueError('parameter {!r}: expected array with shape {!r}, got {!r}' - .format(param, shape, array.shape)) - - -def is_valid_python_name(name): - from keyword import iskeyword - return name.isidentifier() and not iskeyword(name) - - -class NoLock(object): - """A lock that doesn't lock.""" - - def __enter__(self): - pass - - def __exit__(self, *args): - pass - - -nolock = NoLock() - - -class PartialReadBuffer: - def __init__(self, store_key, chunk_store): - self.chunk_store = chunk_store - # is it fsstore or an actual fsspec map object - assert hasattr(self.chunk_store, "map") - self.map = self.chunk_store.map - self.fs = self.chunk_store.fs - self.store_key = store_key - self.buff = None - self.nblocks = None - self.start_points = None - self.n_per_block = None - self.start_points_max = None - self.read_blocks = set() - - _key_path = self.map._key_to_str(store_key) - _key_path = _key_path.split('/') - _chunk_path = [self.chunk_store._normalize_key(_key_path[-1])] - _key_path = '/'.join(_key_path[:-1] + _chunk_path) - self.key_path = _key_path - - def prepare_chunk(self): - assert self.buff is None - header = self.fs.read_block(self.key_path, 0, 16) - nbytes, self.cbytes, blocksize = cbuffer_sizes(header) - typesize, _shuffle, _memcpyd = cbuffer_metainfo(header) - self.buff = mmap.mmap(-1, self.cbytes) - self.buff[0:16] = header - self.nblocks = nbytes / blocksize - self.nblocks = ( - int(self.nblocks) - if self.nblocks == int(self.nblocks) - else int(self.nblocks + 1) - ) - if self.nblocks == 1: - self.buff = self.read_full() - return - start_points_buffer = self.fs.read_block( - self.key_path, 16, int(self.nblocks * 4) - ) - self.start_points = np.frombuffer( - start_points_buffer, count=self.nblocks, dtype=np.int32 - ) - self.start_points_max = self.start_points.max() - self.buff[16: (16 + (self.nblocks * 4))] = start_points_buffer - self.n_per_block = blocksize / typesize - - def read_part(self, start, nitems): - assert self.buff is not None - if self.nblocks == 1: - return - blocks_to_decompress = nitems / self.n_per_block - blocks_to_decompress = ( - blocks_to_decompress - if blocks_to_decompress == int(blocks_to_decompress) - else int(blocks_to_decompress + 1) - ) - start_block = int(start / self.n_per_block) - wanted_decompressed = 0 - while wanted_decompressed < nitems: - if start_block not in self.read_blocks: - start_byte = self.start_points[start_block] - if start_byte == self.start_points_max: - stop_byte = self.cbytes - else: - stop_byte = self.start_points[self.start_points > start_byte].min() - length = stop_byte - start_byte - data_buff = self.fs.read_block(self.key_path, start_byte, length) - self.buff[start_byte:stop_byte] = data_buff - self.read_blocks.add(start_block) - if wanted_decompressed == 0: - wanted_decompressed += ((start_block + 1) * self.n_per_block) - start - else: - wanted_decompressed += self.n_per_block - start_block += 1 - - def read_full(self): - return self.chunk_store[self.store_key] - - -def retry_call(callabl: Callable, - args=None, - kwargs=None, - exceptions: Tuple[Any, ...] = (), - retries: int = 10, - wait: float = 0.1) -> Any: - """ - Make several attempts to invoke the callable. If one of the given exceptions - is raised, wait the given period of time and retry up to the given number of - retries. - """ - - if args is None: - args = () - if kwargs is None: - kwargs = {} - - for attempt in range(1, retries+1): - try: - return callabl(*args, **kwargs) - except exceptions: - if attempt < retries: - time.sleep(wait) - else: - raise