From 30155c7ccb5dc51645ae1185bebd887f79468d55 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Fri, 17 Jan 2025 19:58:09 +0100 Subject: [PATCH 1/9] Remove id from zarr3 config serialization (#685) * Remove id from zarr3 config serialization * add test * tests * release docs --------- Co-authored-by: David Stansby --- docs/release.rst | 2 ++ numcodecs/json.py | 2 +- numcodecs/tests/test_zarr3.py | 9 +++++++-- numcodecs/zarr3.py | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index b31328e7..a012703e 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -53,6 +53,8 @@ Fixes and a warning if the version was incompatible with `zfpy`. This check has been removed because `zfpy` now supports the newer versions of `NumPy`. By :user:`Meher Gajula `, :issue:`672` +* Remove redundant ``id`` from codec metadata serialization in Zarr3 codecs. + By :user:`Norman Rzepka `, :issue:`685` Improvements ~~~~~~~~~~~~ diff --git a/numcodecs/json.py b/numcodecs/json.py index dc52e1a4..72ddb025 100644 --- a/numcodecs/json.py +++ b/numcodecs/json.py @@ -68,7 +68,7 @@ def __init__( def encode(self, buf): try: buf = np.asarray(buf) - except ValueError: + except ValueError: # pragma: no cover buf = np.asarray(buf, dtype=object) items = np.atleast_1d(buf).tolist() items.append(buf.dtype.str) diff --git a/numcodecs/tests/test_zarr3.py b/numcodecs/tests/test_zarr3.py index 1d2749d9..a40b658f 100644 --- a/numcodecs/tests/test_zarr3.py +++ b/numcodecs/tests/test_zarr3.py @@ -225,11 +225,11 @@ def test_generic_bytes_codec( ): try: codec_class()._codec # noqa: B018 - except ValueError as e: + except ValueError as e: # pragma: no cover if "codec not available" in str(e): pytest.xfail(f"{codec_class.codec_name} is not available: {e}") else: - raise # pragma: no cover + raise except ImportError as e: # pragma: no cover pytest.xfail(f"{codec_class.codec_name} is not available: {e}") @@ -272,3 +272,8 @@ def test_delta_astype(store: StorePath): def test_repr(): codec = numcodecs.zarr3.LZ4(level=5) assert repr(codec) == "LZ4(codec_name='numcodecs.lz4', codec_config={'level': 5})" + + +def test_to_dict(): + codec = numcodecs.zarr3.LZ4(level=5) + assert codec.to_dict() == {"name": "numcodecs.lz4", "configuration": {"level": 5}} diff --git a/numcodecs/zarr3.py b/numcodecs/zarr3.py index e3e98935..78293514 100644 --- a/numcodecs/zarr3.py +++ b/numcodecs/zarr3.py @@ -112,6 +112,7 @@ def from_dict(cls, data: dict[str, JSON]) -> Self: def to_dict(self) -> dict[str, JSON]: codec_config = self.codec_config.copy() + codec_config.pop("id", None) return { "name": self.codec_name, "configuration": codec_config, From ee9a110d2921ea3eb28efd839ef143403bca6bc8 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 17 Jan 2025 18:58:23 +0000 Subject: [PATCH 2/9] Enable some stricter typing rules (#679) --- numcodecs/checksum32.py | 2 +- pyproject.toml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/numcodecs/checksum32.py b/numcodecs/checksum32.py index 4a706c71..f1d82623 100644 --- a/numcodecs/checksum32.py +++ b/numcodecs/checksum32.py @@ -13,7 +13,7 @@ _crc32c: Optional[ModuleType] = None with suppress(ImportError): - import crc32c as _crc32c # type: ignore[no-redef] + import crc32c as _crc32c # type: ignore[no-redef, unused-ignore] if TYPE_CHECKING: # pragma: no cover from typing_extensions import Buffer diff --git a/pyproject.toml b/pyproject.toml index 817f138c..ca0de940 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -237,3 +237,6 @@ enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] # TODO: set options below to true strict = false warn_unreachable = false +warn_redundant_casts = true +warn_unused_ignores = true +warn_unused_configs = true From 0162bb8ff306865b54240997af610863cfe5d34a Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 17 Jan 2025 19:58:50 +0100 Subject: [PATCH 3/9] Apply ruff preview rule RUF046 (#687) RUF046 Value being casted is already an integer --- numcodecs/quantize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numcodecs/quantize.py b/numcodecs/quantize.py index b5b6cc3c..329a6d3d 100644 --- a/numcodecs/quantize.py +++ b/numcodecs/quantize.py @@ -65,9 +65,9 @@ def encode(self, buf): precision = 10.0**-self.digits exp = math.log10(precision) if exp < 0: - exp = int(math.floor(exp)) + exp = math.floor(exp) else: - exp = int(math.ceil(exp)) + exp = math.ceil(exp) bits = math.ceil(math.log2(10.0**-exp)) scale = 2.0**bits enc = np.around(scale * arr) / scale From bd517abf42b161cb10666f71081267c5c9ee241a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jan 2025 18:59:36 +0000 Subject: [PATCH 4/9] chore: update pre-commit hooks (#682) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.8.4 → v0.8.6](https://github.com/astral-sh/ruff-pre-commit/compare/v0.8.4...v0.8.6) - [github.com/pre-commit/mirrors-mypy: v1.14.0 → v1.14.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.14.0...v1.14.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4986df2c..26576ca6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: - id: debug-statements - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.4 + rev: v0.8.6 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -26,7 +26,7 @@ repos: - id: sp-repo-review - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.14.0 + rev: v1.14.1 hooks: - id: mypy args: [--config-file, pyproject.toml] From 6d090a3840339d02d71a67aac1001946ab45066b Mon Sep 17 00:00:00 2001 From: Cas Wognum Date: Fri, 17 Jan 2025 14:01:55 -0500 Subject: [PATCH 5/9] Minor UX improvement: Add the `UnknownCodecError`. (#689) * Add the UnknownCodecError * Remove trailing whitespace * Ruff formatting * Add a release note * Add doctest * Fix failing test case * Formatting again * Update numcodecs/tests/test_registry.py Co-authored-by: David Stansby * Make UnknownCodecError inherit from ValueError --------- Co-authored-by: David Stansby --- docs/release.rst | 2 ++ numcodecs/errors.py | 26 ++++++++++++++++++++++++++ numcodecs/registry.py | 3 ++- numcodecs/tests/test_registry.py | 3 ++- 4 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 numcodecs/errors.py diff --git a/docs/release.rst b/docs/release.rst index a012703e..d6852001 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -70,6 +70,8 @@ Improvements Import errors caused by optional dependencies (ZFPY, MsgPack, CRC32C, and PCodec) are still silently caught. By :user:`David Stansby `, :issue:`550`. +* Raise a custom `UnknownCodecError` when trying to retrieve an unavailable codec. + By :user:`Cas Wognum `. .. _release_0.14.1: diff --git a/numcodecs/errors.py b/numcodecs/errors.py new file mode 100644 index 00000000..78e048e6 --- /dev/null +++ b/numcodecs/errors.py @@ -0,0 +1,26 @@ +""" +This module defines custom exceptions that are raised in the `numcodecs` codebase. +""" + + +class UnknownCodecError(ValueError): + """ + An exception that is raised when trying to receive a codec that has not been registered. + + Parameters + ---------- + codec_id : str + Codec identifier. + + Examples + ---------- + >>> import numcodecs + >>> numcodecs.get_codec({"codec_id": "unknown"}) + Traceback (most recent call last): + ... + UnknownCodecError: codec not available: 'unknown' + """ + + def __init__(self, codec_id: str): + self.codec_id = codec_id + super().__init__(f"codec not available: '{codec_id}'") diff --git a/numcodecs/registry.py b/numcodecs/registry.py index cc3f3f42..ccb5b26b 100644 --- a/numcodecs/registry.py +++ b/numcodecs/registry.py @@ -5,6 +5,7 @@ from importlib.metadata import EntryPoints, entry_points from numcodecs.abc import Codec +from numcodecs.errors import UnknownCodecError logger = logging.getLogger("numcodecs") codec_registry: dict[str, Codec] = {} @@ -50,7 +51,7 @@ def get_codec(config): register_codec(cls, codec_id=codec_id) if cls: return cls.from_config(config) - raise ValueError(f'codec not available: {codec_id!r}') + raise UnknownCodecError(f"{codec_id!r}") def register_codec(cls, codec_id=None): diff --git a/numcodecs/tests/test_registry.py b/numcodecs/tests/test_registry.py index 3c779df3..93ed8f03 100644 --- a/numcodecs/tests/test_registry.py +++ b/numcodecs/tests/test_registry.py @@ -3,11 +3,12 @@ import pytest import numcodecs +from numcodecs.errors import UnknownCodecError from numcodecs.registry import get_codec def test_registry_errors(): - with pytest.raises(ValueError): + with pytest.raises(UnknownCodecError, match='foo'): get_codec({'id': 'foo'}) From 8d15c0261272d93cf276ce84043c4b5db21ad09e Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 21 Jan 2025 09:14:47 +0000 Subject: [PATCH 6/9] Update release notes post 0.15.0 (#692) --- docs/release.rst | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index d6852001..c5ba3919 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -11,6 +11,22 @@ Release notes Unreleased ---------- + +.. _unreleased: + +Unreleased +---------- + +Improvements +~~~~~~~~~~~~ +* Raise a custom `UnknownCodecError` when trying to retrieve an unavailable codec. + By :user:`Cas Wognum `. + +Fixes +~~~~~ +* Remove redundant ``id`` from codec metadata serialization in Zarr3 codecs. + By :user:`Norman Rzepka `, :issue:`685` + .. _release_0.15.0: 0.15.0 @@ -53,8 +69,6 @@ Fixes and a warning if the version was incompatible with `zfpy`. This check has been removed because `zfpy` now supports the newer versions of `NumPy`. By :user:`Meher Gajula `, :issue:`672` -* Remove redundant ``id`` from codec metadata serialization in Zarr3 codecs. - By :user:`Norman Rzepka `, :issue:`685` Improvements ~~~~~~~~~~~~ @@ -70,8 +84,6 @@ Improvements Import errors caused by optional dependencies (ZFPY, MsgPack, CRC32C, and PCodec) are still silently caught. By :user:`David Stansby `, :issue:`550`. -* Raise a custom `UnknownCodecError` when trying to retrieve an unavailable codec. - By :user:`Cas Wognum `. .. _release_0.14.1: From b8e40f7ad652ec25e52f976e3931a5f755b93e4b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 7 Feb 2025 12:10:12 -0700 Subject: [PATCH 7/9] chore: update pre-commit hooks (#696) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.8.6 → v0.9.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.8.6...v0.9.4) - [github.com/scientific-python/cookie: 2024.08.19 → 2025.01.22](https://github.com/scientific-python/cookie/compare/2024.08.19...2025.01.22) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 26576ca6..5728b2fb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,14 +14,14 @@ repos: - id: debug-statements - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.6 + rev: v0.9.4 hooks: - id: ruff args: ["--fix", "--show-fixes"] - id: ruff-format - repo: https://github.com/scientific-python/cookie - rev: 2024.08.19 + rev: 2025.01.22 hooks: - id: sp-repo-review From 8a0bd3bc6223e338d58ac9cd7bfa1481b2ad85ab Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 7 Feb 2025 21:02:26 +0100 Subject: [PATCH 8/9] Upgrade ruff to 0.9.1 (#688) Starting with ruff 0.9.1, rules ISC001/ISC002 are documented to be compatible with the ruff formatter, if they are both enabled. Enable them! Co-authored-by: Deepak Cherian --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ca0de940..29ef0bf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -219,8 +219,6 @@ ignore = [ "Q003", "COM812", "COM819", - "ISC001", - "ISC002", ] [tool.ruff.lint.extend-per-file-ignores] From 3cf8ab1f28e6fbda24fa16c33654cf560a8d4ae2 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Sat, 8 Feb 2025 11:20:41 -0700 Subject: [PATCH 9/9] Don't overwrite codec config values when set. (#700) * Don't overwrite codec config values when set. A codec doesn't know where it is in the codec pipeline, so it cannot know whether `array_spec.dtype` should match `codec.dtype` or not. * little more * FIx test * fix test --------- Co-authored-by: Davis Bennett --- numcodecs/tests/test_zarr3.py | 2 +- numcodecs/zarr3.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/numcodecs/tests/test_zarr3.py b/numcodecs/tests/test_zarr3.py index a40b658f..0d8ecc74 100644 --- a/numcodecs/tests/test_zarr3.py +++ b/numcodecs/tests/test_zarr3.py @@ -91,7 +91,7 @@ def test_generic_compressor( (numcodecs.zarr3.Delta, {"dtype": "float32"}), (numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 25.5}), (numcodecs.zarr3.FixedScaleOffset, {"offset": 0, "scale": 51, "astype": "uint16"}), - (numcodecs.zarr3.AsType, {"encode_dtype": "float32", "decode_dtype": "float64"}), + (numcodecs.zarr3.AsType, {"encode_dtype": "float32", "decode_dtype": "float32"}), ], ids=[ "delta", diff --git a/numcodecs/zarr3.py b/numcodecs/zarr3.py index 78293514..f1743ffb 100644 --- a/numcodecs/zarr3.py +++ b/numcodecs/zarr3.py @@ -271,7 +271,7 @@ def __init__(self, **codec_config: JSON) -> None: super().__init__(**codec_config) def evolve_from_array_spec(self, array_spec: ArraySpec) -> Shuffle: - if array_spec.dtype.itemsize != self.codec_config.get("elementsize"): + if self.codec_config.get("elementsize", None) is None: return Shuffle(**{**self.codec_config, "elementsize": array_spec.dtype.itemsize}) return self # pragma: no cover @@ -308,7 +308,7 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: return chunk_spec def evolve_from_array_spec(self, array_spec: ArraySpec) -> FixedScaleOffset: - if str(array_spec.dtype) != self.codec_config.get("dtype"): + if self.codec_config.get("dtype", None) is None: return FixedScaleOffset(**{**self.codec_config, "dtype": str(array_spec.dtype)}) return self @@ -321,7 +321,7 @@ def __init__(self, **codec_config: JSON) -> None: super().__init__(**codec_config) def evolve_from_array_spec(self, array_spec: ArraySpec) -> Quantize: - if str(array_spec.dtype) != self.codec_config.get("dtype"): + if self.codec_config.get("dtype", None) is None: return Quantize(**{**self.codec_config, "dtype": str(array_spec.dtype)}) return self @@ -356,8 +356,7 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: return replace(chunk_spec, dtype=np.dtype(self.codec_config["encode_dtype"])) # type: ignore[arg-type] def evolve_from_array_spec(self, array_spec: ArraySpec) -> AsType: - decode_dtype = self.codec_config.get("decode_dtype") - if str(array_spec.dtype) != decode_dtype: + if self.codec_config.get("decode_dtype", None) is None: return AsType(**{**self.codec_config, "decode_dtype": str(array_spec.dtype)}) return self