From 1d76cb42720398e4c50664eda699edc99dda9fcd Mon Sep 17 00:00:00 2001 From: mzouink Date: Fri, 3 Feb 2023 14:08:11 -0500 Subject: [PATCH 1/9] create a test case for the bug --- zarr/tests/test_n5.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/zarr/tests/test_n5.py b/zarr/tests/test_n5.py index a1a0a83e36..dc4b0fdd2d 100644 --- a/zarr/tests/test_n5.py +++ b/zarr/tests/test_n5.py @@ -1,11 +1,14 @@ import pytest -from zarr.n5 import N5ChunkWrapper +from zarr.n5 import N5ChunkWrapper,N5FSStore +from zarr.storage import MemoryStore from numcodecs import GZip import numpy as np from typing import Tuple - +import shutil +import json +import zarr def test_make_n5_chunk_wrapper(): dtype = 'uint8' @@ -35,3 +38,17 @@ def test_partial_chunk_decode(chunk_shape: Tuple[int, ...]): chunk[subslices] = 1 subchunk = np.ascontiguousarray(chunk[subslices]) assert np.array_equal(codec_wrapped.decode(codec_wrapped.encode(subchunk)), chunk) + + +def test_dtype_decode(): + store = 'data/array.n5' + n5_store = N5FSStore(store) + z = zarr.open(n5_store) + z.create_dataset("test", shape=100, dtype="u1") + dtype_n5 = json.loads(n5_store["test/.zarray"])["dtype"] + shutil.rmtree(store) + zarr_store = MemoryStore() + z = zarr.open(zarr_store) + z.create_dataset("test", shape=100, dtype="u1") + dtype_zarr = json.loads(zarr_store["test/.zarray"])["dtype"] + assert dtype_n5 == dtype_zarr From f3f2bfe47d4308ab2692a1a1d4478fe199a261ce Mon Sep 17 00:00:00 2001 From: mzouink Date: Fri, 3 Feb 2023 14:12:13 -0500 Subject: [PATCH 2/9] fix dtype bug --- zarr/n5.py | 1 + zarr/tests/test_n5.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/zarr/n5.py b/zarr/n5.py index 4c93ce4acb..1eb6ef2b33 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -689,6 +689,7 @@ def array_metadata_to_zarr(array_metadata: Dict[str, Any], array_metadata['order'] = 'C' array_metadata['filters'] = [] array_metadata['dimension_separator'] = '.' + array_metadata['dtype'] = np.dtype(array_metadata['dtype']).str compressor_config = array_metadata['compressor'] compressor_config = compressor_config_to_zarr(compressor_config) diff --git a/zarr/tests/test_n5.py b/zarr/tests/test_n5.py index dc4b0fdd2d..5cadf9fccd 100644 --- a/zarr/tests/test_n5.py +++ b/zarr/tests/test_n5.py @@ -1,7 +1,7 @@ import pytest -from zarr.n5 import N5ChunkWrapper,N5FSStore +from zarr.n5 import N5ChunkWrapper, N5FSStore from zarr.storage import MemoryStore from numcodecs import GZip import numpy as np From fb122ec15a179511376a1e1ff74d8eff20005c4f Mon Sep 17 00:00:00 2001 From: mzouink Date: Fri, 3 Feb 2023 14:21:48 -0500 Subject: [PATCH 3/9] fix formatting --- zarr/tests/test_n5.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zarr/tests/test_n5.py b/zarr/tests/test_n5.py index 5cadf9fccd..a410d1cdad 100644 --- a/zarr/tests/test_n5.py +++ b/zarr/tests/test_n5.py @@ -10,6 +10,7 @@ import json import zarr + def test_make_n5_chunk_wrapper(): dtype = 'uint8' chunk_shape = (10,) From fd0f69140ac95b1bea4bce5b55c411ca962781ac Mon Sep 17 00:00:00 2001 From: mzouink Date: Fri, 3 Feb 2023 15:06:59 -0500 Subject: [PATCH 4/9] optimize code by using create function --- zarr/tests/test_n5.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/zarr/tests/test_n5.py b/zarr/tests/test_n5.py index a410d1cdad..bdc9a81184 100644 --- a/zarr/tests/test_n5.py +++ b/zarr/tests/test_n5.py @@ -2,13 +2,12 @@ import pytest from zarr.n5 import N5ChunkWrapper, N5FSStore -from zarr.storage import MemoryStore +from zarr.creation import create from numcodecs import GZip import numpy as np from typing import Tuple import shutil import json -import zarr def test_make_n5_chunk_wrapper(): @@ -44,12 +43,8 @@ def test_partial_chunk_decode(chunk_shape: Tuple[int, ...]): def test_dtype_decode(): store = 'data/array.n5' n5_store = N5FSStore(store) - z = zarr.open(n5_store) - z.create_dataset("test", shape=100, dtype="u1") + create(100, store=n5_store) dtype_n5 = json.loads(n5_store["test/.zarray"])["dtype"] shutil.rmtree(store) - zarr_store = MemoryStore() - z = zarr.open(zarr_store) - z.create_dataset("test", shape=100, dtype="u1") - dtype_zarr = json.loads(zarr_store["test/.zarray"])["dtype"] + dtype_zarr = json.loads(create(100).store["test/.zarray"])["dtype"] assert dtype_n5 == dtype_zarr From 2e5046195263809e6165210fe1f05e9295fa4b70 Mon Sep 17 00:00:00 2001 From: mzouink Date: Fri, 3 Feb 2023 15:11:28 -0500 Subject: [PATCH 5/9] use at exit delete --- zarr/tests/test_n5.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/zarr/tests/test_n5.py b/zarr/tests/test_n5.py index bdc9a81184..255294fb92 100644 --- a/zarr/tests/test_n5.py +++ b/zarr/tests/test_n5.py @@ -3,11 +3,13 @@ from zarr.n5 import N5ChunkWrapper, N5FSStore from zarr.creation import create +from zarr.storage import atexit_rmtree from numcodecs import GZip import numpy as np from typing import Tuple import shutil import json +import atexit def test_make_n5_chunk_wrapper(): @@ -41,10 +43,10 @@ def test_partial_chunk_decode(chunk_shape: Tuple[int, ...]): def test_dtype_decode(): - store = 'data/array.n5' - n5_store = N5FSStore(store) + path = 'data/array.n5' + atexit.register(atexit_rmtree, path) + n5_store = N5FSStore(path) create(100, store=n5_store) - dtype_n5 = json.loads(n5_store["test/.zarray"])["dtype"] - shutil.rmtree(store) - dtype_zarr = json.loads(create(100).store["test/.zarray"])["dtype"] + dtype_n5 = json.loads(n5_store[".zarray"])["dtype"] + dtype_zarr = json.loads(create(100).store[".zarray"])["dtype"] assert dtype_n5 == dtype_zarr From 1f3f2e27bff923d6b598caadbbcfcdbf8934d95a Mon Sep 17 00:00:00 2001 From: mzouink Date: Fri, 3 Feb 2023 15:12:49 -0500 Subject: [PATCH 6/9] optimize import --- zarr/tests/test_n5.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zarr/tests/test_n5.py b/zarr/tests/test_n5.py index 255294fb92..06191f8364 100644 --- a/zarr/tests/test_n5.py +++ b/zarr/tests/test_n5.py @@ -7,7 +7,6 @@ from numcodecs import GZip import numpy as np from typing import Tuple -import shutil import json import atexit From 540f6af0ed0f5907f7c056faa2cc33858d2bc253 Mon Sep 17 00:00:00 2001 From: mzouink Date: Thu, 9 Feb 2023 11:33:24 -0500 Subject: [PATCH 7/9] update n5 test hexdigest --- zarr/tests/test_core.py | 13 ++++++------- zarr/tests/test_n5.py | 1 + 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index b54fe3ddf0..ba89db3b06 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -2034,13 +2034,12 @@ def test_compressors(self): assert np.all(a2[:] == 1) def expected(self): - return [ - '4e9cf910000506455f82a70938a272a3fce932e5', - 'f9d4cbf1402901f63dea7acf764d2546e4b6aa38', - '1d8199f5f7b70d61aa0d29cc375212c3df07d50a', - '874880f91aa6736825584509144afe6b06b0c05c', - 'e2258fedc74752196a8c8383db49e27193c995e2', - ] + return ['8811a77d54caaa1901d5cc4452d946ae433c8d90', + 'd880b007d9779db5f2cdbe13274eb1cbac4a425a', + 'd80eb66d5521744f051e816ab368d8ccfc2e3edf', + '568f9f837e4b682a3819cb122988e2eebeb6572b', + '4fdf4475d786d6694110db5619acd30c80dfc372' + ] @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") diff --git a/zarr/tests/test_n5.py b/zarr/tests/test_n5.py index 06191f8364..223662828f 100644 --- a/zarr/tests/test_n5.py +++ b/zarr/tests/test_n5.py @@ -43,6 +43,7 @@ def test_partial_chunk_decode(chunk_shape: Tuple[int, ...]): def test_dtype_decode(): path = 'data/array.n5' + atexit_rmtree(path) atexit.register(atexit_rmtree, path) n5_store = N5FSStore(path) create(100, store=n5_store) From f022c08061697fb55ee372d978ad2d0a7a7df027 Mon Sep 17 00:00:00 2001 From: mzouink Date: Fri, 10 Feb 2023 10:37:59 -0500 Subject: [PATCH 8/9] skip dtype decode if no fsspec --- zarr/tests/test_n5.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zarr/tests/test_n5.py b/zarr/tests/test_n5.py index 223662828f..8f6d97dd51 100644 --- a/zarr/tests/test_n5.py +++ b/zarr/tests/test_n5.py @@ -1,4 +1,3 @@ - import pytest from zarr.n5 import N5ChunkWrapper, N5FSStore @@ -10,6 +9,8 @@ import json import atexit +from zarr.tests.util import have_fsspec + def test_make_n5_chunk_wrapper(): dtype = 'uint8' @@ -41,6 +42,7 @@ def test_partial_chunk_decode(chunk_shape: Tuple[int, ...]): assert np.array_equal(codec_wrapped.decode(codec_wrapped.encode(subchunk)), chunk) +@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") def test_dtype_decode(): path = 'data/array.n5' atexit_rmtree(path) From d45bc118793d79bc6fb7e8ebe832efa8db8a5d6a Mon Sep 17 00:00:00 2001 From: mzouink Date: Tue, 14 Feb 2023 16:50:59 -0500 Subject: [PATCH 9/9] add contribution --- docs/release.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 0965109935..01fc2fd318 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -47,6 +47,9 @@ Bug fixes * NestedDirectoryStore.listdir now returns chunk keys with the correct '/' dimension_separator. By :user:`Brett Graham ` :issue:`1334`. +* N5Store/N5FSStore dtype returns zarr Stores readable dtype. + By :user:`Marwan Zouinkhi ` :issue:`1339`. + .. _release_2.13.6: 2.13.6