From 792657c657fde5374fa65e66f0f8aca3a50820b2 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 10:34:55 +0200 Subject: [PATCH 01/39] Move into ZarrCore.jl - Move src/ to lib/ZarrCore/src/ - Move deps, establish workspace, etc. - Change some names around to make Zarr.jl references into ZarrCore.jl references --- Project.toml | 41 +++--------------- lib/ZarrCore/Project.toml | 43 +++++++++++++++++++ {src => lib/ZarrCore/src}/Codecs/Codecs.jl | 0 {src => lib/ZarrCore/src}/Codecs/V3/V3.jl | 6 +-- .../ZarrCore/src}/Compressors/Compressors.jl | 0 .../ZarrCore/src}/Compressors/blosc.jl | 2 +- {src => lib/ZarrCore/src}/Compressors/zlib.jl | 2 +- {src => lib/ZarrCore/src}/Compressors/zstd.jl | 2 +- {src => lib/ZarrCore/src}/Filters/Filters.jl | 0 {src => lib/ZarrCore/src}/Filters/delta.jl | 0 .../ZarrCore/src}/Filters/fixedscaleoffset.jl | 0 .../ZarrCore/src}/Filters/fletcher32.jl | 0 {src => lib/ZarrCore/src}/Filters/quantize.jl | 0 {src => lib/ZarrCore/src}/Filters/shuffle.jl | 0 .../ZarrCore/src}/Filters/vlenfilters.jl | 0 {src => lib/ZarrCore/src}/MaxLengthStrings.jl | 0 {src => lib/ZarrCore/src}/Storage/Storage.jl | 0 .../ZarrCore/src}/Storage/cachingstore.jl | 0 .../ZarrCore/src}/Storage/consolidated.jl | 0 .../ZarrCore/src}/Storage/dictstore.jl | 0 .../ZarrCore/src}/Storage/directorystore.jl | 0 {src => lib/ZarrCore/src}/Storage/gcstore.jl | 4 +- {src => lib/ZarrCore/src}/Storage/http.jl | 2 +- {src => lib/ZarrCore/src}/Storage/zipstore.jl | 0 {src => lib/ZarrCore/src}/ZArray.jl | 0 {src => lib/ZarrCore/src}/ZGroup.jl | 2 +- lib/ZarrCore/src/ZarrCore.jl | 33 ++++++++++++++ {src => lib/ZarrCore/src}/caching.jl | 0 {src => lib/ZarrCore/src}/chunkkeyencoding.jl | 0 {src => lib/ZarrCore/src}/metadata.jl | 4 +- {src => lib/ZarrCore/src}/metadata3.jl | 0 {src => lib/ZarrCore/src}/pipeline.jl | 0 {src => lib/ZarrCore/src}/types.jl | 0 src/Zarr.jl | 32 ++------------ 34 files changed, 99 insertions(+), 74 deletions(-) create mode 100644 lib/ZarrCore/Project.toml rename {src => lib/ZarrCore/src}/Codecs/Codecs.jl (100%) rename {src => lib/ZarrCore/src}/Codecs/V3/V3.jl (99%) rename {src => lib/ZarrCore/src}/Compressors/Compressors.jl (100%) rename {src => lib/ZarrCore/src}/Compressors/blosc.jl (97%) rename {src => lib/ZarrCore/src}/Compressors/zlib.jl (96%) rename {src => lib/ZarrCore/src}/Compressors/zstd.jl (97%) rename {src => lib/ZarrCore/src}/Filters/Filters.jl (100%) rename {src => lib/ZarrCore/src}/Filters/delta.jl (100%) rename {src => lib/ZarrCore/src}/Filters/fixedscaleoffset.jl (100%) rename {src => lib/ZarrCore/src}/Filters/fletcher32.jl (100%) rename {src => lib/ZarrCore/src}/Filters/quantize.jl (100%) rename {src => lib/ZarrCore/src}/Filters/shuffle.jl (100%) rename {src => lib/ZarrCore/src}/Filters/vlenfilters.jl (100%) rename {src => lib/ZarrCore/src}/MaxLengthStrings.jl (100%) rename {src => lib/ZarrCore/src}/Storage/Storage.jl (100%) rename {src => lib/ZarrCore/src}/Storage/cachingstore.jl (100%) rename {src => lib/ZarrCore/src}/Storage/consolidated.jl (100%) rename {src => lib/ZarrCore/src}/Storage/dictstore.jl (100%) rename {src => lib/ZarrCore/src}/Storage/directorystore.jl (100%) rename {src => lib/ZarrCore/src}/Storage/gcstore.jl (96%) rename {src => lib/ZarrCore/src}/Storage/http.jl (96%) rename {src => lib/ZarrCore/src}/Storage/zipstore.jl (100%) rename {src => lib/ZarrCore/src}/ZArray.jl (100%) rename {src => lib/ZarrCore/src}/ZGroup.jl (98%) create mode 100644 lib/ZarrCore/src/ZarrCore.jl rename {src => lib/ZarrCore/src}/caching.jl (100%) rename {src => lib/ZarrCore/src}/chunkkeyencoding.jl (100%) rename {src => lib/ZarrCore/src}/metadata.jl (98%) rename {src => lib/ZarrCore/src}/metadata3.jl (100%) rename {src => lib/ZarrCore/src}/pipeline.jl (100%) rename {src => lib/ZarrCore/src}/types.jl (100%) diff --git a/Project.toml b/Project.toml index c4457bfd..60cdfb53 100644 --- a/Project.toml +++ b/Project.toml @@ -3,25 +3,14 @@ uuid = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" version = "0.10.1" authors = ["Fabian Gans "] +[sources] +ZarrCore = { path = "lib/ZarrCore" } + +[workspace] +projects = ["test", "lib/ZarrCore"] + [deps] -Blosc = "a74b3585-a348-5f62-a45c-50e91977d574" -CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" -ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" -ChunkCodecLibZlib = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" -ChunkCodecLibZstd = "55437552-ac27-4d47-9aa3-63184e8fd398" -DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" -HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" -JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -OpenSSL = "4d8831e6-92b7-49fb-bdf8-b643e874388c" -OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -ZipArchives = "49080126-0e18-4c2a-b176-c102e4b3760c" +ZarrCore = "77f5b75c-4c08-499f-ba13-550b0a0af171" [weakdeps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" @@ -31,20 +20,4 @@ ZarrAWSS3Ext = "AWSS3" [compat] AWSS3 = "0.10, 0.11" -Blosc = "0.5, 0.6, 0.7" -CRC32c = "1.10, 1.11" -ChunkCodecCore = "1" -ChunkCodecLibZlib = "1" -ChunkCodecLibZstd = "1" -DataStructures = "0.17, 0.18, 0.19" -DateTimes64 = "1" -DiskArrays = "0.4.21" -HTTP = "^1.3, 2" -JSON = "0.21, 1" -OffsetArrays = "0.11, 1.0" -OpenSSL = "1" -OrderedCollections = "1.8.2, 2" -URIs = "1" -Unicode = "1.10, 1.11.0" -ZipArchives = "2" julia = "1.10" diff --git a/lib/ZarrCore/Project.toml b/lib/ZarrCore/Project.toml new file mode 100644 index 00000000..e1c2ed6e --- /dev/null +++ b/lib/ZarrCore/Project.toml @@ -0,0 +1,43 @@ +name = "Zarr" +uuid = "77f5b75c-4c08-499f-ba13-550b0a0af171" +version = "0.10.1" +authors = ["Fabian Gans "] + +[deps] +Blosc = "a74b3585-a348-5f62-a45c-50e91977d574" +CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" +ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" +ChunkCodecLibZlib = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" +ChunkCodecLibZstd = "55437552-ac27-4d47-9aa3-63184e8fd398" +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +OpenSSL = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +ZipArchives = "49080126-0e18-4c2a-b176-c102e4b3760c" + +[compat] +Blosc = "0.5, 0.6, 0.7" +CRC32c = "1.10, 1.11" +ChunkCodecCore = "1" +ChunkCodecLibZlib = "1" +ChunkCodecLibZstd = "1" +DataStructures = "0.17, 0.18, 0.19" +DateTimes64 = "1" +DiskArrays = "0.4.21" +HTTP = "^1.3, 2" +JSON = "0.21, 1" +OffsetArrays = "0.11, 1.0" +OpenSSL = "1" +OrderedCollections = "1.8.2, 2" +URIs = "1" +Unicode = "1.10, 1.11.0" +ZipArchives = "2" +julia = "1.10" diff --git a/src/Codecs/Codecs.jl b/lib/ZarrCore/src/Codecs/Codecs.jl similarity index 100% rename from src/Codecs/Codecs.jl rename to lib/ZarrCore/src/Codecs/Codecs.jl diff --git a/src/Codecs/V3/V3.jl b/lib/ZarrCore/src/Codecs/V3/V3.jl similarity index 99% rename from src/Codecs/V3/V3.jl rename to lib/ZarrCore/src/Codecs/V3/V3.jl index d3c4fa41..c3acc446 100644 --- a/src/Codecs/V3/V3.jl +++ b/lib/ZarrCore/src/Codecs/V3/V3.jl @@ -2,9 +2,9 @@ module V3Codecs import ..Codecs: zencode, zdecode, zencode!, zdecode! # Import compressor types and functions from Zarr (grandparent module) -import ...Zarr: ZlibCompressor, ZstdCompressor, zcompress, zuncompress -import ...Zarr: BloscCompressor as ZarrBloscCompressor -import ...Zarr: AbstractCodecPipeline, V3Pipeline, pipeline_encode, pipeline_decode! +import ...ZarrCore: ZlibCompressor, ZstdCompressor, zcompress, zuncompress +import ...ZarrCore: BloscCompressor as ZarrBloscCompressor +import ...ZarrCore: AbstractCodecPipeline, V3Pipeline, pipeline_encode, pipeline_decode! using CRC32c: CRC32c using JSON: JSON using ChunkCodecLibZlib: GzipCodec as LibZGzipCodec, GzipEncodeOptions diff --git a/src/Compressors/Compressors.jl b/lib/ZarrCore/src/Compressors/Compressors.jl similarity index 100% rename from src/Compressors/Compressors.jl rename to lib/ZarrCore/src/Compressors/Compressors.jl diff --git a/src/Compressors/blosc.jl b/lib/ZarrCore/src/Compressors/blosc.jl similarity index 97% rename from src/Compressors/blosc.jl rename to lib/ZarrCore/src/Compressors/blosc.jl index 789a298f..72d37b79 100644 --- a/src/Compressors/blosc.jl +++ b/lib/ZarrCore/src/Compressors/blosc.jl @@ -67,4 +67,4 @@ end JSON.lower(c::BloscCompressor) = Dict("id"=>"blosc", "cname"=>c.cname, "clevel"=>c.clevel, "shuffle"=>c.shuffle, "blocksize"=>c.blocksize) -Zarr.compressortypes["blosc"] = BloscCompressor \ No newline at end of file +ZarrCore.compressortypes["blosc"] = BloscCompressor \ No newline at end of file diff --git a/src/Compressors/zlib.jl b/lib/ZarrCore/src/Compressors/zlib.jl similarity index 96% rename from src/Compressors/zlib.jl rename to lib/ZarrCore/src/Compressors/zlib.jl index 6b82feef..92f2371a 100644 --- a/src/Compressors/zlib.jl +++ b/lib/ZarrCore/src/Compressors/zlib.jl @@ -42,4 +42,4 @@ end JSON.lower(z::ZlibCompressor) = Dict("id"=>"zlib", "level" => z.config.level) -Zarr.compressortypes["zlib"] = ZlibCompressor \ No newline at end of file +ZarrCore.compressortypes["zlib"] = ZlibCompressor \ No newline at end of file diff --git a/src/Compressors/zstd.jl b/lib/ZarrCore/src/Compressors/zstd.jl similarity index 97% rename from src/Compressors/zstd.jl rename to lib/ZarrCore/src/Compressors/zstd.jl index 6cd80a08..c6648ee2 100644 --- a/src/Compressors/zstd.jl +++ b/lib/ZarrCore/src/Compressors/zstd.jl @@ -52,4 +52,4 @@ function JSON.lower(z::ZstdCompressor) end end -Zarr.compressortypes["zstd"] = ZstdCompressor +ZarrCore.compressortypes["zstd"] = ZstdCompressor diff --git a/src/Filters/Filters.jl b/lib/ZarrCore/src/Filters/Filters.jl similarity index 100% rename from src/Filters/Filters.jl rename to lib/ZarrCore/src/Filters/Filters.jl diff --git a/src/Filters/delta.jl b/lib/ZarrCore/src/Filters/delta.jl similarity index 100% rename from src/Filters/delta.jl rename to lib/ZarrCore/src/Filters/delta.jl diff --git a/src/Filters/fixedscaleoffset.jl b/lib/ZarrCore/src/Filters/fixedscaleoffset.jl similarity index 100% rename from src/Filters/fixedscaleoffset.jl rename to lib/ZarrCore/src/Filters/fixedscaleoffset.jl diff --git a/src/Filters/fletcher32.jl b/lib/ZarrCore/src/Filters/fletcher32.jl similarity index 100% rename from src/Filters/fletcher32.jl rename to lib/ZarrCore/src/Filters/fletcher32.jl diff --git a/src/Filters/quantize.jl b/lib/ZarrCore/src/Filters/quantize.jl similarity index 100% rename from src/Filters/quantize.jl rename to lib/ZarrCore/src/Filters/quantize.jl diff --git a/src/Filters/shuffle.jl b/lib/ZarrCore/src/Filters/shuffle.jl similarity index 100% rename from src/Filters/shuffle.jl rename to lib/ZarrCore/src/Filters/shuffle.jl diff --git a/src/Filters/vlenfilters.jl b/lib/ZarrCore/src/Filters/vlenfilters.jl similarity index 100% rename from src/Filters/vlenfilters.jl rename to lib/ZarrCore/src/Filters/vlenfilters.jl diff --git a/src/MaxLengthStrings.jl b/lib/ZarrCore/src/MaxLengthStrings.jl similarity index 100% rename from src/MaxLengthStrings.jl rename to lib/ZarrCore/src/MaxLengthStrings.jl diff --git a/src/Storage/Storage.jl b/lib/ZarrCore/src/Storage/Storage.jl similarity index 100% rename from src/Storage/Storage.jl rename to lib/ZarrCore/src/Storage/Storage.jl diff --git a/src/Storage/cachingstore.jl b/lib/ZarrCore/src/Storage/cachingstore.jl similarity index 100% rename from src/Storage/cachingstore.jl rename to lib/ZarrCore/src/Storage/cachingstore.jl diff --git a/src/Storage/consolidated.jl b/lib/ZarrCore/src/Storage/consolidated.jl similarity index 100% rename from src/Storage/consolidated.jl rename to lib/ZarrCore/src/Storage/consolidated.jl diff --git a/src/Storage/dictstore.jl b/lib/ZarrCore/src/Storage/dictstore.jl similarity index 100% rename from src/Storage/dictstore.jl rename to lib/ZarrCore/src/Storage/dictstore.jl diff --git a/src/Storage/directorystore.jl b/lib/ZarrCore/src/Storage/directorystore.jl similarity index 100% rename from src/Storage/directorystore.jl rename to lib/ZarrCore/src/Storage/directorystore.jl diff --git a/src/Storage/gcstore.jl b/lib/ZarrCore/src/Storage/gcstore.jl similarity index 96% rename from src/Storage/gcstore.jl rename to lib/ZarrCore/src/Storage/gcstore.jl index 02a2051c..86499d46 100644 --- a/src/Storage/gcstore.jl +++ b/lib/ZarrCore/src/Storage/gcstore.jl @@ -5,7 +5,7 @@ const GOOGLE_STORAGE_REST_API = GOOGLE_STORAGE_API * "/storage/v1" const GOOGLE_STORAGE_CREDENTIALS = Dict{String,String}() """ - Zarr.gcs_credentials(user_project,access_token,token_type) + gcs_credentials(user_project,access_token,token_type) Set the user project, access token and and token type for the Google Cloud Store. @@ -18,7 +18,7 @@ function gcs_credentials(user_project,access_token,token_type) end """ - Zarr.gcs_credentials(; metadata_url = "http://metadata.google.internal/computeMetadata/v1/") + gcs_credentials(; metadata_url = "http://metadata.google.internal/computeMetadata/v1/") Set (or renew) the user project, access token and and token type for the Google Cloud Store from the Metadata server (assuming the function is executed from diff --git a/src/Storage/http.jl b/lib/ZarrCore/src/Storage/http.jl similarity index 96% rename from src/Storage/http.jl rename to lib/ZarrCore/src/Storage/http.jl index 07b1f7bf..2f2e47f5 100644 --- a/src/Storage/http.jl +++ b/lib/ZarrCore/src/Storage/http.jl @@ -28,7 +28,7 @@ if r.status >= 300 """Received error code $(r.status) when connecting to $(s.url) with message $(String(r.body)). This might be an actual error, or an indication that the server returns a different error code than 404 for missing chunks. In the latter case, you can run - `Zarr.missing_chunk_return_code!(a.storage,$(r.status))` where `a` is your Zarr array or group, + `ZarrCore.missing_chunk_return_code!(a.storage,$(r.status))` where `a` is your Zarr array or group, to fix the issue. """ throw(ErrorException(err_msg)) diff --git a/src/Storage/zipstore.jl b/lib/ZarrCore/src/Storage/zipstore.jl similarity index 100% rename from src/Storage/zipstore.jl rename to lib/ZarrCore/src/Storage/zipstore.jl diff --git a/src/ZArray.jl b/lib/ZarrCore/src/ZArray.jl similarity index 100% rename from src/ZArray.jl rename to lib/ZarrCore/src/ZArray.jl diff --git a/src/ZGroup.jl b/lib/ZarrCore/src/ZGroup.jl similarity index 98% rename from src/ZGroup.jl rename to lib/ZarrCore/src/ZGroup.jl index b4e99cef..83bb1e99 100644 --- a/src/ZGroup.jl +++ b/lib/ZarrCore/src/ZGroup.jl @@ -62,7 +62,7 @@ function zopen_noerr(s::AbstractStore, mode, zv::ZarrFormat; consolidated && return zopen(ConsolidatedStore(s, path), mode, path=path, lru=lru, fill_as_missing=fill_as_missing) if lru !== 0 - error("LRU caches are not supported anymore by the current Zarr version. Please use an earlier version of Zarr for now and open an issue at Zarr.jl if you need this functionality") + error("LRU caches are not supported anymore by the current Zarr version. Please use an earlier version of Zarr for now and open an issue at https://github.com/JuliaIO/Zarr.jl if you need this functionality") end if is_zarray(zv, s, path) return ZArray(s, mode, path, zv; fill_as_missing=fill_as_missing) diff --git a/lib/ZarrCore/src/ZarrCore.jl b/lib/ZarrCore/src/ZarrCore.jl new file mode 100644 index 00000000..9d04fe1a --- /dev/null +++ b/lib/ZarrCore/src/ZarrCore.jl @@ -0,0 +1,33 @@ +module ZarrCore + +import JSON +import Blosc +import Unicode +using OrderedCollections: OrderedDict + +struct ZarrFormat{V} + version::Val{V} +end +Base.Int(v::ZarrFormat{V}) where V = V +@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) +ZarrFormat(v::ZarrFormat) = v +#Default Zarr Version +const DV = ZarrFormat(Val(2)) + +include("types.jl") +include("chunkkeyencoding.jl") +include("metadata.jl") +include("metadata3.jl") +include("Compressors/Compressors.jl") +include("Codecs/Codecs.jl") +include("Storage/Storage.jl") +include("Filters/Filters.jl") +include("ZArray.jl") +include("pipeline.jl") +include("ZGroup.jl") +include("caching.jl") + +export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, + zinfo, DirectoryStore, S3Store, GCStore, zgroup + +end # module diff --git a/src/caching.jl b/lib/ZarrCore/src/caching.jl similarity index 100% rename from src/caching.jl rename to lib/ZarrCore/src/caching.jl diff --git a/src/chunkkeyencoding.jl b/lib/ZarrCore/src/chunkkeyencoding.jl similarity index 100% rename from src/chunkkeyencoding.jl rename to lib/ZarrCore/src/chunkkeyencoding.jl diff --git a/src/metadata.jl b/lib/ZarrCore/src/metadata.jl similarity index 98% rename from src/metadata.jl rename to lib/ZarrCore/src/metadata.jl index 108844c0..2e17326a 100644 --- a/src/metadata.jl +++ b/lib/ZarrCore/src/metadata.jl @@ -198,7 +198,7 @@ Metadata(s::Union{AbstractString, IO}, fill_as_missing) = Metadata(JSON.parse(s; "Construct Metadata from Dict" function Metadata(d::AbstractDict, fill_as_missing) zarr_format = d["zarr_format"]::Int - zarr_format ∉ (2, 3) && throw(ArgumentError("Zarr.jl currently only supports v2 or v3 of the specification")) + zarr_format ∉ (2, 3) && throw(ArgumentError("ZarrCore.jl currently only supports v2 or v3 of the specification")) return Metadata(d, fill_as_missing, ZarrFormat(zarr_format)) end @@ -284,7 +284,7 @@ fill_value_decoding(v::Nothing, ::Any) = v fill_value_decoding(v, T) = T(v) fill_value_decoding(v::Number, T::Type{String}) = v == 0 ? "" : T(UInt8[v]) fill_value_decoding(v, ::Type{ASCIIChar}) = v == "" ? nothing : v -fill_value_decoding(v::Nothing, ::Type{Zarr.ASCIIChar}) = v +fill_value_decoding(v::Nothing, ::Type{ZarrCore.ASCIIChar}) = v # Sometimes when translating between CF (climate and forecast) convention data # and Zarr groups, fill values are left as "negative integers" to encode unsigned # integers. So, we have to convert to the signed type with the same number of bytes diff --git a/src/metadata3.jl b/lib/ZarrCore/src/metadata3.jl similarity index 100% rename from src/metadata3.jl rename to lib/ZarrCore/src/metadata3.jl diff --git a/src/pipeline.jl b/lib/ZarrCore/src/pipeline.jl similarity index 100% rename from src/pipeline.jl rename to lib/ZarrCore/src/pipeline.jl diff --git a/src/types.jl b/lib/ZarrCore/src/types.jl similarity index 100% rename from src/types.jl rename to lib/ZarrCore/src/types.jl diff --git a/src/Zarr.jl b/src/Zarr.jl index 8292da9e..55a57747 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -1,33 +1,9 @@ module Zarr -import JSON -import Blosc -import Unicode -using OrderedCollections: OrderedDict +import ZarrCore -struct ZarrFormat{V} - version::Val{V} +for name in names(ZarrCore; all = false) + @eval export $name end -Base.Int(v::ZarrFormat{V}) where V = V -@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) -ZarrFormat(v::ZarrFormat) = v -#Default Zarr Version -const DV = ZarrFormat(Val(2)) -include("types.jl") -include("chunkkeyencoding.jl") -include("metadata.jl") -include("metadata3.jl") -include("Compressors/Compressors.jl") -include("Codecs/Codecs.jl") -include("Storage/Storage.jl") -include("Filters/Filters.jl") -include("ZArray.jl") -include("pipeline.jl") -include("ZGroup.jl") -include("caching.jl") - -export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, - zinfo, DirectoryStore, S3Store, GCStore, zgroup - -end # module +end \ No newline at end of file From 9808610caf22a1da29cb016c59ba32d93fba86f6 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 11:32:42 +0200 Subject: [PATCH 02/39] Fix S3Ext --- ext/ZarrAWSS3Ext.jl | 4 ++-- ext/s3store.jl | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ext/ZarrAWSS3Ext.jl b/ext/ZarrAWSS3Ext.jl index 151f9e0d..b7f5f303 100644 --- a/ext/ZarrAWSS3Ext.jl +++ b/ext/ZarrAWSS3Ext.jl @@ -1,14 +1,14 @@ module ZarrAWSS3Ext import Zarr -import Zarr: +import ZarrCore: S3Store, AbstractStore, cloud_list_objects, ConcurrentRead, storageregexlist, concurrent_io_tasks, - zopen + zopen, ZarrCore using AWSS3: AWSS3, s3_put, s3_get, s3_delete, s3_list_objects, s3_exists, S3Path, get_config diff --git a/ext/s3store.jl b/ext/s3store.jl index fb41fc61..be0e27c8 100644 --- a/ext/s3store.jl +++ b/ext/s3store.jl @@ -1,4 +1,4 @@ -function Zarr.S3Store(bucket::String; +function ZarrCore.S3Store(bucket::String; aws = nothing, ) if aws === nothing @@ -27,7 +27,7 @@ end Base.delete!(s::S3Store, d::String) = s3_delete(s.aws,s.bucket,d) -function Zarr.storagesize(s::S3Store,p) +function ZarrCore.storagesize(s::S3Store,p) prefix = (isempty(p) || endswith(p,"/")) ? p : string(p,"/") r = s3_list_objects(s.aws,s.bucket,prefix) s = 0 @@ -40,12 +40,12 @@ function Zarr.storagesize(s::S3Store,p) s end -function Zarr.isinitialized(s::S3Store, i::String) +function ZarrCore.isinitialized(s::S3Store, i::String) s3_exists(s.aws,s.bucket,i) end -function Zarr.cloud_list_objects(s::S3Store,p) +function ZarrCore.cloud_list_objects(s::S3Store,p) prefix = (isempty(p) || endswith(p,"/")) ? p : string(p,"/") s3_list_objects_delim(s.aws, s.bucket, prefix) end @@ -71,12 +71,12 @@ function s3_list_objects_delim(aws, bucket, prefix, delimiter="/") end result end -function Zarr.subdirs(s::S3Store, p) +function ZarrCore.subdirs(s::S3Store, p) s3_resp = cloud_list_objects(s, p) !haskey(s3_resp,"CommonPrefixes") && return String[] allstrings(s3_resp["CommonPrefixes"],"Prefix") end -function Zarr.subkeys(s::S3Store, p) +function ZarrCore.subkeys(s::S3Store, p) s3_resp = cloud_list_objects(s, p) !haskey(s3_resp,"Contents") && return String[] r = allstrings(s3_resp["Contents"],"Key") @@ -87,16 +87,16 @@ allstrings(v,prefixkey) = [rstrip(String(v[prefixkey]),'/')] # push!(storageregexlist,r"^s3://"=>S3Store) -function Zarr.storefromstring(::Type{<:S3Store}, s, _) +function ZarrCore.storefromstring(::Type{<:S3Store}, s, _) decomp = split(s,"/",keepempty=false) bucket = decomp[2] path = join(decomp[3:end],"/") S3Store(String(bucket),aws=AWSS3.AWS.current_aws_config()),path end -Zarr.store_read_strategy(::S3Store) = ConcurrentRead(concurrent_io_tasks[]) +ZarrCore.store_read_strategy(::S3Store) = ConcurrentRead(concurrent_io_tasks[]) -function Zarr.zopen(s::S3Path, mode="r"; kwargs...) +function ZarrCore.zopen(s::S3Path, mode="r"; kwargs...) decomp = split(string(s),"/",keepempty=false) bucket = decomp[2] path = join(decomp[3:end],"/") From eee1ad47582358c40b6320ce105f396b1dafd935 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 11:32:49 +0200 Subject: [PATCH 03/39] Decrease amount of CI --- .github/workflows/CI.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0e723bb5..92b555cc 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -19,10 +19,15 @@ jobs: - 'pre' os: - ubuntu-latest - - macOS-latest - - windows-latest arch: - 'default' + include: + - os: macOS-latest + version: '1' + arch: 'default' + - os: windows-latest + version: '1' + arch: 'default' steps: - uses: actions/checkout@v7 - uses: julia-actions/setup-julia@v3 From aaf5a248576ccf11cf0dde9220f97a0dc8f75b4e Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Mon, 17 Aug 2026 06:39:10 -0400 Subject: [PATCH 04/39] Remove HTTP 2 compat due to CI --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index c4457bfd..92d90df1 100644 --- a/Project.toml +++ b/Project.toml @@ -39,7 +39,7 @@ ChunkCodecLibZstd = "1" DataStructures = "0.17, 0.18, 0.19" DateTimes64 = "1" DiskArrays = "0.4.21" -HTTP = "^1.3, 2" +HTTP = "^1.3" JSON = "0.21, 1" OffsetArrays = "0.11, 1.0" OpenSSL = "1" From 241c2e9d5b59e8d1df13d1763f59971d3e15b0be Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 17:38:09 +0200 Subject: [PATCH 05/39] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fa90bfe..5bf54a38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - Drop HTTP.jl 2.x support - Move code to ZarrCore.jl with low dependencies +- Declare an explicit public API [#317](https://github.com/JuliaIO/Zarr.jl/pull/317). Every store, codec, filter and compressor type, and every documented extension point, is now `public`; the set of exported names is unchanged. Internals (`Metadata`, `ZarrFormat`, `is_zarray`, `is_zgroup`, `normalize_path`, `MaxLengthString`, ...) are no longer reachable as `Zarr.x` and must be accessed via `Zarr.ZarrCore.x` - Add logo and favicon to docs [#307](https://github.com/JuliaIO/Zarr.jl/pull/307) - Support reading and writing variable-length strings [#311](https://github.com/JuliaIO/Zarr.jl/pull/311) From 14f721e66c3a8da7c333a77663133267bfd19f99 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 17:38:16 +0200 Subject: [PATCH 06/39] Update claude md --- CLAUDE.md | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index dfbdb435..7278ef31 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,11 +16,18 @@ julia --project -e 'using Pkg; Pkg.test()' julia --project=test -e 'using Test, Zarr, JSON; include("test/v3_codecs.jl")' # Instantiate test dependencies (after Julia version change or first time setup) -julia --project=test -e 'using Pkg; Pkg.instantiate()' +julia --project=test -e 'using Pkg; Pkg.develop(path=pwd()); Pkg.resolve(); Pkg.instantiate()' + +# Generate the v3 test fixtures (required before running the suite; CI does this +# in a separate step, `runtests.jl` does not do it for you) +julia --project=test test/v3_julia.jl +julia --project=test test/v3_python.jl ``` Julia version requirement: 1.10+. CI tests against Julia LTS, stable (`1`), nightly, and pre-release on Ubuntu, macOS, and Windows. +Because 1.10 is still supported, the `public` keyword (Julia 1.11+) cannot be used directly. Use the `ZarrCore.@public` macro instead — it expands to `public` on 1.11+ and to nothing on 1.10. + ## Architecture ### Core Type Hierarchy @@ -43,16 +50,35 @@ ZArray{T,N,S<:AbstractStore,M<:AbstractMetadata} <: AbstractDiskArray{T,N} ZGroup{S<:AbstractStore} ``` +### Package Layout + +The repo is a Pkg workspace with two packages: + +- `ZarrCore` (`lib/ZarrCore/`) — the entire implementation. Every type, method and docstring lives here. +- `Zarr` (`src/Zarr.jl`) — a thin facade that re-exports `ZarrCore`'s API. It mirrors the export/public split: names `ZarrCore` exports are re-exported, names it only marks `@public` stay public (not exported). Nothing else is forwarded, so internals must be reached as `Zarr.ZarrCore.foo`. + +### Public API Policy + +Set by [a maintainer comment on PR #317](https://github.com/JuliaIO/Zarr.jl/pull/317#issuecomment-5314176722). Anything a downstream consumer could need, plus every documented extension point, is part of the public API; everything else is internal and may change. + +- **Exported** (in scope after `using Zarr`): `ZArray`, `ZGroup`, `zopen`, `zzeros`, `zcreate`, `zgroup`, `zarrcache`, `storagesize`, `storageratio`, `zinfo`, `DirectoryStore`, `S3Store`, `GCStore`. +- **Public but not exported**: every store, codec, filter and compressor type; the store/filter/compressor/codec/chunk-key-encoding extension interfaces; `typestr`, `fill_value_encoding`, `fill_value_decoding`, `zname`, `writezip`, `consolidate_metadata`, `DateTime64`. +- **Internal**: `Metadata`/`MetadataV2`/`MetadataV3`, `ZarrFormat`, `DV`, `is_zarray`, `is_zgroup`, `normalize_path`, `MaxLengthString`, `ASCIIChar`, `ShapeOnlyArray` (should be removed), `getattrs`/`writeattrs`/`getmetadata`/`writemetadata`, `V2Pipeline`/`V3Pipeline`/`pipeline_encode`/`pipeline_decode!`, and the `store_*` helpers. + +Tests follow the same rule: public names are used as `Zarr.foo`, internals as `ZarrCore.foo` (each test file does `import Zarr: ZarrCore`). If a test needs `ZarrCore.` for something a downstream user would plausibly need, that is a signal the name should be made public rather than the test qualified. + ### Module/File Layout -- `src/Zarr.jl` — Module entry point, defines `ZarrFormat{V}` (Val-parameterized version tag, default `DV = ZarrFormat(Val(2))`) +All paths below are relative to `lib/ZarrCore/`. + +- `src/ZarrCore.jl` — Module entry point, defines `ZarrFormat{V}` (Val-parameterized version tag, default `DV = ZarrFormat(Val(2))`), the `@public` macro, and the export/public declarations - `src/metadata.jl` — `MetadataV2` struct, type string encoding (`typestr`), fill value encoding/decoding, `Metadata()` constructors for V2; dispatches V3 to `metadata3.jl` - `src/metadata3.jl` — All V3-specific code: `MetadataV3` struct and constructors, `Metadata3(dict)` parsing, `lower3` serialization, codec pipeline parsing, `get_order`, `JSON.lower(::MetadataV3)` -- `src/chunkencoding.jl` — `ChunkEncoding` struct (separator char + prefix bool), `citostring()` for chunk path generation. V2 default: `'.'` separator, no prefix. V3 default: `'/'` separator, `"c/"` prefix +- `src/chunkkeyencoding.jl` — `ChunkKeyEncoding` struct (separator char + prefix bool), `citostring()` for chunk path generation, plus the `register_chunk_key_encoding` registry. V2 default: `'.'` separator, no prefix. V3 default: `'/'` separator, `"c/"` prefix - `src/ZArray.jl` — Core array type, `readblock!`/`writeblock!` (DiskArrays interface), `zcreate`, `zzeros`, `zopen`, resize/append - `src/ZGroup.jl` — Hierarchical group support, `zopen`, `zgroup`, auto-detection of zarr version via `ZarrFormat(store, path)` -- `src/Compressors/` — `Compressor` abstract type, `compressortypes` registry (keyed by spec name string), implementations: `blosc.jl`, `zlib.jl`, `zstd.jl`, `v3.jl` (v3 wrapper `Compressor_v3{C}`) -- `src/Codecs/` — V3 codec system (`Codec` abstract type), `V3/V3.jl` defines `V3Codec{In,Out}` with `BloscV3Codec`, `BytesCodec`, `CRC32cV3Codec`, `GzipV3Codec`, `ShardingCodec`, `TransposeCodec`, `ZstdV3Codec` +- `src/Compressors/` — `Compressor` abstract type, `compressortypes` registry (keyed by spec name string), implementations: `blosc.jl`, `zlib.jl`, `zstd.jl` +- `src/Codecs/` — V3 codec system (`Codec` abstract type), `V3/V3.jl` defines `V3Codec{In,Out}` with `BloscV3Codec`, `BytesCodec`, `CRC32cV3Codec`, `GzipV3Codec`, `ShardingCodec`, `TransposeCodec`, `ZstdV3Codec`, and the `register_codec` registry - `src/Filters/` — `Filter{T,TENC}` abstract type, implementations for variable-length arrays, strings, Fletcher32, shuffle, delta, quantize - `src/Storage/Storage.jl` — `AbstractStore` interface, I/O strategy (`SequentialRead`/`ConcurrentRead`), chunk read/write/delete helpers, metadata read/write dispatched on `ZarrFormat{2}` vs `ZarrFormat{3}` @@ -73,14 +99,14 @@ New store backends must implement: `getindex(store, key)::Union{Vector{UInt8}, N V3 support is under active development. Current state: -**Codecs (`src/Codecs/V3/V3.jl`)** +**Codecs (`lib/ZarrCore/src/Codecs/V3/V3.jl`)** - `BytesCodec` — stores `endian::Symbol` (`:little` or `:big`); encode/decode byte-swap elements when the target endian differs from the system byte order (`Base.ENDIAN_BOM`). Default is `:little`. - `TransposeCodec` — array→array permutation codec (renamed from `TransposeCodecImpl`) - `BloscV3Codec` — shuffle stored as integer (0=noshuffle, 1=shuffle, 2=bitshuffle); parsed from spec strings (`"noshuffle"`, `"shuffle"`, `"bitshuffle"`) and serialized back to strings - Sharding codec (`sharding_indexed`) has struct definitions and encode/decode logic but is not yet wired into the main read/write pipeline (throws `ArgumentError` when encountered) - `crc32c` codec has encode/decode implementations and is parseable from metadata -**Metadata (`src/metadata3.jl`)** +**Metadata (`lib/ZarrCore/src/metadata3.jl`)** - `MetadataV3{T,N,P}` has no `order` field; storage order is encoded in the pipeline via `TransposeCodec` - Two constructors: - Primary inner constructor: `MetadataV3{T,N,P}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_encoding)` — takes a pre-built pipeline, no `order` argument From 3da09d28852c4cc72cc1c861d8c3592b4a95ceec Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 17:38:33 +0200 Subject: [PATCH 07/39] Fix docs --- docs/make.jl | 5 ++++- docs/src/reference.md | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index f13ec745..15a27701 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,11 +1,14 @@ using DocumenterVitepress using Documenter, Zarr +# All docstrings live in the `ZarrCore` implementation package; `Zarr` is only +# the facade that re-exports its API. +using Zarr: ZarrCore cp(joinpath(@__DIR__, "..", "CHANGELOG.md"), joinpath(@__DIR__, "src", "changelog.md"), force = true) cp(joinpath(@__DIR__, "..", "CONTRIBUTING.md"), joinpath(@__DIR__, "src", "contributing.md"), force = true) makedocs( - modules = [Zarr], + modules = [Zarr, ZarrCore], clean = false, doctest = true, format = DocumenterVitepress.MarkdownVitepress( diff --git a/docs/src/reference.md b/docs/src/reference.md index 7d0e31c7..91efb08c 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -10,13 +10,13 @@ zzeros ## Group hierarchy ```@autodocs -Modules = [Zarr] +Modules = [ZarrCore] Pages = ["ZGroup.jl"] ``` ## Compressors ```@autodocs -Modules = [Zarr] +Modules = [ZarrCore] Pages = ["Compressors/Compressors.jl", "Compressors/blosc.jl", "Compressors/zlib.jl", "Compressors/zstd.jl"] ``` From ba87f30000e5b171439954430d9742d889887212 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 17:38:39 +0200 Subject: [PATCH 08/39] Fix zarrcore project --- lib/ZarrCore/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ZarrCore/Project.toml b/lib/ZarrCore/Project.toml index c87deb76..331298a5 100644 --- a/lib/ZarrCore/Project.toml +++ b/lib/ZarrCore/Project.toml @@ -1,4 +1,4 @@ -name = "Zarr" +name = "ZarrCore" uuid = "77f5b75c-4c08-499f-ba13-550b0a0af171" version = "0.10.1" authors = ["Fabian Gans "] From 9b157ac105f7544d756531e007cab3b2a0cad95f Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 17:41:11 +0200 Subject: [PATCH 09/39] fixup and make things public in zarrcore --- lib/ZarrCore/src/Codecs/Codecs.jl | 3 ++ lib/ZarrCore/src/Codecs/V3/V3.jl | 9 ++++ lib/ZarrCore/src/ZarrCore.jl | 79 ++++++++++++++++++++++++++++++- lib/ZarrCore/src/caching.jl | 1 - src/Zarr.jl | 14 +++++- 5 files changed, 101 insertions(+), 5 deletions(-) diff --git a/lib/ZarrCore/src/Codecs/Codecs.jl b/lib/ZarrCore/src/Codecs/Codecs.jl index ec6e6205..a7783134 100644 --- a/lib/ZarrCore/src/Codecs/Codecs.jl +++ b/lib/ZarrCore/src/Codecs/Codecs.jl @@ -46,4 +46,7 @@ getCodec(::Type{<:Codec}, d::Dict) = error("Unimplemented") include("V3/V3.jl") +import ..ZarrCore: @public +@public Codec, V3Codecs, zencode, zencode!, zdecode, zdecode!, getCodec + end diff --git a/lib/ZarrCore/src/Codecs/V3/V3.jl b/lib/ZarrCore/src/Codecs/V3/V3.jl index c3acc446..1d26379c 100644 --- a/lib/ZarrCore/src/Codecs/V3/V3.jl +++ b/lib/ZarrCore/src/Codecs/V3/V3.jl @@ -823,4 +823,13 @@ function codec_decode(::VLenUTF8V3Codec, encoded::Vector{UInt8}, ::Type{T}, shap out end +# The codec types themselves are re-exported (as public names) from `ZarrCore`; +# these are the extension points for defining and registering new v3 codecs. +import ...ZarrCore: @public +@public V3Codec, getCodec, register_codec, codec_parsers, codec_encode, + codec_decode, is_fixed_size, name +@public BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, ShardingCodec, + TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, CRC32cV3Codec, + VLenUTF8V3Codec + end diff --git a/lib/ZarrCore/src/ZarrCore.jl b/lib/ZarrCore/src/ZarrCore.jl index 9d04fe1a..467d8917 100644 --- a/lib/ZarrCore/src/ZarrCore.jl +++ b/lib/ZarrCore/src/ZarrCore.jl @@ -5,6 +5,24 @@ import Blosc import Unicode using OrderedCollections: OrderedDict +""" + @public name + @public name1, name2, ... + +Mark `name`s as public API without exporting them, i.e. `Base.ispublic` returns +`true` and they show up in `names(ZarrCore)`, but `using ZarrCore` does not +bring them into scope. + +This is the `public` keyword, which only exists from Julia 1.11 onwards; on +Julia 1.10 the declaration expands to nothing. +""" +macro public(ex) + syms = ex isa Symbol ? [ex] : ex.args + all(s -> s isa Symbol, syms) || + throw(ArgumentError("@public expects one or more symbols, got $ex")) + VERSION >= v"1.11" ? esc(Expr(:public, syms...)) : nothing +end + struct ZarrFormat{V} version::Val{V} end @@ -27,7 +45,64 @@ include("pipeline.jl") include("ZGroup.jl") include("caching.jl") -export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, - zinfo, DirectoryStore, S3Store, GCStore, zgroup +import .Codecs: Codec +import .Codecs.V3Codecs: V3Codec, BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, + ShardingCodec, TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, + CRC32cV3Codec, VLenUTF8V3Codec + +# ## Public API +# +# The rule of thumb is: anything a downstream consumer could need, and every +# documented extension point, is part of the public API. `export` is reserved +# for the handful of names that are convenient to have in scope after +# `using Zarr`; everything else is marked `@public` and must be qualified. +# +# Things that are deliberately *not* public (and may change without notice): +# `Metadata`/`MetadataV2`/`MetadataV3`, `ZarrFormat`, `is_zarray`, `is_zgroup`, +# `normalize_path`, `MaxLengthString`, `ShapeOnlyArray` and the various +# `pipeline_*`/`store_*` internals. + +export ZArray, ZGroup, zopen, zzeros, zcreate, zgroup, zarrcache, + storagesize, storageratio, zinfo, + DirectoryStore, S3Store, GCStore + +@public zname, zopen_noerr + +# Stores. Every store type is public; `DirectoryStore`, `S3Store` and `GCStore` +# are additionally exported for backwards compatibility. +@public AbstractStore, DictStore, HTTPStore, ZipStore, CachingStore, + ConsolidatedStore, PermanentZarrCache +@public consolidate_metadata, writezip + +# The interface a new store backend has to implement, see `?AbstractStore`. +# (`storagesize` is part of it too, but is exported above.) +@public subdirs, subkeys, isinitialized, storefromstring, + store_read_strategy, SequentialRead, ConcurrentRead, storageregexlist, + cloud_list_objects, concurrent_io_tasks + +# Chunk key encodings and the registry used to add new ones. +@public AbstractChunkKeyEncoding, ChunkKeyEncoding, SuffixChunkKeyEncoding, + citostring, register_chunk_key_encoding, parse_chunk_key_encoding, + lower_chunk_key_encoding + +# Filters and the interface a new filter has to implement, see `?Filter`. +@public Filter, VLenArrayFilter, VLenUTF8Filter, Fletcher32Filter, + FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter +@public zencode, zdecode, getfilter, sourcetype, desttype, filterdict + +# Compressors and the interface a new compressor has to implement. +@public Compressor, NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor +@public zcompress, zcompress!, zuncompress, zuncompress!, getCompressor, + compressortypes + +# v3 codecs and the interface a new codec has to implement, see `?Codec`. +@public Codecs, Codec, V3Codec, BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, + ShardingCodec, TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, + CRC32cV3Codec, VLenUTF8V3Codec + +# Data type and fill value encoding, needed to map Zarr dtypes to Julia types. +# `DateTime64` is re-exported from DateTimes64.jl because it shows up as the +# `eltype` of datetime arrays. +@public typestr, fill_value_encoding, fill_value_decoding, DateTime64 end # module diff --git a/lib/ZarrCore/src/caching.jl b/lib/ZarrCore/src/caching.jl index b404e623..2fda6cde 100644 --- a/lib/ZarrCore/src/caching.jl +++ b/lib/ZarrCore/src/caching.jl @@ -1,5 +1,4 @@ import DiskArrays: approx_chunksize, eachchunk, CachedDiskArray, ChunkIndex -export zarrcache struct PermanentZarrCache{T,N,A<:ZArray{T,N}} a::A diff --git a/src/Zarr.jl b/src/Zarr.jl index 55a57747..e7d5aa23 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -2,8 +2,18 @@ module Zarr import ZarrCore +# Mirror ZarrCore's export/public split. Internals stay at Zarr.ZarrCore. for name in names(ZarrCore; all = false) - @eval export $name + if name !== :ZarrCore + @eval import ZarrCore: $name + end + if Base.isexported(ZarrCore, name) && name !== :ZarrCore + @eval export $name + else + @static if VERSION >= v"1.11" + Core.eval(@__MODULE__, Expr(:public, name)) + end + end end -end \ No newline at end of file +end From eab9563f773d4e8cd4472c30eda94b44d8411476 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 17:41:35 +0200 Subject: [PATCH 10/39] rename things to ZarrCore in tests --- test/Filters.jl | 3 +- test/arraycache.jl | 6 +- test/consolidated.jl | 110 ++++++++++--------- test/python.jl | 5 +- test/runtests.jl | 113 ++++++++++--------- test/storage.jl | 104 +++++++++--------- test/v3_codecs.jl | 253 ++++++++++++++++++++++--------------------- test/v3_julia.jl | 17 +-- 8 files changed, 313 insertions(+), 298 deletions(-) diff --git a/test/Filters.jl b/test/Filters.jl index 5c09a8ca..3168ca5b 100644 --- a/test/Filters.jl +++ b/test/Filters.jl @@ -3,6 +3,7 @@ using Zarr: DateTime64 # for datetime reinterpret using Zarr: zencode, zdecode using Zarr: Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter +import Zarr: ZarrCore @testset "Fletcher32Filter" begin # These tests are copied exactly from the [`numcodecs`](https://github.com/zarr-developers/numcodecs/) Python package, @@ -72,7 +73,7 @@ end LinRange(1000, 1001, 1000), # equivalent to np.linspace(1000, 1001, 1000, dtype='f8') reshape(randn(1000) .* 1 .+ 1000, (100, 10)), # equivalent to np.random.normal(loc=1000, scale=1, size=(100, 10)) reshape(rand(Bool, 1000), (10, 100)), # equivalent to np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F') - reshape(rand(Zarr.MaxLengthString{3, UInt8}["a", "bb", "ccc"], 1000), (10, 10, 10)), # equivalent to np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10) + reshape(rand(ZarrCore.MaxLengthString{3, UInt8}["a", "bb", "ccc"], 1000), (10, 10, 10)), # equivalent to np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10) reinterpret(DateTime64{Dates.Nanosecond}, rand(UInt64(0):UInt64(2^60)-1, 1000)), # equivalent to np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]') Nanosecond.(rand(UInt64(0):UInt64(2^60-1), 1000)), # equivalent to np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]') reinterpret(DateTime64{Dates.Minute}, rand(UInt64(0):UInt64(2^25-1), 1000)), # equivalent to np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]') diff --git a/test/arraycache.jl b/test/arraycache.jl index 2dcf9ce5..c5b94693 100644 --- a/test/arraycache.jl +++ b/test/arraycache.jl @@ -5,7 +5,7 @@ a .= reshape(1:200, 10, 20) # Start HTTP server - using Zarr.HTTP, Sockets + using Zarr.ZarrCore.HTTP, Sockets server = Sockets.listen(0) ip, port = getsockname(server) @async HTTP.serve(g, ip, port,server=server) @@ -17,7 +17,7 @@ g2 = zarrcache(base_array, cache_dir) - @test g2["a1"].cache.a.storage.folder == Zarr.normalize_path(joinpath(cache_dir, "a1")) + @test g2["a1"].cache.a.storage.folder == ZarrCore.normalize_path(joinpath(cache_dir, "a1")) # We also open the cache array on disk directly g_disk = zopen(cache_dir) @test g_disk.attrs == Dict("groupatt"=>5) @@ -32,7 +32,7 @@ # Now test if we can open the cache store from an existing path g3 = zarrcache(base_array, cache_dir) - @test g3["a1"].cache.a.storage.folder == Zarr.normalize_path(joinpath(cache_dir, "a1")) + @test g3["a1"].cache.a.storage.folder == ZarrCore.normalize_path(joinpath(cache_dir, "a1")) @test g3["a1"][1:5,6:10] == a[1:5,6:10] @test g_disk["a1"][1:5,6:10] == a[1:5,6:10] # Stop server diff --git a/test/consolidated.jl b/test/consolidated.jl index 553e3b19..9e0e607e 100644 --- a/test/consolidated.jl +++ b/test/consolidated.jl @@ -1,6 +1,8 @@ using CondaPkg: CondaPkg, PkgSpec using JSON using PythonCall +using Zarr +import Zarr: ZarrCore CondaPkg.add([ PkgSpec("numpy"), @@ -68,20 +70,20 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") @testset "ConsolidatedStore v3 constructor error paths" begin s = Zarr.DictStore() # missing zarr.json - @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", ZarrCore.ZarrFormat(3)) # zarr.json present but no consolidated_metadata s["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"node_type":"group"}""") - @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", ZarrCore.ZarrFormat(3)) # consolidated_metadata present but no metadata subkey s["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"consolidated_metadata":{"kind":"inline"}}""") - @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", ZarrCore.ZarrFormat(3)) end @testset "getmetadata v3 on ConsolidatedStore" begin path_jl = joinpath(path_v3_julia, "consolidated") cs = zopen(path_jl, consolidated=true) # getmetadata v3 reads from cons["metadata"][key] - meta = Zarr.getmetadata(Zarr.ZarrFormat(3), cs.storage, "1d.chunked.i2", false) + meta = ZarrCore.getmetadata(ZarrCore.ZarrFormat(3), cs.storage, "1d.chunked.i2", false) @test eltype(meta) == Int16 @test meta.chunks == (2,) end @@ -90,18 +92,18 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") path_jl = joinpath(path_v3_julia, "consolidated") cs = zopen(path_jl, consolidated=true) s = cs.storage # the ConsolidatedStore - V3 = Zarr.ZarrFormat(3) - @test Zarr.is_zarray(V3, s, "1d.chunked.i2") - @test !Zarr.is_zarray(V3, s, "nested") - @test !Zarr.is_zarray(V3, s, "nonexistent") - @test Zarr.is_zgroup(V3, s, "nested") - @test !Zarr.is_zgroup(V3, s, "1d.chunked.i2") - @test !Zarr.is_zgroup(V3, s, "nonexistent") + V3 = ZarrCore.ZarrFormat(3) + @test ZarrCore.is_zarray(V3, s, "1d.chunked.i2") + @test !ZarrCore.is_zarray(V3, s, "nested") + @test !ZarrCore.is_zarray(V3, s, "nonexistent") + @test ZarrCore.is_zgroup(V3, s, "nested") + @test !ZarrCore.is_zgroup(V3, s, "1d.chunked.i2") + @test !ZarrCore.is_zgroup(V3, s, "nonexistent") end @testset "subdirs v3 on ConsolidatedStore" begin ds = Zarr.DirectoryStore(joinpath(path_v3_julia, "consolidated")) - cs = Zarr.ConsolidatedStore(ds, "", Zarr.ZarrFormat(3)) + cs = Zarr.ConsolidatedStore(ds, "", ZarrCore.ZarrFormat(3)) # v3 subdirs returns only immediate children (length(sp) == lp + 1) dirs = Zarr.subdirs(cs, "") @test sort(dirs) == ["1d.chunked.i2", "2d.contiguous.i2", "nested"] @@ -149,28 +151,28 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") # Missing .zmetadata s = Zarr.DictStore() zgroup(s) - @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", Zarr.ZarrFormat(2)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", ZarrCore.ZarrFormat(2)) # .zmetadata present but missing "metadata" field s2 = Zarr.DictStore() s2[".zmetadata"] = Vector{UInt8}("""{"zarr_consolidated_format":1}""") - @test_throws ArgumentError Zarr.ConsolidatedStore(s2, "", Zarr.ZarrFormat(2)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s2, "", ZarrCore.ZarrFormat(2)) end @testset "v3 ConsolidatedStore constructor errors" begin # Missing zarr.json s = Zarr.DictStore() - @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", ZarrCore.ZarrFormat(3)) # zarr.json present but no consolidated_metadata s2 = Zarr.DictStore() s2["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"node_type":"group"}""") - @test_throws ArgumentError Zarr.ConsolidatedStore(s2, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s2, "", ZarrCore.ZarrFormat(3)) # consolidated_metadata present but no metadata subkey s3 = Zarr.DictStore() s3["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"consolidated_metadata":{"kind":"inline"}}""") - @test_throws ArgumentError Zarr.ConsolidatedStore(s3, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s3, "", ZarrCore.ZarrFormat(3)) end @testset "auto-detect constructor errors when no format found" begin @@ -184,12 +186,12 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") zcreate(Int, g, "arr", 4, chunks=(2,), attrs=Dict("b" => 2)) zgroup(g, "sub") # group with no attrs cs = Zarr.consolidate_metadata(s, "") - V2 = Zarr.ZarrFormat(2) - @test Zarr.getattrs(V2, cs, "") == Dict("a" => 1) - @test Zarr.getattrs(V2, cs, "arr") == Dict("b" => 2) + V2 = ZarrCore.ZarrFormat(2) + @test ZarrCore.getattrs(V2, cs, "") == Dict("a" => 1) + @test ZarrCore.getattrs(V2, cs, "arr") == Dict("b" => 2) # No .zattrs written for sub → returns empty dict - @test Zarr.getattrs(V2, cs, "sub") == Dict{String,Any}() - @test Zarr.getattrs(V2, cs, "nonexistent") == Dict{String,Any}() + @test ZarrCore.getattrs(V2, cs, "sub") == Dict{String,Any}() + @test ZarrCore.getattrs(V2, cs, "nonexistent") == Dict{String,Any}() end @testset "v2 is_zarray / is_zgroup" begin @@ -198,15 +200,15 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") zgroup(g, "sub") zcreate(Float64, g, "data", 8, chunks=(4,)) cs = Zarr.consolidate_metadata(s, "") - V2 = Zarr.ZarrFormat(2) - @test Zarr.is_zgroup(V2, cs, "") - @test !Zarr.is_zarray(V2, cs, "") - @test Zarr.is_zgroup(V2, cs, "sub") - @test !Zarr.is_zarray(V2, cs, "sub") - @test Zarr.is_zarray(V2, cs, "data") - @test !Zarr.is_zgroup(V2, cs, "data") - @test !Zarr.is_zarray(V2, cs, "nonexistent") - @test !Zarr.is_zgroup(V2, cs, "nonexistent") + V2 = ZarrCore.ZarrFormat(2) + @test ZarrCore.is_zgroup(V2, cs, "") + @test !ZarrCore.is_zarray(V2, cs, "") + @test ZarrCore.is_zgroup(V2, cs, "sub") + @test !ZarrCore.is_zarray(V2, cs, "sub") + @test ZarrCore.is_zarray(V2, cs, "data") + @test !ZarrCore.is_zgroup(V2, cs, "data") + @test !ZarrCore.is_zarray(V2, cs, "nonexistent") + @test !ZarrCore.is_zgroup(V2, cs, "nonexistent") end @testset "v2 subdirs" begin @@ -227,8 +229,8 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") a = zcreate(Int32, g, "arr", 10, 20, chunks=(5,5), fill_value=Int32(-1)) # I think this `fill_value` way of passing things is related to issue: https://github.com/JuliaIO/Zarr.jl/issues/292 cs = Zarr.consolidate_metadata(s, "") - V2 = Zarr.ZarrFormat(2) - meta = Zarr.getmetadata(V2, cs, "arr", false) + V2 = ZarrCore.ZarrFormat(2) + meta = ZarrCore.getmetadata(V2, cs, "arr", false) @test meta.dtype == Int32 || eltype(meta) == Int32 @test meta.chunks == (5, 5) end @@ -242,16 +244,16 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") ) ) cs = Zarr.ConsolidatedStore(Zarr.DictStore(), "", cons) - @test Zarr.getattrs(Zarr.ZarrFormat(3), cs, "") == Dict("foo" => "bar") + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), cs, "") == Dict("foo" => "bar") # zarr.json present but no "attributes" cs2 = Zarr.ConsolidatedStore(Zarr.DictStore(), "", Dict{String,Any}("zarr.json" => Dict{String,Any}("node_type" => "group"))) - @test Zarr.getattrs(Zarr.ZarrFormat(3), cs2, "") == Dict{String,Any}() + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), cs2, "") == Dict{String,Any}() # zarr.json key absent entirely cs3 = Zarr.ConsolidatedStore(Zarr.DictStore(), "", Dict{String,Any}()) - @test Zarr.getattrs(Zarr.ZarrFormat(3), cs3, "") == Dict{String,Any}() + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), cs3, "") == Dict{String,Any}() end @testset "v3 is_zarray / is_zgroup on ConsolidatedStore (unit)" begin @@ -270,15 +272,15 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") s = Zarr.DictStore() s["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"node_type":"group"}""") cs = Zarr.ConsolidatedStore(s, "", cons) - V3 = Zarr.ZarrFormat(3) + V3 = ZarrCore.ZarrFormat(3) # is_zarray: key = "group1/arr" (no suffix) - @test Zarr.is_zarray(V3, cs, "group1/arr") - @test !Zarr.is_zarray(V3, cs, "group1") # no bare "group1" key - @test !Zarr.is_zarray(V3, cs, "nonexistent") + @test ZarrCore.is_zarray(V3, cs, "group1/arr") + @test !ZarrCore.is_zarray(V3, cs, "group1") # no bare "group1" key + @test !ZarrCore.is_zarray(V3, cs, "nonexistent") # is_zgroup: key = "group1/zarr.json" (_unconcpath + "zarr.json") - @test Zarr.is_zgroup(V3, cs, "group1") - @test !Zarr.is_zgroup(V3, cs, "group1/arr") # "group1/arr/zarr.json" not in metadata - @test !Zarr.is_zgroup(V3, cs, "nonexistent") + @test ZarrCore.is_zgroup(V3, cs, "group1") + @test !ZarrCore.is_zgroup(V3, cs, "group1/arr") # "group1/arr/zarr.json" not in metadata + @test !ZarrCore.is_zgroup(V3, cs, "nonexistent") end @testset "is_zgroup v3 fallback to parent" begin @@ -290,18 +292,18 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") # write a real group zarr.json to the parent so the fallback succeeds s["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"node_type":"group"}""") cs = Zarr.ConsolidatedStore(s, "", cons) - V3 = Zarr.ZarrFormat(3) + V3 = ZarrCore.ZarrFormat(3) # not in cons["metadata"], falls back to parent — parent has zarr.json with node_type=group - @test Zarr.is_zgroup(V3, cs, "") + @test ZarrCore.is_zgroup(V3, cs, "") # not in cons["metadata"], fallback finds nothing → false - @test !Zarr.is_zgroup(V3, cs, "nonexistent") + @test !ZarrCore.is_zgroup(V3, cs, "nonexistent") end @testset "consolidate_metadata v3 adds consolidated_metadata when missing" begin tmp = mktempdir() try ds = Zarr.DirectoryStore(tmp) - g = zgroup(ds, "", Zarr.ZarrFormat(3)) + g = zgroup(ds, "", ZarrCore.ZarrFormat(3)) zcreate(Int16, g, "arr", 4, chunks=(2,), compressor=Zarr.NoCompressor()) zj_path = joinpath(tmp, "zarr.json") @@ -313,7 +315,7 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") JSON.print(io, root, 4) end # act - cs = Zarr.consolidate_metadata(ds, "", Zarr.ZarrFormat(3)) + cs = Zarr.consolidate_metadata(ds, "", ZarrCore.ZarrFormat(3)) # type check @test cs isa Zarr.ConsolidatedStore # reload file @@ -374,17 +376,17 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") s = Zarr.DictStore() zgroup(s) cs = Zarr.consolidate_metadata(s, "") - @test Zarr.ZarrFormat(cs, "") == Zarr.ZarrFormat(s, "") + @test ZarrCore.ZarrFormat(cs, "") == ZarrCore.ZarrFormat(s, "") end @testset "_unconcpath" begin s = Zarr.DictStore() cs = Zarr.ConsolidatedStore(s, "a/b", Dict{String,Any}()) - @test Zarr._unconcpath(cs, "a/b/c/d") == "c/d" - @test Zarr._unconcpath(cs, "a/b") == "" - @test_throws ErrorException Zarr._unconcpath(cs, "x/y") + @test ZarrCore._unconcpath(cs, "a/b/c/d") == "c/d" + @test ZarrCore._unconcpath(cs, "a/b") == "" + @test_throws ErrorException ZarrCore._unconcpath(cs, "x/y") # with suffix - @test Zarr._unconcpath(cs, "a/b/c", ".zarray") == "c/.zarray" + @test ZarrCore._unconcpath(cs, "a/b/c", ".zarray") == "c/.zarray" end @testset "store_read_strategy and has_configurable_missing_chunks delegate" begin @@ -392,7 +394,7 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") zgroup(s) cs = Zarr.consolidate_metadata(s, "") @test Zarr.store_read_strategy(cs) == Zarr.store_read_strategy(s) - @test Zarr.has_configurable_missing_chunks(cs) == Zarr.has_configurable_missing_chunks(s) + @test ZarrCore.has_configurable_missing_chunks(cs) == ZarrCore.has_configurable_missing_chunks(s) end @testset "v2 full data round-trip through ConsolidatedStore" begin diff --git a/test/python.jl b/test/python.jl index 7509ef41..d50b41de 100644 --- a/test/python.jl +++ b/test/python.jl @@ -29,8 +29,9 @@ groupattrs = Dict("String attribute"=>"One", "Int attribute"=>5, "Float attribut g = zgroup(pjulia,attrs=groupattrs) # Test all supported data types and compressors -import Zarr: NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor, MaxLengthString, +import Zarr: NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor, Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter +import Zarr: ZarrCore using Random: randstring numeric_dtypes = (UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, @@ -38,7 +39,7 @@ numeric_dtypes = (UInt8, UInt16, UInt32, UInt64, Complex{Float32}, Complex{Float64}, Bool,) dtypes = (numeric_dtypes..., - MaxLengthString{10,UInt8},MaxLengthString{10,UInt32}, + ZarrCore.MaxLengthString{10,UInt8},ZarrCore.MaxLengthString{10,UInt32}, String) dtypesp = ("uint8","uint16","uint32","uint64", "int8","int16","int32","int64", diff --git a/test/runtests.jl b/test/runtests.jl index 65ecbc62..184e6248 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using Test using Zarr +import Zarr: ZarrCore using JSON using JSON: json using Pkg @@ -8,11 +9,39 @@ using Dates @testset "Zarr" begin - +@testset "public API surface" begin + # `names(M)` returns exported *and* public names, so `Zarr`'s re-export loop + # has to split them with `Base.isexported`; using `names` alone would export + # the whole extension API, and using `Base.ispublic` would export nothing. + # These tests pin that split down so it cannot regress silently. + # + # On Julia 1.10 there is no `public`, so `names` yields only exports and the + # public-only sets are empty; the assertions still hold. + # A module always lists its own name, and `Zarr` additionally carries the + # `ZarrCore` binding (public, so `Zarr.ZarrCore` is a documented escape hatch + # rather than something `using Zarr` drags in). Both are structural, not API. + modnames = Set((:Zarr, :ZarrCore)) + exported(m) = setdiff(Set(filter(n -> Base.isexported(m, n), names(m))), modnames) + publiconly(m) = setdiff(Set(filter(n -> !Base.isexported(m, n), names(m))), modnames) + + # `Zarr.ZarrCore` is reachable, but `using Zarr` must not bring it into scope. + @test isdefined(Zarr, :ZarrCore) + @test !Base.isexported(Zarr, :ZarrCore) + + # `Zarr` mirrors `ZarrCore`'s API surface exactly, split intact. + @test exported(Zarr) == exported(ZarrCore) + @test publiconly(Zarr) == publiconly(ZarrCore) + + # The specific failure mode: a name that is only `public` in `ZarrCore` must + # not become an export of `Zarr`, and vice versa. + @test isempty(intersect(publiconly(ZarrCore), exported(Zarr))) + @test isempty(intersect(exported(ZarrCore), publiconly(Zarr))) +end + @testset "ZArray" begin @testset "fields" begin z = zzeros(Int64, 2, 3) - @test z isa ZArray{Int64,2,Zarr.DictStore,Zarr.MetadataV2{Int64,2,Zarr.BloscCompressor,Nothing}} + @test z isa ZArray{Int64,2,ZarrCore.DictStore,ZarrCore.MetadataV2{Int64,2,ZarrCore.BloscCompressor,Nothing}} @test :a ∈ propertynames(z.storage) @test length(z.storage.a) === 3 @test length(z.storage.a["0.0"]) === 64 @@ -30,15 +59,15 @@ using Dates @test z.metadata.compressor.shuffle === 1 @test z.attrs == Dict{Any, Any}() @test z.writeable === true - @test z.metadata.chunk_key_encoding === Zarr.ChunkKeyEncoding(Zarr.default_sep(Zarr.DV), Zarr.default_prefix(Zarr.DV)) + @test z.metadata.chunk_key_encoding === Zarr.ChunkKeyEncoding(ZarrCore.default_sep(ZarrCore.DV), ZarrCore.default_prefix(ZarrCore.DV)) @test_throws ArgumentError zzeros(Int64,2,3, chunks = (0,1)) @test_throws ArgumentError zzeros(Int64,0,-1) - @test_throws ArgumentError Zarr.Metadata(zeros(2,2), (2,2), order = 'F') + @test_throws ArgumentError ZarrCore.Metadata(zeros(2,2), (2,2), order = 'F') end @testset "methods" begin z = zzeros(Int64, 2, 3) - @test z isa ZArray{Int64,2,Zarr.DictStore,Zarr.MetadataV2{Int64,2,Zarr.BloscCompressor,Nothing}} + @test z isa ZArray{Int64,2,Zarr.DictStore,ZarrCore.MetadataV2{Int64,2,ZarrCore.BloscCompressor,Nothing}} @test eltype(z) === Int64 @test ndims(z) === 2 @test size(z) === (2, 3) @@ -96,7 +125,7 @@ using Dates z3 = zcreate(Float32, 4, 4, 2; path=joinpath(dir, "disp"), chunks=(4, 4, 2), compressor=Zarr.NoCompressor()) ain = rand(Float32, 4, 4, 2) - m = which(Zarr.write_singlechunk_fastpath!, (typeof(z3), typeof(ain), CartesianIndex{3})) + m = which(ZarrCore.write_singlechunk_fastpath!, (typeof(z3), typeof(ain), CartesianIndex{3})) @test occursin("MetadataV2", string(m.sig)) @test occursin("NoCompressor", string(m.sig)) GC.gc() @@ -176,8 +205,8 @@ end store = DirectoryStore(tempname()) g = zgroup(store,"mygroup") g2 = zgroup(g,"asubgroup",attrs = Dict("a1"=>5)) - @test Zarr.is_zgroup(Zarr.DV, store, "mygroup") - @test Zarr.is_zgroup(Zarr.DV, store, "mygroup/asubgroup") + @test ZarrCore.is_zgroup(ZarrCore.DV, store, "mygroup") + @test ZarrCore.is_zgroup(ZarrCore.DV, store, "mygroup/asubgroup") @test g2.attrs["a1"]==5 @test isdir(joinpath(store.folder,"mygroup")) @test isdir(joinpath(store.folder,"mygroup","asubgroup")) @@ -185,38 +214,38 @@ end @testset "Groups format inheritance v2" begin store = DirectoryStore(tempname()) - zv = Zarr.ZarrFormat(2) + zv = ZarrCore.ZarrFormat(2) g = zgroup(store, "rootgroup", zv) sg = zgroup(g, "subgroup", attrs=Dict("a1" => 5)) - @test Zarr.is_zgroup(zv, store, "rootgroup") - @test Zarr.is_zgroup(zv, store, "rootgroup/subgroup") + @test ZarrCore.is_zgroup(zv, store, "rootgroup") + @test ZarrCore.is_zgroup(zv, store, "rootgroup/subgroup") @test sg.attrs["a1"] == 5 @test ispath(joinpath(store.folder, "rootgroup", ".zgroup")) @test ispath(joinpath(store.folder, "rootgroup", "subgroup", ".zgroup")) a_root = zcreate(Float64, g, "temperature", 2, 3) a_sub = zcreate(Float64, sg, "pressure", 2, 3) - @test Zarr.zarr_format(a_root) == zv - @test Zarr.zarr_format(a_sub) == zv + @test ZarrCore.zarr_format(a_root) == zv + @test ZarrCore.zarr_format(a_sub) == zv end @testset "Groups format inheritance v3" begin store = DirectoryStore(tempname()) - zv = Zarr.ZarrFormat(3) + zv = ZarrCore.ZarrFormat(3) g = zgroup(store, "rootgroup", zv) sg = zgroup(g, "subgroup", attrs=Dict("a1" => 5)) - @test Zarr.is_zgroup(zv, store, "rootgroup") - @test Zarr.is_zgroup(zv, store, "rootgroup/subgroup") + @test ZarrCore.is_zgroup(zv, store, "rootgroup") + @test ZarrCore.is_zgroup(zv, store, "rootgroup/subgroup") @test sg.attrs["a1"] == 5 @test ispath(joinpath(store.folder, "rootgroup", "zarr.json")) @test ispath(joinpath(store.folder, "rootgroup", "subgroup", "zarr.json")) a_root = zcreate(Float64, g, "temperature", 2, 3) a_sub = zcreate(Float64, sg, "pressure", 2, 3) - @test Zarr.zarr_format(a_root) == zv - @test Zarr.zarr_format(a_sub) == zv + @test ZarrCore.zarr_format(a_root) == zv + @test ZarrCore.zarr_format(a_sub) == zv end @testset "Metadata" begin @@ -231,9 +260,9 @@ end @test Zarr.typestr(Complex{Float64}) === " "b")) - @test Zarr.getattrs(V, ds, "bar") == Dict("a" => "b") + ZarrCore.writeattrs(V, ds, "bar", Dict("a" => "b")) + @test ZarrCore.getattrs(V, ds, "bar") == Dict("a" => "b") delete!(ds,"bar/" * first_ci_str) - @test !Zarr.store_isinitialized(ds, "bar", CartesianIndex((1, 1, 1)), enc) + @test !ZarrCore.store_isinitialized(ds, "bar", CartesianIndex((1, 1, 1)), enc) @test !Zarr.isinitialized(ds,"bar/" * first_ci_str) ds["bar/" * first_ci_str] = data - @test !Zarr.store_isinitialized(ds, "bar", CartesianIndex(0, 0, 0), enc) - @test Zarr.store_isinitialized(ds, "bar", CartesianIndex(1, 1, 1), enc) + @test !ZarrCore.store_isinitialized(ds, "bar", CartesianIndex(0, 0, 0), enc) + @test ZarrCore.store_isinitialized(ds, "bar", CartesianIndex(1, 1, 1), enc) #Add tests for empty storage - @test Zarr.isemptysub(ds,"ba") - @test Zarr.isemptysub(ds,"ba/") - @test !Zarr.isemptysub(ds,"bar") - @test !Zarr.isemptysub(ds,"bar/") + @test ZarrCore.isemptysub(ds,"ba") + @test ZarrCore.isemptysub(ds,"ba/") + @test !ZarrCore.isemptysub(ds,"bar") + @test !ZarrCore.isemptysub(ds,"bar/") end """ @@ -96,11 +96,11 @@ Function to test the interface of a read only AbstractStore. Every complete impl `closer` is a function that gets called to close the read only store. """ function test_read_only_store_common(converter, closer=Returns(nothing)) - V = Zarr.DV - enc = Zarr.ChunkKeyEncoding(Zarr.default_sep(V), Zarr.default_prefix(V)) + V = ZarrCore.DV + enc = Zarr.ChunkKeyEncoding(ZarrCore.default_sep(V), ZarrCore.default_prefix(V)) ds = Zarr.DictStore() rs = converter(ds) - @test !Zarr.is_zgroup(V, rs, "") + @test !ZarrCore.is_zgroup(V, rs, "") closer(rs) ds[".zgroup"]=rand(UInt8,50) @@ -108,20 +108,20 @@ function test_read_only_store_common(converter, closer=Returns(nothing)) @test haskey(rs,".zgroup") - @test Zarr.is_zgroup(V, rs, "") - @test !Zarr.is_zarray(V, rs, "") + @test ZarrCore.is_zgroup(V, rs, "") + @test !ZarrCore.is_zarray(V, rs, "") @test isempty(Zarr.subdirs(rs,"")) @test sort(collect(Zarr.subkeys(rs,"")))==[".zgroup"] #Create a subgroup - @test !Zarr.is_zarray(V, rs, "bar") + @test !ZarrCore.is_zarray(V, rs, "bar") closer(rs) ds["bar/.zarray"] = rand(UInt8,50) rs = converter(ds) - @test Zarr.is_zarray(V, rs, "bar") + @test ZarrCore.is_zarray(V, rs, "bar") @test Zarr.subdirs(rs,"") == ["bar"] @test Zarr.subdirs(rs,"bar") == String[] #Test getindex and setindex @@ -137,16 +137,16 @@ function test_read_only_store_common(converter, closer=Returns(nothing)) @test !Zarr.isinitialized(rs,"bar/0.0.1") closer(rs) - Zarr.writeattrs(V, ds, "bar", Dict("a" => "b")) + ZarrCore.writeattrs(V, ds, "bar", Dict("a" => "b")) rs = converter(ds) - @test Zarr.getattrs(V, rs, "bar") == Dict("a" => "b") + @test ZarrCore.getattrs(V, rs, "bar") == Dict("a" => "b") closer(rs) delete!(ds,"bar/0.0.0") rs = converter(ds) - @test !Zarr.store_isinitialized(rs, "bar", CartesianIndex((0, 0, 0)), enc) + @test !ZarrCore.store_isinitialized(rs, "bar", CartesianIndex((0, 0, 0)), enc) @test !Zarr.isinitialized(rs,"bar/0.0.0") closer(rs) @@ -154,17 +154,17 @@ function test_read_only_store_common(converter, closer=Returns(nothing)) rs = converter(ds) #Add tests for empty storage - @test Zarr.isemptysub(rs,"ba") - @test Zarr.isemptysub(rs,"ba/") - @test !Zarr.isemptysub(rs,"bar") - @test !Zarr.isemptysub(rs,"bar/") + @test ZarrCore.isemptysub(rs,"ba") + @test ZarrCore.isemptysub(rs,"ba/") + @test !ZarrCore.isemptysub(rs,"bar") + @test !ZarrCore.isemptysub(rs,"bar/") closer(rs) end @testset "DirectoryStore" begin A = fill(1.0, 30, 20) chunks = (5,10) - metadata = Zarr.Metadata(A, chunks; fill_value=-1.5) + metadata = ZarrCore.Metadata(A, chunks; fill_value=-1.5) p = tempname() mkpath(joinpath(p,"foo")) ds = Zarr.DirectoryStore(joinpath(p,"foo")) @@ -182,7 +182,7 @@ end @testset "DictStore" begin A = fill(1.0, 30, 20) chunks = (5,10) - metadata = Zarr.Metadata(A, chunks; fill_value=-1.5) + metadata = ZarrCore.Metadata(A, chunks; fill_value=-1.5) ds = Zarr.DictStore() test_store_common(ds) @test haskey(ds.a,".zgroup") @@ -195,7 +195,7 @@ end @info "Testing Minio S3 storage" A = fill(1.0, 30, 20) chunks = (5,10) - metadata = Zarr.Metadata(A, chunks; fill_value=-1.5) + metadata = ZarrCore.Metadata(A, chunks; fill_value=-1.5) using Minio if !isnothing(Minio.minio()) s = Minio.Server(joinpath("./",tempname()), address="localhost:9001") @@ -230,12 +230,12 @@ end end @testset "AWS S3 Storage" begin - V = Zarr.DV + V = ZarrCore.DV @info "Testing AWS S3 storage" S3, p = AWSS3.AWS.with_aws_config(AWSS3.AWS.AWSConfig(creds=nothing, region="us-west-2")) do Zarr.storefromstring("s3://mur-sst/zarr-v1") end - @test Zarr.is_zgroup(V, S3, p) + @test ZarrCore.is_zgroup(V, S3, p) @test storagesize(S3, p) == 10551 S3group = zopen(S3,path=p) S3Array = S3group["time"] @@ -280,7 +280,7 @@ end g = zgroup(s, attrs = Dict("groupatt"=>5)) a = zcreate(Int,g,"a1",10,20,chunks=(5,5),attrs=Dict("arratt"=>2.5)) a .= reshape(1:200,10,20) - using Zarr.HTTP, Sockets + using Zarr.ZarrCore.HTTP, Sockets server = Sockets.listen(0) ip,port = getsockname(server) @async HTTP.serve(g,ip,port,server=server) @@ -304,10 +304,10 @@ end @testset "missing_chunk_return_code! on HTTPStore" begin hs = Zarr.HTTPStore("http://example.com") @test 403 ∉ hs.allowed_codes - Zarr.missing_chunk_return_code!(hs, 403) + ZarrCore.missing_chunk_return_code!(hs, 403) @test 403 ∈ hs.allowed_codes # Vector form - Zarr.missing_chunk_return_code!(hs, [410, 451]) + ZarrCore.missing_chunk_return_code!(hs, [410, 451]) @test 410 ∈ hs.allowed_codes @test 451 ∈ hs.allowed_codes end @@ -316,18 +316,18 @@ end hs = Zarr.HTTPStore("http://example.com") # Build a ConsolidatedStore wrapping the HTTPStore directly cs = Zarr.ConsolidatedStore(hs, "", Dict{String,Any}()) - Zarr.missing_chunk_return_code!(cs, 403) + ZarrCore.missing_chunk_return_code!(cs, 403) @test 403 ∈ hs.allowed_codes end @testset "store_read_strategy and has_configurable_missing_chunks" begin hs = Zarr.HTTPStore("http://example.com") @test Zarr.store_read_strategy(hs) isa Zarr.ConcurrentRead - @test Zarr.has_configurable_missing_chunks(hs) == true + @test ZarrCore.has_configurable_missing_chunks(hs) == true # ConsolidatedStore delegates both to parent cs = Zarr.ConsolidatedStore(hs, "", Dict{String,Any}()) @test Zarr.store_read_strategy(cs) isa Zarr.ConcurrentRead - @test Zarr.has_configurable_missing_chunks(cs) == true + @test ZarrCore.has_configurable_missing_chunks(cs) == true end @testset "storefromstring HTTP/HTTPS regex" begin @@ -367,7 +367,7 @@ end server4 = Sockets.listen(0) ip4, port4 = getsockname(server4) # zarr_req_handler with default notfound=404 - @async HTTP.serve(Zarr.zarr_req_handler(s3, g3.path), ip4, port4, server=server4) + @async HTTP.serve(ZarrCore.zarr_req_handler(s3, g3.path), ip4, port4, server=server4) sleep(0.1) g4 = zopen("http://$ip4:$port4") @test g4.attrs == Dict("x" => 1) @@ -396,11 +396,11 @@ end s6 = Zarr.DictStore() g6 = zgroup(s6, attrs = Dict("groupatt"=>5)) a6 = zcreate(Int, g6, "a", 10, 20, chunks=(5,5), attrs=Dict("arratt"=>2.5), fill_value=-1) - @async HTTP.serve(Zarr.zarr_req_handler(s6, g6.path, 403), ip6, port6, server=server6) + @async HTTP.serve(ZarrCore.zarr_req_handler(s6, g6.path, 403), ip6, port6, server=server6) sleep(0.1) httpstore6 = Zarr.HTTPStore("http://$ip6:$port6") @test_throws "Received error code 403" Zarr.ConsolidatedStore(httpstore6, "") - Zarr.missing_chunk_return_code!(httpstore6, 403) + ZarrCore.missing_chunk_return_code!(httpstore6, 403) g7 = zopen(Zarr.ConsolidatedStore(httpstore6, "")) @test all(==(-1), g7["a"][:,:]) close(server6) @@ -441,18 +441,18 @@ end "attributes" => Dict{String,Any}("foo" => "bar") ) )) - @test Zarr.getattrs(Zarr.ZarrFormat(3), store, "") == Dict("foo" => "bar") + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), store, "") == Dict("foo" => "bar") # zarr.json present but no "attributes" key: fallback return Dict{String,Any}() node_meta = Dict{String,Any}("node_type" => "group", "zarr_format" => 3) store_noattrs = Zarr.ConsolidatedStore(Zarr.DictStore(), "", Dict{String,Any}( "zarr.json" => node_meta )) - @test Zarr.getattrs(Zarr.ZarrFormat(3), store_noattrs, "") == Dict{String,Any}() + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), store_noattrs, "") == Dict{String,Any}() # missing zarr.json key entirely: empty dict store_empty = Zarr.ConsolidatedStore(Zarr.DictStore(), "", Dict{String,Any}()) - @test Zarr.getattrs(Zarr.ZarrFormat(3), store_empty, "") == Dict{String,Any}() + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), store_empty, "") == Dict{String,Any}() end @testset "Caching Storage" begin # Create source data @@ -462,7 +462,7 @@ end a .= reshape(1:200, 10, 20) # Start HTTP server - using Zarr.HTTP, Sockets + using Zarr.ZarrCore.HTTP, Sockets server = Sockets.listen(0) ip, port = getsockname(server) @async HTTP.serve(g, ip, port, server=server) diff --git a/test/v3_codecs.jl b/test/v3_codecs.jl index 4e79e561..1d7f28d4 100644 --- a/test/v3_codecs.jl +++ b/test/v3_codecs.jl @@ -1,5 +1,6 @@ using Test using Zarr +import Zarr: ZarrCore using JSON @testset "V3 Codecs" begin @@ -69,38 +70,38 @@ end bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() # No array->array codecs → 'C' - p = Zarr.V3Pipeline((), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test Zarr.get_order(md) == 'C' + p = ZarrCore.V3Pipeline((), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + @test ZarrCore.get_order(md) == 'C' # Single TransposeCodec with identity permutation → 'C' tc_c = Zarr.Codecs.V3Codecs.TransposeCodec((1,2,3)) - p = Zarr.V3Pipeline((tc_c,), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test Zarr.get_order(md) == 'C' + p = ZarrCore.V3Pipeline((tc_c,), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + @test ZarrCore.get_order(md) == 'C' # Single TransposeCodec with reverse permutation → 'F' tc_f = Zarr.Codecs.V3Codecs.TransposeCodec((3,2,1)) - p = Zarr.V3Pipeline((tc_f,), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test Zarr.get_order(md) == 'F' + p = ZarrCore.V3Pipeline((tc_f,), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + @test ZarrCore.get_order(md) == 'F' # Single TransposeCodec with arbitrary (non-C, non-F) permutation → ArgumentError tc_other = Zarr.Codecs.V3Codecs.TransposeCodec((2,1,3)) - p = Zarr.V3Pipeline((tc_other,), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test_throws ArgumentError Zarr.get_order(md) + p = ZarrCore.V3Pipeline((tc_other,), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + @test_throws ArgumentError ZarrCore.get_order(md) # Multiple array->array codecs → ArgumentError - p = Zarr.V3Pipeline((tc_f, tc_f), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test_throws ArgumentError Zarr.get_order(md) + p = ZarrCore.V3Pipeline((tc_f, tc_f), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + @test_throws ArgumentError ZarrCore.get_order(md) # Unrecognized array->array codec type → ArgumentError struct _FakeCodec <: Zarr.Codecs.V3Codecs.V3Codec{:array,:array} end - p = Zarr.V3Pipeline((_FakeCodec(),), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test_throws ArgumentError Zarr.get_order(md) + p = ZarrCore.V3Pipeline((_FakeCodec(),), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + @test_throws ArgumentError ZarrCore.get_order(md) end @testset "VLenUTF8V3Codec" begin @@ -151,8 +152,8 @@ end {"name":"bytes","configuration":{"endian":"little"}}, {"name":"blosc","configuration":{"cname":"lz4","clevel":5,"shuffle":"$shuffle_str","blocksize":0,"typesize":4}} ]}""" - md = Zarr.Metadata(json_str, false) - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata(json_str, false) + pipeline = ZarrCore.get_pipeline(md) blosc = pipeline.bytes_bytes[1] @test blosc isa Zarr.Codecs.V3Codecs.BloscV3Codec @test blosc.shuffle == expected_int @@ -167,8 +168,8 @@ end {"name":"bytes","configuration":{"endian":"little"}}, {"name":"blosc","configuration":{"cname":"lz4","clevel":5,"shuffle":$shuffle_int,"blocksize":0,"typesize":4}} ]}""" - md = Zarr.Metadata(json_str, false) - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata(json_str, false) + pipeline = ZarrCore.get_pipeline(md) blosc = pipeline.bytes_bytes[1] @test blosc.shuffle == expected_int end @@ -181,7 +182,7 @@ end {"name":"bytes","configuration":{"endian":"little"}}, {"name":"blosc","configuration":{"cname":"lz4","clevel":5,"shuffle":"invalid","blocksize":0,"typesize":4}} ]}""" - @test_throws ArgumentError Zarr.Metadata(bad_json, false) + @test_throws ArgumentError ZarrCore.Metadata(bad_json, false) # --- serialization: integer -> shuffle string --- for (shuffle_int, expected_str) in ((0, "noshuffle"), (1, "shuffle"), (2, "bitshuffle")) @@ -192,7 +193,7 @@ end {"name":"bytes","configuration":{"endian":"little"}}, {"name":"blosc","configuration":{"cname":"lz4","clevel":5,"shuffle":$shuffle_int,"blocksize":0,"typesize":4}} ]}""" - md = Zarr.Metadata(json_str, false) + md = ZarrCore.Metadata(json_str, false) lowered = JSON.lower(md) blosc_config = lowered["codecs"][2]["configuration"] @test blosc_config["shuffle"] == expected_str @@ -201,8 +202,8 @@ end # --- serialization: unknown shuffle integer raises ArgumentError via lower3 --- let bad_blosc = Zarr.Codecs.V3Codecs.BloscV3Codec("lz4", 5, 99, 0, 4), bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec(), - bad_pipeline = Zarr.V3Pipeline((), bytes_codec, (bad_blosc,)) - bad_md = Zarr.MetadataV3{Int32,1,typeof(bad_pipeline)}( + bad_pipeline = ZarrCore.V3Pipeline((), bytes_codec, (bad_blosc,)) + bad_md = ZarrCore.MetadataV3{Int32,1,typeof(bad_pipeline)}( 3, "array", (4,), (4,), "int32", bad_pipeline, Int32(0), Zarr.ChunkKeyEncoding('/', true) ) @@ -268,8 +269,8 @@ end {"name":"numcodecs.blosc","configuration":{"cname":"lz4","clevel":5, "shuffle":"noshuffle","blocksize":0,"typesize":2}} ]}""" - md = @test_nowarn Zarr.Metadata(json_str, false) - pipeline = Zarr.get_pipeline(md) + md = @test_nowarn ZarrCore.Metadata(json_str, false) + pipeline = ZarrCore.get_pipeline(md) @test pipeline.bytes_bytes[1] isa Zarr.Codecs.V3Codecs.BloscV3Codec end @@ -283,7 +284,7 @@ end {"name":"numcodecs.blosc","configuration":{"cname":"lz4","clevel":5, "shuffle":"noshuffle","blocksize":0,"typesize":2}} ]}""" - md = Zarr.Metadata(json_str, false) + md = ZarrCore.Metadata(json_str, false) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) data = Int16[1, 2, 3, 4] @@ -303,7 +304,7 @@ end "shuffle":"noshuffle","blocksize":0,"typesize":2}} ]}""" store["zarr.json"] = Vector{UInt8}(json_str) - @test_nowarn @test Zarr.is_zarray(Zarr.ZarrFormat(Val(3)), store, "") == true + @test_nowarn @test ZarrCore.is_zarray(ZarrCore.ZarrFormat(Val(3)), store, "") == true end end @@ -322,8 +323,8 @@ end # but data reads and writes must still work correctly. tc = Zarr.Codecs.V3Codecs.TransposeCodec((2, 1, 3)) bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() - pipeline = Zarr.V3Pipeline((tc,), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(pipeline)}( + pipeline = ZarrCore.V3Pipeline((tc,), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(pipeline)}( 3, "array", (2,3,4), (2,3,4), "int32", pipeline, Int32(0), Zarr.ChunkKeyEncoding('/', true) ) @@ -334,12 +335,12 @@ end @test z[:,:,:] == data # get_order throws for non-canonical permutation - @test_throws ArgumentError Zarr.get_order(z.metadata) + @test_throws ArgumentError ZarrCore.get_order(z.metadata) end @testset "V3 group attributes round-trip" begin store = Zarr.DictStore() - g = zgroup(store, "", Zarr.ZarrFormat(3)) + g = zgroup(store, "", ZarrCore.ZarrFormat(3)) zgroup(g, "sub"; attrs=Dict("key" => "val", "num" => 42)) # Re-open the store and verify attributes are preserved @@ -351,8 +352,8 @@ end @testset "CRC32c end-to-end ZArray" begin crc32c_codec = Zarr.Codecs.V3Codecs.CRC32cV3Codec() bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() - pipeline = Zarr.V3Pipeline((), bytes_codec, (crc32c_codec,)) - md = Zarr.MetadataV3{Int32,1,typeof(pipeline),Zarr.ChunkKeyEncoding}( + pipeline = ZarrCore.V3Pipeline((), bytes_codec, (crc32c_codec,)) + md = ZarrCore.MetadataV3{Int32,1,typeof(pipeline),Zarr.ChunkKeyEncoding}( 3, "array", (4,), (4,), "int32", pipeline, Int32(0), Zarr.ChunkKeyEncoding('/', true) ) @@ -373,72 +374,72 @@ end @testset "V2Pipeline encode/decode round-trip" begin comp = Zarr.BloscCompressor() - pipeline = Zarr.V2Pipeline(comp, nothing) + pipeline = ZarrCore.V2Pipeline(comp, nothing) data = zeros(Int64, 4, 4) data[1, 1] = 42 - encoded = Zarr.pipeline_encode(pipeline, data, nothing) + encoded = ZarrCore.pipeline_encode(pipeline, data, nothing) @test encoded isa Vector{UInt8} @test !isempty(encoded) output = zeros(Int64, 4, 4) - Zarr.pipeline_decode!(pipeline, output, encoded) + ZarrCore.pipeline_decode!(pipeline, output, encoded) @test output == data end @testset "V2Pipeline with fill_value returns nothing" begin comp = Zarr.BloscCompressor() - pipeline = Zarr.V2Pipeline(comp, nothing) + pipeline = ZarrCore.V2Pipeline(comp, nothing) data = fill(Int64(-1), 4, 4) - encoded = Zarr.pipeline_encode(pipeline, data, Int64(-1)) + encoded = ZarrCore.pipeline_encode(pipeline, data, Int64(-1)) @test encoded === nothing end @testset "V3Pipeline encode/decode round-trip" begin bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() gzip_codec = Zarr.Codecs.V3Codecs.GzipV3Codec(6) - pipeline = Zarr.V3Pipeline((), bytes_codec, (gzip_codec,)) + pipeline = ZarrCore.V3Pipeline((), bytes_codec, (gzip_codec,)) data = Int32[1, 2, 3, 4] - encoded = Zarr.pipeline_encode(pipeline, data, nothing) + encoded = ZarrCore.pipeline_encode(pipeline, data, nothing) @test encoded isa Vector{UInt8} output = zeros(Int32, 4) - Zarr.pipeline_decode!(pipeline, output, encoded) + ZarrCore.pipeline_decode!(pipeline, output, encoded) @test output == data end @testset "V3Pipeline with no compression" begin bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() - pipeline = Zarr.V3Pipeline((), bytes_codec, ()) + pipeline = ZarrCore.V3Pipeline((), bytes_codec, ()) data = Float64[1.5, 2.5, 3.5] - encoded = Zarr.pipeline_encode(pipeline, data, nothing) + encoded = ZarrCore.pipeline_encode(pipeline, data, nothing) @test encoded isa Vector{UInt8} output = zeros(Float64, 3) - Zarr.pipeline_decode!(pipeline, output, encoded) + ZarrCore.pipeline_decode!(pipeline, output, encoded) @test output == data end @testset "V3Pipeline fill_value returns nothing" begin bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() - pipeline = Zarr.V3Pipeline((), bytes_codec, ()) + pipeline = ZarrCore.V3Pipeline((), bytes_codec, ()) data = fill(Int32(0), 4) - encoded = Zarr.pipeline_encode(pipeline, data, Int32(0)) + encoded = ZarrCore.pipeline_encode(pipeline, data, Int32(0)) @test encoded === nothing end @testset "V3 Metadata Parsing" begin json_str = """{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}},"chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}},"fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":[0]}},{"name":"bytes","configuration":{"endian":"little"}},{"name":"gzip","configuration":{"level":6}}]}""" - md = Zarr.Metadata(json_str, false) - @test md isa Zarr.MetadataV3 + md = ZarrCore.Metadata(json_str, false) + @test md isa ZarrCore.MetadataV3 @test md.shape[] == (4,) @test md.chunks == (4,) @test md.fill_value == Int32(0) - pipeline = Zarr.get_pipeline(md) - @test pipeline isa Zarr.V3Pipeline + pipeline = ZarrCore.get_pipeline(md) + @test pipeline isa ZarrCore.V3Pipeline @test length(pipeline.array_array) == 1 @test pipeline.array_bytes isa Zarr.Codecs.V3Codecs.BytesCodec @test length(pipeline.bytes_bytes) == 1 @@ -446,8 +447,8 @@ end @testset "V3 Metadata JSON round-trip" begin json_str = """{"zarr_format":3,"node_type":"array","shape":[4,4],"data_type":"float64","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[2,2]}},"chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}},"fill_value":0.0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}},{"name":"blosc","configuration":{"cname":"lz4","clevel":5,"shuffle":"noshuffle","blocksize":0,"typesize":4}}]}""" - md = Zarr.Metadata(json_str, false) - @test md isa Zarr.MetadataV3 + md = ZarrCore.Metadata(json_str, false) + @test md isa ZarrCore.MetadataV3 # Serialize back to JSON lowered = JSON.lower(md) @@ -458,17 +459,17 @@ end @testset "V3 Group Metadata Parsing" begin json_str = """{"zarr_format":3,"node_type":"group"}""" - md = Zarr.Metadata(json_str, false) - @test md isa Zarr.MetadataV3 + md = ZarrCore.Metadata(json_str, false) + @test md isa ZarrCore.MetadataV3 @test md.node_type == "group" end @testset "typestr3 raw types" begin - @test Zarr.typestr3("r8") == NTuple{1,UInt8} - @test Zarr.typestr3("r16") == NTuple{2,UInt8} - @test Zarr.typestr3("r64") == NTuple{8,UInt8} - @test_throws ArgumentError Zarr.typestr3("rxyz") # non-numeric bits - @test_throws ArgumentError Zarr.typestr3("r7") # not a multiple of 8 + @test ZarrCore.typestr3("r8") == NTuple{1,UInt8} + @test ZarrCore.typestr3("r16") == NTuple{2,UInt8} + @test ZarrCore.typestr3("r64") == NTuple{8,UInt8} + @test_throws ArgumentError ZarrCore.typestr3("rxyz") # non-numeric bits + @test_throws ArgumentError ZarrCore.typestr3("r7") # not a multiple of 8 end @testset "V3 Metadata parsing error paths" begin @@ -478,57 +479,57 @@ end "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""" # Unknown node_type - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"unknown"}""", false) + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"unknown"}""", false) # Extra key in group metadata - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"group","bad_key":1}""", false) + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"group","bad_key":1}""", false) # Missing required key (shape) - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Unknown chunk_grid name - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"unknown","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Shape/chunk rank mismatch - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[2,2]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Unknown chunk_key_encoding name - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"unknown"}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Unknown codec - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}},{"name":"unknown_codec"}]}""", false) # Deprecated string transpose order "C" - @test_logs (:warn,) Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_logs (:warn,) ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":"C"}}, {"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Deprecated string transpose order "F" - @test_logs (:warn,) Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_logs (:warn,) ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":"F"}}, {"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Unknown string transpose order - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":"X"}}, @@ -542,8 +543,8 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}, {"name":"zstd","configuration":{"level":3}}]}""" - md = Zarr.Metadata(json_zstd, false) - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata(json_zstd, false) + pipeline = ZarrCore.get_pipeline(md) @test pipeline.bytes_bytes[1] isa Zarr.Codecs.V3Codecs.ZstdV3Codec @test pipeline.bytes_bytes[1].level == 3 @@ -553,8 +554,8 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}, {"name":"crc32c"}]}""" - md = Zarr.Metadata(json_crc, false) - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata(json_crc, false) + pipeline = ZarrCore.get_pipeline(md) @test pipeline.bytes_bytes[1] isa Zarr.Codecs.V3Codecs.CRC32cV3Codec # F-order from numeric reverse permutation sets order='F' @@ -563,15 +564,15 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":[1,0]}}, {"name":"bytes","configuration":{"endian":"little"}}]}""" - md = Zarr.Metadata(json_f, false) - @test Zarr.get_order(md) == 'F' + md = ZarrCore.Metadata(json_f, false) + @test ZarrCore.get_order(md) == 'F' # v2 chunk_key_encoding (prefix=false, separator='.') json_v2enc = """{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"v2","configuration":{"separator":"."}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""" - md = Zarr.Metadata(json_v2enc, false) + md = ZarrCore.Metadata(json_v2enc, false) @test md.chunk_key_encoding.prefix == false @test md.chunk_key_encoding.sep == '.' end @@ -585,7 +586,7 @@ end "base_encoding":{"name":"default"} }}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""" - md = Zarr.Metadata(json_str, false) + md = ZarrCore.Metadata(json_str, false) @test md.chunk_key_encoding isa Zarr.SuffixChunkKeyEncoding @test md.chunk_key_encoding.suffix == ".tiff" @test md.chunk_key_encoding.base_encoding isa Zarr.ChunkKeyEncoding @@ -606,7 +607,7 @@ end "base_encoding":{"name":"v2"} }}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""" - md2 = Zarr.Metadata(json_v2base, false) + md2 = ZarrCore.Metadata(json_v2base, false) @test md2.chunk_key_encoding.suffix == ".shard.zip" @test md2.chunk_key_encoding.base_encoding.prefix == false # "v2" has no prefix @test Zarr.citostring(md2.chunk_key_encoding, CartesianIndex(1, 1)) == "0.0.shard.zip" @@ -622,10 +623,10 @@ end store = Zarr.DictStore() cke = Zarr.SuffixChunkKeyEncoding(".tiff", Zarr.ChunkKeyEncoding('/', true)) bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() - pipeline = Zarr.V3Pipeline((), bytes_codec, ()) + pipeline = ZarrCore.V3Pipeline((), bytes_codec, ()) P = typeof(pipeline) E = typeof(cke) - md = Zarr.MetadataV3{Int32,2,P,E}(3, "array", (4,4), (2,2), "int32", pipeline, Int32(0), cke) + md = ZarrCore.MetadataV3{Int32,2,P,E}(3, "array", (4,4), (2,2), "int32", pipeline, Int32(0), cke) z = Zarr.ZArray(md, store, "", Dict(), true) z[:,:] = reshape(Int32.(1:16), 4, 4) @test z[:,:] == reshape(Int32.(1:16), 4, 4) @@ -639,7 +640,7 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}, {"name":"zstd","configuration":{"level":5}}]}""" - md = Zarr.Metadata(json_zstd, false) + md = ZarrCore.Metadata(json_zstd, false) lowered = JSON.lower(md) @test lowered["codecs"][2]["name"] == "zstd" @test lowered["codecs"][2]["configuration"]["level"] == 5 @@ -650,7 +651,7 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}, {"name":"crc32c"}]}""" - md = Zarr.Metadata(json_crc, false) + md = ZarrCore.Metadata(json_crc, false) lowered = JSON.lower(md) @test lowered["codecs"][2]["name"] == "crc32c" @@ -660,7 +661,7 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":[1,0]}}, {"name":"bytes","configuration":{"endian":"little"}}]}""" - md = Zarr.Metadata(json_trans, false) + md = ZarrCore.Metadata(json_trans, false) lowered = JSON.lower(md) @test lowered["codecs"][1]["name"] == "transpose" @test lowered["codecs"][1]["configuration"]["order"] == [1, 0] @@ -669,24 +670,24 @@ end @testset "MetadataV3 convenience constructor" begin # order='F' creates a TransposeCodec data = zeros(Int32, 4, 4) - md = Zarr.Metadata3(data, (4,4); order='F') - @test Zarr.get_order(md) == 'F' - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata3(data, (4,4); order='F') + @test ZarrCore.get_order(md) == 'F' + pipeline = ZarrCore.get_pipeline(md) @test length(pipeline.array_array) == 1 @test pipeline.array_array[1] isa Zarr.Codecs.V3Codecs.TransposeCodec # ZstdCompressor translates to ZstdV3Codec - md_zstd = Zarr.Metadata3(data, (4,4); compressor=Zarr.ZstdCompressor()) - pipeline_zstd = Zarr.get_pipeline(md_zstd) + md_zstd = ZarrCore.Metadata3(data, (4,4); compressor=Zarr.ZstdCompressor()) + pipeline_zstd = ZarrCore.get_pipeline(md_zstd) @test pipeline_zstd.bytes_bytes[1] isa Zarr.Codecs.V3Codecs.ZstdV3Codec # fill_value=nothing defaults to zero(T) - md_nofv = Zarr.Metadata3(data, (4,4)) + md_nofv = ZarrCore.Metadata3(data, (4,4)) @test md_nofv.fill_value == Int32(0) # Unsupported compressor throws ArgumentError struct _BadCompressor <: Zarr.Compressor end - @test_throws ArgumentError Zarr.Metadata3(data, (4,4); compressor=_BadCompressor()) + @test_throws ArgumentError ZarrCore.Metadata3(data, (4,4); compressor=_BadCompressor()) end @testset "Metadata3 fixed_length_utf32" begin @@ -705,9 +706,9 @@ end "codecs" => [Dict{String, Any}("name" => "bytes", "configuration" => Dict{String, Any}("endian" => "little"))] ) - md = Zarr.Metadata3(d, false) + md = ZarrCore.Metadata3(d, false) # 40 bytes / 4 bytes per code unit = 10 code units - @test eltype(md) == Zarr.MaxLengthStrings.MaxLengthString{10, UInt32} + @test eltype(md) == ZarrCore.MaxLengthString{10, UInt32} # Test lowering back to JSON preserves the dict structure lowered = JSON.lower(md) @@ -744,7 +745,7 @@ end @testset "V3 Group Creation" begin store = Zarr.DictStore() - g = zgroup(store, "", Zarr.ZarrFormat(3)) + g = zgroup(store, "", ZarrCore.ZarrFormat(3)) @test haskey(store, "zarr.json") md = JSON.parse(String(copy(store["zarr.json"]))) @test md["zarr_format"] == 3 @@ -885,7 +886,7 @@ end @testset "V3 group with arrays" begin store = Zarr.DictStore() - g = zgroup(store, "", Zarr.ZarrFormat(3)) + g = zgroup(store, "", ZarrCore.ZarrFormat(3)) a = zcreate(Float64, g, "myarray", 10; zarr_format=3, chunks=(5,), fill_value=0.0) a[:] = Float64.(1:10) @@ -1123,19 +1124,19 @@ end @testset "ShardingCodec ragged inner chunks" begin # Outer chunk (shard) size does not evenly divide by inner chunk size. # shard shape (3,), inner chunk shape (2,): 2 inner chunks — full (1:2) + partial (3:3) - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{Int16,1,typeof(pipeline)}( + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline)}( 3, "array", (3,), (3,), "int16", pipeline, Int16(0), Zarr.ChunkKeyEncoding('/', true) ) @@ -1147,19 +1148,19 @@ end @test z[:] == data # 2D: shard (3,3), inner (2,2) — partial chunks on both axes - inner_pipeline2 = Zarr.V3Pipeline( + inner_pipeline2 = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), () ) - index_pipeline2 = Zarr.V3Pipeline( + index_pipeline2 = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding2 = Zarr.Codecs.V3Codecs.ShardingCodec((2,2), inner_pipeline2, index_pipeline2, :end) - pipeline2 = Zarr.V3Pipeline((), sharding2, ()) - md2 = Zarr.MetadataV3{Int32,2,typeof(pipeline2)}( + pipeline2 = ZarrCore.V3Pipeline((), sharding2, ()) + md2 = ZarrCore.MetadataV3{Int32,2,typeof(pipeline2)}( 3, "array", (3,3), (3,3), "int32", pipeline2, Int32(0), Zarr.ChunkKeyEncoding('/', true) ) @@ -1174,19 +1175,19 @@ end @testset "ShardingCodec ZArray write and read" begin # Build a pipeline where ShardingCodec is the array->bytes codec. # Shard shape (outer chunk): (4,). Inner chunk shape: (2,). - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.GzipV3Codec(6),) ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{Int16,1,typeof(pipeline)}( + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline)}( 3, "array", (4,), (4,), "int16", pipeline, Int16(0), Zarr.ChunkKeyEncoding('/', true) ) @@ -1203,19 +1204,19 @@ end # :start index location. zencode! stored absolute offsets (shifted by # index_size), and zdecode! added chunk_data_offset=index_size again, # reading at 2×index_size + relative. - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), () ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :start) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{Int16,1,typeof(pipeline)}( + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline)}( 3, "array", (4,), (4,), "int16", pipeline, Int16(0), Zarr.ChunkKeyEncoding('/', true) ) @@ -1228,12 +1229,12 @@ end end @testset "ShardingCodec zdecode! fill_value for empty shard" begin - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), () ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) @@ -1271,8 +1272,8 @@ end "index_location":"end" }} ]}""" - md = Zarr.Metadata(json_str, false) - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata(json_str, false) + pipeline = ZarrCore.get_pipeline(md) sharding = pipeline.array_bytes blosc = sharding.codecs.bytes_bytes[1] @test blosc isa Zarr.Codecs.V3Codecs.BloscV3Codec @@ -1282,19 +1283,19 @@ end @testset "ShardingCodec multi-shard array" begin # Array size (8,) with shard (outer chunk) size (4,) and inner chunk size (2,). # Two shards, each containing 2 inner chunks. - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), () ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{Int16,1,typeof(pipeline)}( + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline)}( 3, "array", (8,), (4,), "int16", pipeline, Int16(0), Zarr.ChunkKeyEncoding('/', true) ) @@ -1311,19 +1312,19 @@ end @testset "ShardingCodec non-zero fill_value" begin # Shard shape (4,), inner chunk (2,); only write to first inner chunk. # The second inner chunk should read back as fill_value (Int16(99)), not zero. - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), () ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{Int16,1,typeof(pipeline)}( + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline)}( 3, "array", (4,), (4,), "int16", pipeline, Int16(99), Zarr.ChunkKeyEncoding('/', true) ) @@ -1353,7 +1354,7 @@ end "index_location":"end" }} ]}""" - @test_throws ArgumentError Zarr.Metadata(json_str, false) + @test_throws ArgumentError ZarrCore.Metadata(json_str, false) end end # V3 Codecs diff --git a/test/v3_julia.jl b/test/v3_julia.jl index 8600d041..46ab75f5 100644 --- a/test/v3_julia.jl +++ b/test/v3_julia.jl @@ -2,6 +2,7 @@ # Mirrors the examples from v3_python.jl using Zarr +import Zarr: ZarrCore using JSON # Paths @@ -14,7 +15,7 @@ end # Create store and root group for v3 store = Zarr.DirectoryStore(path_v3) -g = zgroup(store, "", Zarr.ZarrFormat(3)) +g = zgroup(store, "", ZarrCore.ZarrFormat(3)) # Helper: create array and set data function create_and_fill(store, name, data; @@ -216,25 +217,25 @@ function create_sharded(store, name, data, outer_chunk_shape, inner_chunk_shape; index_location::Symbol=:end, index_crc32c::Bool=true) T = eltype(data) N = ndims(data) - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.GzipV3Codec(1),), ) index_bytes_bytes = index_crc32c ? (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) : () - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), index_bytes_bytes, ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec(inner_chunk_shape, inner_pipeline, index_pipeline, index_location) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{T, N, typeof(pipeline)}( - 3, "array", size(data), outer_chunk_shape, Zarr.typestr3(T), pipeline, zero(T), + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + md = ZarrCore.MetadataV3{T, N, typeof(pipeline)}( + 3, "array", size(data), outer_chunk_shape, ZarrCore.typestr3(T), pipeline, zero(T), Zarr.ChunkKeyEncoding('/', true), ) z = Zarr.ZArray(md, store, name, Dict(), true) - Zarr.writemetadata(Zarr.zarr_format(md), store, name, md) + ZarrCore.writemetadata(ZarrCore.zarr_format(md), store, name, md) z[:] = data return z end @@ -327,6 +328,6 @@ create_and_fill(store, "consolidated/nested/1d.i2", Int16[10, 20, 30, 40]; compressor=Zarr.NoCompressor(), ) # Consolidate metadata for the consolidated group only -Zarr.consolidate_metadata(store, "consolidated", Zarr.ZarrFormat(3)) +Zarr.consolidate_metadata(store, "consolidated", ZarrCore.ZarrFormat(3)) @info "Zarr v3 fixtures generated at: $path_v3" From 997ed6297b9f9d21d261f1c96275457201cc281b Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 17:41:44 +0200 Subject: [PATCH 11/39] ai wants to drop numpy compat, not sure why but ok --- test/CondaPkg.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CondaPkg.toml b/test/CondaPkg.toml index 62c66cf3..d805cbbe 100644 --- a/test/CondaPkg.toml +++ b/test/CondaPkg.toml @@ -1,4 +1,4 @@ [deps] zarr = "3.*" -numpy = ">=2.3.3,<3" +numpy = "" numcodecs = "" From ef3416cbd4a0fe2252d53a81068e7285ee43c57f Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 18:42:41 +0200 Subject: [PATCH 12/39] restore numpy compat --- test/CondaPkg.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CondaPkg.toml b/test/CondaPkg.toml index d805cbbe..62c66cf3 100644 --- a/test/CondaPkg.toml +++ b/test/CondaPkg.toml @@ -1,4 +1,4 @@ [deps] zarr = "3.*" -numpy = "" +numpy = ">=2.3.3,<3" numcodecs = "" From 04f53207e2132a4e70240e7d07893b1d71682205 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 18:42:56 +0200 Subject: [PATCH 13/39] fix underlying problem --- test/consolidated.jl | 2 +- test/v3_python.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/consolidated.jl b/test/consolidated.jl index 9e0e607e..65c8fc73 100644 --- a/test/consolidated.jl +++ b/test/consolidated.jl @@ -5,7 +5,7 @@ using Zarr import Zarr: ZarrCore CondaPkg.add([ - PkgSpec("numpy"), + PkgSpec("numpy"; version=">=2.3.3,<3"), PkgSpec("zarr"; version="3.*"), PkgSpec("numcodecs") ]) diff --git a/test/v3_python.jl b/test/v3_python.jl index ec319a4d..20e4f1c7 100644 --- a/test/v3_python.jl +++ b/test/v3_python.jl @@ -6,7 +6,7 @@ using JSON # Install Python deps into Conda env used by PythonCall (zarr v3 and numpy) CondaPkg.add([ - PkgSpec("numpy"), + PkgSpec("numpy"; version=">=2.3.3,<3"), PkgSpec("zarr"; version="3.*"), PkgSpec("numcodecs") ]) From 9e157d3dbf390b9c1ae16eb90375017b2e2fcb92 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 22:47:01 +0200 Subject: [PATCH 14/39] Fix CI and docs for the ZarrCore split Co-Authored-By: Claude Opus 5 --- .github/workflows/CI.yml | 5 ++++- docs/src/UserGuide/caching.md | 4 ++-- docs/src/get_started.md | 2 +- docs/src/tutorials/tutorial.md | 6 +++--- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 92b555cc..475db016 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -50,7 +50,10 @@ jobs: julia --project=test -e 'using Pkg; Pkg.develop(path=pwd()); Pkg.resolve(); Pkg.instantiate()' julia --project=test test/v3_julia.jl julia --project=test test/v3_python.jl - rm test/Manifest.toml + # In the workspace layout the resolved manifest lives at the repo + # root, not in test/, so this file no longer exists; -f keeps the + # step from failing while still cleaning up a stale manifest. + rm -f test/Manifest.toml - uses: julia-actions/julia-runtest@v1 env: PYTHON: diff --git a/docs/src/UserGuide/caching.md b/docs/src/UserGuide/caching.md index 8d3aac1d..0d7fdc4f 100644 --- a/docs/src/UserGuide/caching.md +++ b/docs/src/UserGuide/caching.md @@ -23,7 +23,7 @@ and wrap it into a `CachedDiskArray`: ````jldoctest cache julia> a_lrucache = DiskArrays.cache(a,maxsize=1) -10000×10000 DiskArrays.CachedDiskArray{Float64, 2, ZArray{Float64, 2, DirectoryStore, Zarr.MetadataV2{Float64, 2, Zarr.BloscCompressor, Nothing}}, LRUCache.LRU{ChunkIndex{2, DiskArrays.OffsetChunks}, OffsetArrays.OffsetMatrix{Float64, Matrix{Float64}}}} +10000×10000 DiskArrays.CachedDiskArray{Float64, 2, ZArray{Float64, 2, DirectoryStore, ZarrCore.MetadataV2{Float64, 2, ZarrCore.BloscCompressor, Nothing}}, LRUCache.LRU{ChunkIndex{2, DiskArrays.OffsetChunks}, OffsetArrays.OffsetMatrix{Float64, Matrix{Float64}}}} Chunked: ( [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000] @@ -55,7 +55,7 @@ So, if we want to hold an incomplete local copy of any `AbstractDiskArray` you c ````jldoctest cache julia> a_arraycache = zarrcache(a,"./my_persistent_store.zarr") -10000×10000 DiskArrays.CachedDiskArray{Float64, 2, ZArray{Float64, 2, DirectoryStore, Zarr.MetadataV2{Float64, 2, Zarr.BloscCompressor, Nothing}}, Zarr.PermanentZarrCache{Float64, 2, ZArray{Float64, 2, DirectoryStore, Zarr.MetadataV2{Float64, 2, Zarr.BloscCompressor, Nothing}}}} +10000×10000 DiskArrays.CachedDiskArray{Float64, 2, ZArray{Float64, 2, DirectoryStore, ZarrCore.MetadataV2{Float64, 2, ZarrCore.BloscCompressor, Nothing}}, ZarrCore.PermanentZarrCache{Float64, 2, ZArray{Float64, 2, DirectoryStore, ZarrCore.MetadataV2{Float64, 2, ZarrCore.BloscCompressor, Nothing}}}} Chunked: ( [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000] diff --git a/docs/src/get_started.md b/docs/src/get_started.md index 501b186d..26dd93be 100644 --- a/docs/src/get_started.md +++ b/docs/src/get_started.md @@ -232,7 +232,7 @@ Zarr allows you to create hierarchical groups, similar to directories: using Zarr store = Zarr.DirectoryStore("experiment.zarr") -g = zgroup(store, "", Zarr.ZarrFormat(3)) +g = zgroup(store, "", 3) # 3 selects the Zarr v3 format zcreate(Float64, g, "temperature", 100, 100; chunks=(50, 50), fill_value=0.0) zcreate(Float64, g, "precipitation", 100, 100; chunks=(50, 50), fill_value=0.0) diff --git a/docs/src/tutorials/tutorial.md b/docs/src/tutorials/tutorial.md index 00cdf96d..93d16408 100644 --- a/docs/src/tutorials/tutorial.md +++ b/docs/src/tutorials/tutorial.md @@ -173,7 +173,7 @@ A number of different compressors can be used with Zarr. In this Julia package w julia> using Zarr julia> compressor = Zarr.BloscCompressor(cname="zstd", clevel=3, shuffle=true) -Zarr.BloscCompressor(0, 3, "zstd", 1) +ZarrCore.BloscCompressor(0, 3, "zstd", 1) julia> data = Int32(1):Int32(100000000) 1:100000000 @@ -197,7 +197,7 @@ Shape : (10000, 10000) Chunk Shape : (1000, 1000) Order : C Read-Only : false -Compressor : Zarr.BloscCompressor(0, 3, "zstd", 1) +Compressor : ZarrCore.BloscCompressor(0, 3, "zstd", 1) Filters : nothing Store type : Dictionary Storage No. bytes : 400000000 @@ -227,7 +227,7 @@ julia> z = zcreate(Vector{Int}, 4) ZArray{Vector{Int64}} of size 4 julia> z.metadata.filters -(Zarr.VLenArrayFilter{Int64}(),) +(ZarrCore.VLenArrayFilter{Int64}(),) julia> z[1:3] = [[1,3,5],[4],[7,9,14]]; From 558aefb59df2fe824309b4581aaf36f7538a82d8 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 22:55:37 +0200 Subject: [PATCH 15/39] CI: dev ZarrCore explicitly so LTS (Julia 1.10) can resolve it Co-Authored-By: Claude Opus 5 --- .github/workflows/CI.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 475db016..9f4d9f1f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -36,6 +36,15 @@ jobs: arch: ${{ matrix.arch }} show-versioninfo: true - uses: julia-actions/cache@v3 + # Pkg only understands `[sources]`/`[workspace]` from Julia 1.11 on, so on + # LTS (1.10) the unregistered path dependency on ZarrCore is invisible and + # resolution fails with "expected package ZarrCore to be registered". + # Dev'ing it explicitly fixes that; on 1.11+ it is a no-op because + # `[sources]` already points at the same path. This only affects developing + # from a checkout -- once both packages are registered, users resolve + # ZarrCore normally on any supported Julia. + - name: Develop ZarrCore (Pkg < 1.11 ignores [sources]) + run: julia --project=. -e 'using Pkg; Pkg.develop(path="lib/ZarrCore")' - uses: julia-actions/julia-buildpkg@v1 env: PYTHON: @@ -47,7 +56,7 @@ jobs: timeout_minutes: 5 max_attempts: 3 command: | - julia --project=test -e 'using Pkg; Pkg.develop(path=pwd()); Pkg.resolve(); Pkg.instantiate()' + julia --project=test -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path="lib/ZarrCore")]); Pkg.resolve(); Pkg.instantiate()' julia --project=test test/v3_julia.jl julia --project=test test/v3_python.jl # In the workspace layout the resolved manifest lives at the repo From 3077f5b687a54e0f8c9fb57420c994fdf17277ab Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 23:02:59 +0200 Subject: [PATCH 16/39] CI: fix Windows shell handling in the fixture step Co-Authored-By: Claude Opus 5 --- .github/workflows/CI.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9f4d9f1f..f434caf4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -44,6 +44,9 @@ jobs: # from a checkout -- once both packages are registered, users resolve # ZarrCore normally on any supported Julia. - name: Develop ZarrCore (Pkg < 1.11 ignores [sources]) + # bash on every platform: PowerShell mangles the embedded double quotes + # when forwarding them to a native executable. + shell: bash run: julia --project=. -e 'using Pkg; Pkg.develop(path="lib/ZarrCore")' - uses: julia-actions/julia-buildpkg@v1 env: @@ -55,14 +58,13 @@ jobs: with: timeout_minutes: 5 max_attempts: 3 + # bash on every platform, for the same quoting reason as above (and so + # `rm`-style POSIX commands added here behave consistently). + shell: bash command: | julia --project=test -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path="lib/ZarrCore")]); Pkg.resolve(); Pkg.instantiate()' julia --project=test test/v3_julia.jl julia --project=test test/v3_python.jl - # In the workspace layout the resolved manifest lives at the repo - # root, not in test/, so this file no longer exists; -f keeps the - # step from failing while still cleaning up a stale manifest. - rm -f test/Manifest.toml - uses: julia-actions/julia-runtest@v1 env: PYTHON: From 052a96ee4fc3552f9df293e11bceccb353bcfa89 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 23:27:40 +0200 Subject: [PATCH 17/39] Fix public API vanishing from `Zarr` on Julia 1.10 Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 2 ++ lib/ZarrCore/src/Codecs/Codecs.jl | 1 + lib/ZarrCore/src/Codecs/V3/V3.jl | 1 + lib/ZarrCore/src/ZarrCore.jl | 28 +++++++++++++++++++++++++--- src/Zarr.jl | 8 +++++++- test/runtests.jl | 17 ++++++++++++++++- 6 files changed, 52 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7278ef31..83c4956c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,8 @@ Julia version requirement: 1.10+. CI tests against Julia LTS, stable (`1`), nigh Because 1.10 is still supported, the `public` keyword (Julia 1.11+) cannot be used directly. Use the `ZarrCore.@public` macro instead — it expands to `public` on 1.11+ and to nothing on 1.10. +Since there is no `public` on 1.10, `names()` cannot report public names there, so `@public` *also* appends to the calling module's `PUBLIC_NAMES::Vector{Symbol}`. **Any module that uses `@public` must define its own `const PUBLIC_NAMES = Symbol[]`** (per-module, exactly like `public` itself); forgetting it is a load-time `UndefVarError`. The `Zarr` facade unions `names(ZarrCore)` with `ZarrCore.PUBLIC_NAMES`, which is what keeps the public API present on LTS — without it, every public-but-not-exported name silently vanishes from `Zarr` on 1.10 while 1.11+ looks fine. + ## Architecture ### Core Type Hierarchy diff --git a/lib/ZarrCore/src/Codecs/Codecs.jl b/lib/ZarrCore/src/Codecs/Codecs.jl index a7783134..3c767162 100644 --- a/lib/ZarrCore/src/Codecs/Codecs.jl +++ b/lib/ZarrCore/src/Codecs/Codecs.jl @@ -47,6 +47,7 @@ getCodec(::Type{<:Codec}, d::Dict) = error("Unimplemented") include("V3/V3.jl") import ..ZarrCore: @public +const PUBLIC_NAMES = Symbol[] # `@public` records here; see `?ZarrCore.@public` @public Codec, V3Codecs, zencode, zencode!, zdecode, zdecode!, getCodec end diff --git a/lib/ZarrCore/src/Codecs/V3/V3.jl b/lib/ZarrCore/src/Codecs/V3/V3.jl index 1d26379c..b957e154 100644 --- a/lib/ZarrCore/src/Codecs/V3/V3.jl +++ b/lib/ZarrCore/src/Codecs/V3/V3.jl @@ -826,6 +826,7 @@ end # The codec types themselves are re-exported (as public names) from `ZarrCore`; # these are the extension points for defining and registering new v3 codecs. import ...ZarrCore: @public +const PUBLIC_NAMES = Symbol[] # `@public` records here; see `?ZarrCore.@public` @public V3Codec, getCodec, register_codec, codec_parsers, codec_encode, codec_decode, is_fixed_size, name @public BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, ShardingCodec, diff --git a/lib/ZarrCore/src/ZarrCore.jl b/lib/ZarrCore/src/ZarrCore.jl index 467d8917..402b02f1 100644 --- a/lib/ZarrCore/src/ZarrCore.jl +++ b/lib/ZarrCore/src/ZarrCore.jl @@ -5,6 +5,21 @@ import Blosc import Unicode using OrderedCollections: OrderedDict +""" + PUBLIC_NAMES::Vector{Symbol} + +Every name this module declares with [`@public`](@ref), in declaration order. + +On Julia 1.11+ this duplicates what `names(ZarrCore)` already reports, but on +1.10 the `public` keyword does not exist, so this registry is the only record +of the public-but-not-exported API. The `Zarr` facade uses it to re-export the +same set of names on every supported Julia version. + +Like `public` itself, this is per-module: every module that uses `@public` owns +a `PUBLIC_NAMES`, and a submodule's public names live in *its* list, not here. +""" +const PUBLIC_NAMES = Symbol[] + """ @public name @public name1, name2, ... @@ -13,14 +28,21 @@ Mark `name`s as public API without exporting them, i.e. `Base.ispublic` returns `true` and they show up in `names(ZarrCore)`, but `using ZarrCore` does not bring them into scope. -This is the `public` keyword, which only exists from Julia 1.11 onwards; on -Julia 1.10 the declaration expands to nothing. +This is the `public` keyword, which only exists from Julia 1.11 onwards. On +Julia 1.10 the declaration itself expands to nothing, so the names are recorded +in the calling module's [`PUBLIC_NAMES`](@ref) as well -- `names()` cannot +report them there. A module using `@public` must therefore define its own +`const PUBLIC_NAMES = Symbol[]`. """ macro public(ex) syms = ex isa Symbol ? [ex] : ex.args all(s -> s isa Symbol, syms) || throw(ArgumentError("@public expects one or more symbols, got $ex")) - VERSION >= v"1.11" ? esc(Expr(:public, syms...)) : nothing + decl = VERSION >= v"1.11" ? esc(Expr(:public, syms...)) : nothing + quote + append!($(esc(:PUBLIC_NAMES)), $(QuoteNode(syms))) + $decl + end end struct ZarrFormat{V} diff --git a/src/Zarr.jl b/src/Zarr.jl index e7d5aa23..084150e5 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -3,7 +3,13 @@ module Zarr import ZarrCore # Mirror ZarrCore's export/public split. Internals stay at Zarr.ZarrCore. -for name in names(ZarrCore; all = false) +# +# `names` reports exported *and* public names, but only from Julia 1.11 on -- +# 1.10 has no `public` keyword, so ZarrCore additionally records every `@public` +# name in `PUBLIC_NAMES`. Taking the union keeps the surface identical on every +# supported version; without it, every public-but-not-exported name would be +# missing from `Zarr` on LTS. +for name in union(names(ZarrCore; all = false), ZarrCore.PUBLIC_NAMES) if name !== :ZarrCore @eval import ZarrCore: $name end diff --git a/test/runtests.jl b/test/runtests.jl index 184e6248..c0606e1c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -16,7 +16,8 @@ using Dates # These tests pin that split down so it cannot regress silently. # # On Julia 1.10 there is no `public`, so `names` yields only exports and the - # public-only sets are empty; the assertions still hold. + # public-only sets are empty; the assertions still hold. The final pair of + # tests covers what that blind spot hides. # A module always lists its own name, and `Zarr` additionally carries the # `ZarrCore` binding (public, so `Zarr.ZarrCore` is a documented escape hatch # rather than something `using Zarr` drags in). Both are structural, not API. @@ -36,6 +37,20 @@ using Dates # not become an export of `Zarr`, and vice versa. @test isempty(intersect(publiconly(ZarrCore), exported(Zarr))) @test isempty(intersect(exported(ZarrCore), publiconly(Zarr))) + + # Version-independent: the two assertions above compare `names` against + # `names`, so on 1.10 -- where `@public` expands to nothing and both + # public-only sets are empty -- they pass no matter what `Zarr` re-exports. + # `PUBLIC_NAMES` is populated on every version, so this catches a facade + # that silently drops the entire public API on LTS. + @test !isempty(ZarrCore.PUBLIC_NAMES) + @test isempty(filter(n -> !isdefined(Zarr, n), ZarrCore.PUBLIC_NAMES)) + + # Where both sources exist, they must agree -- otherwise LTS and 1.11+ would + # drift apart again, which is exactly what the registry is there to prevent. + @static if VERSION >= v"1.11" + @test Set(ZarrCore.PUBLIC_NAMES) == publiconly(ZarrCore) + end end @testset "ZArray" begin From 932baa594609f0e09771261254cd50535cf42955 Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 17 Aug 2026 23:40:31 +0200 Subject: [PATCH 18/39] CI: drop the test manifest so Pkg.test can sandbox on LTS Co-Authored-By: Claude Opus 5 --- .github/workflows/CI.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f434caf4..690b2ee3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -65,6 +65,18 @@ jobs: julia --project=test -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path="lib/ZarrCore")]); Pkg.resolve(); Pkg.instantiate()' julia --project=test test/v3_julia.jl julia --project=test test/v3_python.jl + # The step above dev's ZarrCore into the test environment so the fixture + # scripts can `using Zarr` on LTS, which makes ZarrCore a *direct* dep of + # test/Project.toml. `Pkg.test` then builds its sandbox by merging the + # active manifest (where ZarrCore is also dev'd, hence fixed) with the + # fixed deps of the test manifest, and refuses to merge a package that + # appears in both: "ERROR: can not merge projects". Dropping the test + # manifest leaves nothing to merge; the sandbox resolves ZarrCore from the + # active manifest instead. On 1.11+ the workspace keeps the only manifest + # at the repo root, so this is a no-op there. + - name: Drop the test manifest so Pkg.test can build its sandbox + shell: bash + run: rm -f test/Manifest.toml - uses: julia-actions/julia-runtest@v1 env: PYTHON: From e2d0c6076f21c58d777f73a1153f28bc25434a89 Mon Sep 17 00:00:00 2001 From: Fabian Gans Date: Mon, 17 Aug 2026 16:30:19 +0200 Subject: [PATCH 19/39] implement irregular chunk extension --- lib/ZarrCore/src/ZArray.jl | 289 +++++++++++++++++----------------- lib/ZarrCore/src/metadata.jl | 86 +++++----- lib/ZarrCore/src/metadata3.jl | 174 ++++++++++++-------- 3 files changed, 300 insertions(+), 249 deletions(-) diff --git a/lib/ZarrCore/src/ZArray.jl b/lib/ZarrCore/src/ZArray.jl index 857b9a6b..543d0e99 100644 --- a/lib/ZarrCore/src/ZArray.jl +++ b/lib/ZarrCore/src/ZArray.jl @@ -10,23 +10,23 @@ Number of tasks to use for async reading of chunks. Warning: setting this to ver """ const concurrent_io_tasks = Ref(50) -getfillval(::Type{T}, t::String) where {T <: Number} = parse(T, t) +getfillval(::Type{T}, t::String) where {T<:Number} = parse(T, t) getfillval(::Type{T}, t::Union{T,Nothing}) where {T} = t struct SenMissArray{T,N} <: AbstractArray{Union{T,Missing},N} x::Array{T,N} senval::T end -SenMissArray(x::Array{T,N},v) where {T,N} = SenMissArray{T,N}(x,convert(T,v)) +SenMissArray(x::Array{T,N}, v) where {T,N} = SenMissArray{T,N}(x, convert(T, v)) Base.size(x::SenMissArray) = size(x.x) senval(x::SenMissArray) = x.senval -function Base.getindex(x::SenMissArray,i::Int) +function Base.getindex(x::SenMissArray, i::Int) v = x.x[i] - isequal(v,senval(x)) ? missing : v + isequal(v, senval(x)) ? missing : v end -Base.setindex!(x::SenMissArray,v,i::Int) = x.x[i] = v -Base.setindex!(x::SenMissArray,::Missing,i::Int) = x.x[i] = senval(x) -Base.IndexStyle(::Type{<:SenMissArray})=Base.IndexLinear() +Base.setindex!(x::SenMissArray, v, i::Int) = x.x[i] = v +Base.setindex!(x::SenMissArray, ::Missing, i::Int) = x.x[i] = senval(x) +Base.IndexStyle(::Type{<:SenMissArray}) = Base.IndexLinear() # Struct representing a Zarr Array in Julia, note that # chunks(chunk size) and size are always in Julia column-major order @@ -40,7 +40,7 @@ end Base.eltype(::ZArray{T}) where {T} = T Base.ndims(::ZArray{<:Any,N}) where {N} = N -Base.size(z::ZArray{<:Any,N}) where {N} = z.metadata.shape[]::NTuple{N, Int} +Base.size(z::ZArray{<:Any,N}) where {N} = z.metadata.shape[]::NTuple{N,Int} function Base.size(z::ZArray{<:Any,N}, i::Integer) where {N} len = length(z.metadata.shape[]) if 0 < i <= len @@ -55,17 +55,17 @@ Base.length(z::ZArray) = prod(z.metadata.shape[])::Int Base.lastindex(z::ZArray{<:Any,N}, n::Integer) where {N} = size(z, n)::Int Base.lastindex(z::ZArray{<:Any,1}) = size(z, 1)::Int -function Base.show(io::IO,z::ZArray) - print(io, "ZArray{", eltype(z) ,"} of size ",join(string.(size(z)), " x ")) +function Base.show(io::IO, z::ZArray) + print(io, "ZArray{", eltype(z), "} of size ", join(string.(size(z)), " x ")) end -function Base.show(io::IO,::MIME"text/plain",z::ZArray) - print(io, "ZArray{", eltype(z) ,"} of size ",join(string.(size(z)), " x ")) +function Base.show(io::IO, ::MIME"text/plain", z::ZArray) + print(io, "ZArray{", eltype(z), "} of size ", join(string.(size(z)), " x ")) end zname(z::ZArray) = zname(z.path) function zname(s::String) - spl = split(rstrip(s,'/'),'/') + spl = split(rstrip(s, '/'), '/') isempty(last(spl)) ? "root" : last(spl) end @@ -75,7 +75,7 @@ end Returns the size of the compressed data stored in the ZArray `z` in bytes """ -storagesize(z::ZArray) = storagesize(z.storage,z.path) +storagesize(z::ZArray) = storagesize(z.storage, z.path) """ storageratio(z::ZArray) @@ -90,28 +90,32 @@ nobytes(z::ZArray) = length(z)*sizeof(eltype(z)) nobytes(z::ZArray{<:Vector}) = "unknown" nobytes(z::ZArray{<:String}) = "unknown" -zinfo(z::ZArray) = zinfo(stdout,z) -function zinfo(io::IO,z::ZArray) +zinfo(z::ZArray) = zinfo(stdout, z) +function zinfo(io::IO, z::ZArray) ninit = sum(chunkindices(z)) do i store_isinitialized(z.storage, z.path, i, z.metadata.chunk_key_encoding) end allinfos = [ - "Type" => "ZArray", - "Data type" => eltype(z), - "Shape" => size(z), - "Chunk Shape" => z.metadata.chunks, - "Order" => try get_order(z.metadata) catch e "unknown ($(e.msg))" end, - "Read-Only" => !z.writeable, - "Compressor" => z.metadata isa MetadataV2 ? z.metadata.compressor : get_pipeline(z.metadata), - "Filters" => z.metadata isa MetadataV2 ? z.metadata.filters : nothing, - "Store type" => z.storage, - "No. bytes" => nobytes(z), - "No. bytes stored" => storagesize(z), - "Storage ratio" => storageratio(z), - "Chunks initialized" => "$(ninit)/$(length(chunkindices(z)))" + "Type" => "ZArray", + "Data type" => eltype(z), + "Shape" => size(z), + "Chunk Shape" => z.metadata.chunks, + "Order" => try + get_order(z.metadata) + catch e + "unknown ($(e.msg))" + end, + "Read-Only" => !z.writeable, + "Compressor" => z.metadata isa MetadataV2 ? z.metadata.compressor : get_pipeline(z.metadata), + "Filters" => z.metadata isa MetadataV2 ? z.metadata.filters : nothing, + "Store type" => z.storage, + "No. bytes" => nobytes(z), + "No. bytes stored" => storagesize(z), + "Storage ratio" => storageratio(z), + "Chunks initialized" => "$(ninit)/$(length(chunkindices(z)))" ] foreach(allinfos) do ii - println(io,rpad(ii[1],20),": ",ii[2]) + println(io, rpad(ii[1], 20), ": ", ii[2]) end end @@ -124,7 +128,7 @@ function ZArray(s::T, mode="r", path="", zarr_format=:auto; fill_as_missing=fals metadata = getmetadata(zv, s, path, fill_as_missing) attrs = getattrs(zv, s, path) writeable = mode == "w" - startswith(path,"/") && error("Paths should never start with a leading '/'") + startswith(path, "/") && error("Paths should never start with a leading '/'") ZArray(metadata, s, string(path), attrs, writeable) end @@ -132,33 +136,24 @@ zarr_format(z::ZArray) = zarr_format(z.metadata) dimension_separator(z::ZArray) = dimension_separator(z.metadata) -""" - trans_ind(r, bs) - -For a given index and blocksize determines which chunks of the Zarray will have to -be accessed. -""" -trans_ind(r::AbstractUnitRange, bs) = fld1(first(r),bs):fld1(last(r),bs) -trans_ind(r::Integer, bs) = fld1(r,bs) - function boundint(r1, s2, o2) - r2 = range(o2+1,length=s2) - f1, f2 = first(r1), first(r2) - l1, l2 = last(r1),last(r2) + r2 = range(o2+1, length=s2) + f1, f2 = first(r1), first(r2) + l1, l2 = last(r1), last(r2) UnitRange(f1 > f2 ? f1 : f2, l1 < l2 ? l1 : l2) end function getchunkarray(z::ZArray{>:Missing}) # temporary workaround to use strings as data values inner = fill(z.metadata.fill_value, z.metadata.chunks) - a = SenMissArray(inner,z.metadata.fill_value) + a = SenMissArray(inner, z.metadata.fill_value) end _zero(T) = zero(T) _zero(T::Type{<:MaxLengthString}) = zero(T) _zero(T::Type{ASCIIChar}) = ASCIIChar(0) _zero(::Type{<:Vector{T}}) where T = T[] _zero(::Type{Char}) = Char(0) -getchunkarray(z::ZArray) = fill(_zero(eltype(z)), z.metadata.chunks) +getchunkarray(z::ZArray) = fill(_zero(eltype(z)), DiskArrays.max_chunksize.(z.metadata.chunks.chunks)) # Same as `getchunkarray` but skips the zero/fill_value-fill. Use only when # the caller guarantees the buffer will be fully overwritten before any read @@ -169,14 +164,14 @@ getchunkarray(z::ZArray) = fill(_zero(eltype(z)), z.metadata.chunks) # of a `SenMissArray`, not an `Array{Union{Missing,T}}` directly (Blosc and # friends reject non-isbits eltypes). function getchunkarray_undef(z::ZArray{T}) where {T} - Missing <: T && return getchunkarray(z) - return Array{T}(undef, z.metadata.chunks) + Missing <: T && return getchunkarray(z) + return Array{T}(undef, DiskArrays.max_chunksize.(z.metadata.chunks.chunks)) end maybeinner(a::Array) = a maybeinner(a::SenMissArray) = a.x -resetbuffer!(fv,a::Array) = fv === nothing || fill!(a,fv) -resetbuffer!(_,a::SenMissArray) = fill!(a,missing) +resetbuffer!(fv, a::Array) = fv === nothing || fill!(a, fv) +resetbuffer!(_, a::SenMissArray) = fill!(a, missing) # Returns the chunk index when the call qualifies for the single-chunk fast # path: a plain `Array{T,N}` matching the chunk shape, `Missing <: T` false, @@ -185,7 +180,7 @@ resetbuffer!(_,a::SenMissArray) = fill!(a,missing) # coverage check is needed. function singlechunk_fastpath(arr, z::ZArray{T,N}, blockr::CartesianIndices{N}) where {T,N} arr isa Array{T,N} && !(Missing <: T) && - size(arr) == z.metadata.chunks && length(blockr) == 1 || return nothing + size(arr) == z.metadata.chunks && length(blockr) == 1 || return nothing return first(blockr) end @@ -226,17 +221,18 @@ function write_singlechunk_fastpath!( return nothing end store_writechunk(z.storage, _reinterpret(UInt8, ain), - z.path, bI, z.metadata.chunk_key_encoding) + z.path, bI, z.metadata.chunk_key_encoding) return nothing end # Function to read or write from a zarr array. Could be refactored # using type system to get rid of the `if readmode` statements. -function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::CartesianIndices{N}) where {N} +function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::CartesianIndices{N}) where {N} - output_base_offsets = map(i->first(i)-1,r.indices) + output_base_offsets = map(i->first(i)-1, r.indices) # Determines which chunks are affected - blockr = CartesianIndices(map(trans_ind, r.indices, z.metadata.chunks)) + blockr = CartesianIndices(map(DiskArrays.findchunk, z.metadata.chunks.chunks, r.indices)) + @show blockr # Fast path: single-chunk full-read decodes directly into `aout`, skipping the readtask channel and scratch buffer. bI = singlechunk_fastpath(aout, z, blockr) if bI !== nothing @@ -251,37 +247,37 @@ function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes a = getchunkarray_undef(z) # Now loop through the chunks c = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) - + task = @async begin read_items!($(z.storage), c, $(z.metadata.chunk_key_encoding), $(z.path), $(blockr)) end - bind(c,task) + bind(c, task) - try + try for i in 1:length(blockr) - - bI,chunk_compressed = take!(c) - - current_chunk_offsets = map((s,i)->s*(i-1),size(a),Tuple(bI)) - - indranges = map(boundint,r.indices,size(a),current_chunk_offsets) - - uncompress_to_output!(aout,output_base_offsets,z,chunk_compressed,current_chunk_offsets,a,indranges) + + bI, chunk_compressed = take!(c) + + current_chunk_offsets = map((s, i)->s*(i-1), size(a), Tuple(bI)) + + indranges = map(boundint, r.indices, size(a), current_chunk_offsets) + + uncompress_to_output!(aout, output_base_offsets, z, chunk_compressed, current_chunk_offsets, a, indranges) nothing end finally close(c) end - + aout end -function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::CartesianIndices{N}) where {N} +function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::CartesianIndices{N}) where {N} z.writeable || error("Can not write to read-only ZArray") - input_base_offsets = map(i->first(i)-1,r.indices) + input_base_offsets = map(i->first(i)-1, r.indices) # Determines which chunks are affected - blockr = CartesianIndices(map(trans_ind, r.indices, z.metadata.chunks)) + blockr = CartesianIndices(map(DiskArrays.findchunk, z.metadata.chunks.chunks, r.indices)) # Fast path: single-chunk full-overwrite skips the readtask/writetask channels and the scratch buffer. bI = singlechunk_fastpath(ain, z, blockr) if bI !== nothing @@ -295,45 +291,45 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes a = z.metadata.fill_value === nothing ? getchunkarray(z) : getchunkarray_undef(z) # Now loop through the chunks readchannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) - - readtask = @async begin + + readtask = @async begin read_items!(z.storage, readchannel, z.metadata.chunk_key_encoding, z.path, blockr) end - bind(readchannel,readtask) + bind(readchannel, readtask) writechannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) writetask = @async begin write_items!(z.storage, writechannel, z.metadata.chunk_key_encoding, z.path, blockr) end - bind(writechannel,writetask) - - try + bind(writechannel, writetask) + + try for i in 1:length(blockr) - - bI,chunk_compressed = take!(readchannel) - - current_chunk_offsets = map((s,i)->s*(i-1),size(a),Tuple(bI)) - indranges = map(boundint,r.indices,size(a),current_chunk_offsets) + bI, chunk_compressed = take!(readchannel) + + current_chunk_offsets = map((s, i)->s*(i-1), size(a), Tuple(bI)) + + indranges = map(boundint, r.indices, size(a), current_chunk_offsets) if isnothing(chunk_compressed) || (length.(indranges) != size(a)) - resetbuffer!(z.metadata.fill_value,a) + resetbuffer!(z.metadata.fill_value, a) end curchunk = if length.(indranges) != size(a) - view(a,dotminus.(indranges,current_chunk_offsets)...) + view(a, dotminus.(indranges, current_chunk_offsets)...) else a end - + if chunk_compressed !== nothing - uncompress_raw!(a,z,chunk_compressed) + uncompress_raw!(a, z, chunk_compressed) end - curchunk .= view(ain,dotminus.(indranges,input_base_offsets)...) + curchunk .= view(ain, dotminus.(indranges, input_base_offsets)...) - put!(writechannel,bI=>compress_raw(maybeinner(a),z)) + put!(writechannel, bI=>compress_raw(maybeinner(a), z)) nothing end finally @@ -344,19 +340,19 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes ain end -DiskArrays.readblock!(a::ZArray,aout,i::AbstractUnitRange...) = readblock!(aout,a,CartesianIndices(i)) -DiskArrays.writeblock!(a::ZArray,v,i::AbstractUnitRange...) = writeblock!(v,a,CartesianIndices(i)) +DiskArrays.readblock!(a::ZArray, aout, i::AbstractUnitRange...) = readblock!(aout, a, CartesianIndices(i)) +DiskArrays.writeblock!(a::ZArray, v, i::AbstractUnitRange...) = writeblock!(v, a, CartesianIndices(i)) DiskArrays.haschunks(::ZArray) = DiskArrays.Chunked() -DiskArrays.eachchunk(a::ZArray) = DiskArrays.GridChunks(a,a.metadata.chunks) +DiskArrays.eachchunk(a::ZArray) = a.metadata.chunks """ uncompress_raw!(a::DenseArray{T},z::ZArray{T,N},i::CartesianIndex{N}) Read the chunk specified by `i` from the Zarray `z` and write its content to `a` """ -function uncompress_raw!(a,z::ZArray{<:Any,N},curchunk) where N +function uncompress_raw!(a, z::ZArray{<:Any,N}, curchunk) where N if curchunk === nothing - if isnothing(z.metadata.fill_value) + if isnothing(z.metadata.fill_value) throw(ArgumentError("The array $z got missing chunks and no fill_value")) end fill!(a, z.metadata.fill_value) @@ -366,22 +362,22 @@ function uncompress_raw!(a,z::ZArray{<:Any,N},curchunk) where N a end -dotminus(x,y) = x.-y +dotminus(x, y) = x .- y + +function uncompress_to_output!(aout, output_base_offsets, z, chunk_compressed, current_chunk_offsets, a, indranges) + + uncompress_raw!(a, z, chunk_compressed) -function uncompress_to_output!(aout,output_base_offsets,z,chunk_compressed,current_chunk_offsets,a,indranges) - - uncompress_raw!(a,z,chunk_compressed) - if length.(indranges) == size(a) aout[dotminus.(indranges, output_base_offsets)...] = ndims(a) == 0 ? a[1] : a else - curchunk = a[dotminus.(indranges,current_chunk_offsets)...] + curchunk = a[dotminus.(indranges, current_chunk_offsets)...] aout[dotminus.(indranges, output_base_offsets)...] = curchunk end end -function compress_raw(a,z) - length(a) == prod(z.metadata.chunks) || throw(DimensionMismatch("Array size does not equal chunk size")) +function compress_raw(a, z) + length(a) == prod(DiskArrays.max_chunksize.(z.metadata.chunks.chunks)) || throw(DimensionMismatch("Array size does not equal chunk size")) pipeline_encode(get_pipeline(z.metadata), a, z.metadata.fill_value) end @@ -412,7 +408,7 @@ function zcreate(::Type{T}, dims::Integer...; zarr_format=DV, dimension_separator=default_sep(zarr_format), kwargs... - ) where T +) where T if path===nothing store = DictStore() @@ -423,26 +419,26 @@ function zcreate(::Type{T}, dims::Integer...; end struct ShapeOnlyArray{T,N} <: AbstractArray{T,N} - sz::Dims{N} + sz::Dims{N} end Base.size(a::ShapeOnlyArray) = a.sz Base.getindex(::ShapeOnlyArray, ::Vararg{Any}) = - error("ShapeOnlyArray carries no data") + error("ShapeOnlyArray carries no data") -function zcreate(::Type{T},storage::AbstractStore, +function zcreate(::Type{T}, storage::AbstractStore, dims...; - path = "", - zarr_format = DV, + path="", + zarr_format=DV, chunks=dims, fill_value=nothing, fill_as_missing=false, compressor=BloscCompressor(), - filters = filterfromtype(T), + filters=filterfromtype(T), attrs=Dict(), writeable=true, indent_json=false, dimension_separator=nothing - ) where {T} +) where {T} v = ZarrFormat(zarr_format) if isnothing(dimension_separator) @@ -453,31 +449,37 @@ function zcreate(::Type{T},storage::AbstractStore, dimension_separator = only(dimension_separator) end chunk_key_encoding = ChunkKeyEncoding(dimension_separator, default_prefix(v)) - - length(dims) == length(chunks) || throw(DimensionMismatch("Dims must have the same length as chunks")) N = length(dims) + + if chunks isa Tuple + length(dims) == length(chunks) || throw(DimensionMismatch("Dims must have the same length as chunks")) + elseif chunks isa GridChunks + length(dims) == ndims(chunks) || throw(DimensionMismatch("Dims must have the same length as chunks")) + else + throw(ArgumentError("chunks must be provided either as a Tuple of Ints or as a DiskArrays.GridChunks object")) + end C = typeof(compressor) - + # Create a dummy array to use with Metadata constructor # This allows us to leverage the multiple dispatch in Metadata constructors dummy_array = ShapeOnlyArray{T,N}(dims) metadata = Metadata(dummy_array, chunks, v; - compressor=compressor, - fill_value=fill_value, - filters=filters, - fill_as_missing=fill_as_missing, + compressor=compressor, + fill_value=fill_value, + filters=filters, + fill_as_missing=fill_as_missing, chunk_key_encoding=chunk_key_encoding ) - + # Extract the element type from the metadata (handles T2 calculation) T2 = eltype(metadata) - - isemptysub(storage,path) || error("$storage $path is not empty") - + + isemptysub(storage, path) || error("$storage $path is not empty") + writemetadata(v, storage, path, metadata, indent_json=indent_json) - + writeattrs(v, storage, path, attrs, indent_json=indent_json) - + ZArray(metadata, storage, path, attrs, writeable) end @@ -488,8 +490,8 @@ function filterfromtype(::Type{<:AbstractArray{T}}) where T (VLenArrayFilter{T}(),) end -filterfromtype(::Type{<:Union{<:AbstractString, Union{<:AbstractString, Missing}}}) = (VLenUTF8Filter(),) -filterfromtype(::Type{<:Union{MaxLengthString, Union{MaxLengthString, Missing}}}) = nothing +filterfromtype(::Type{<:Union{<:AbstractString,Union{<:AbstractString,Missing}}}) = (VLenUTF8Filter(),) +filterfromtype(::Type{<:Union{MaxLengthString,Union{MaxLengthString,Missing}}}) = nothing #Not all Array types can be mapped directly to a valid ZArray encoding. #Here we try to determine the correct element type @@ -515,16 +517,17 @@ chunkindices(z::ZArray) = CartesianIndices(map((s, c) -> 1:ceil(Int, s/c), z.met Creates a zarr array and initializes all values with zero. Accepts the same keyword arguments as `zcreate` """ -function zzeros(T,dims...;kwargs...) - z = zcreate(T,dims...;kwargs...) - as = zeros(T, z.metadata.chunks...) - data_encoded = compress_raw(as,z) - p = z.path - if data_encoded !== nothing - for i in chunkindices(z) - store_writechunk(z.storage, data_encoded, p, i, z.metadata.chunk_key_encoding) - end - end +function zzeros(T, dims...; kwargs...) + z = zcreate(T, dims...; kwargs...) + # as = zeros(T, DiskArrays.max_chunksize.(z.metadata.chunks.chunks)) + # data_encoded = compress_raw(as, z) + # p = z.path + # if data_encoded !== nothing + # for i in chunkindices(z) + # store_writechunk(z.storage, data_encoded, p, i, z.metadata.chunk_key_encoding) + # end + # end + z .= zero(T) z end @@ -540,13 +543,13 @@ function Base.resize!(z::ZArray{T,N}, newsize::NTuple{N}) where {T,N} oldsize = z.metadata.shape[] z.metadata.shape[] = newsize #Check if array was shrunk - if any(map(<,newsize, oldsize)) + if any(map(<, newsize, oldsize)) prune_oob_chunks(z.storage, z.path, oldsize, newsize, z.metadata.chunks, z.metadata.chunk_key_encoding) end writemetadata(zarr_format(z), z.storage, z.path, z.metadata) nothing end -Base.resize!(z::ZArray, newsize::Integer...) = resize!(z,newsize) +Base.resize!(z::ZArray, newsize::Integer...) = resize!(z, newsize) """ append!(z::ZArray{<:Any, N},a;dims = N) @@ -563,12 +566,12 @@ append!(z,ones(Int,6,2)) #Add two new columns z[:,:] ```` """ -function Base.append!(z::ZArray{<:Any, N},a;dims = N) where N +function Base.append!(z::ZArray{<:Any,N}, a; dims=N) where N #Determine how many entries to add to axis - otherdims = sort!(setdiff(1:N,dims)) + otherdims = sort!(setdiff(1:N, dims)) othersize = size(z)[otherdims] if ndims(a)==N - nadd = size(a,dims) + nadd = size(a, dims) size(a)[otherdims]==othersize || throw(DimensionMismatch("Array to append does not have the correct size, expected: $(othersize)")) elseif ndims(a)==N-1 size(a)==othersize || throw(DimensionMismatch("Array to append does not have the correct size, expected: $(othersize)")) @@ -578,18 +581,18 @@ function Base.append!(z::ZArray{<:Any, N},a;dims = N) where N end oldsize = size(z) newsize = ntuple(i->i==dims ? oldsize[i]+nadd : oldsize[i], N) - resize!(z,newsize) - appendinds = ntuple(i->i==dims ? (oldsize[i]+1:newsize[i]) : Colon(),N) + resize!(z, newsize) + appendinds = ntuple(i->i==dims ? ((oldsize[i]+1):newsize[i]) : Colon(), N) z[appendinds...] = a nothing end function prune_oob_chunks(s::AbstractStore, path, oldsize, newsize, chunks, chunk_key_encoding) - dimstoshorten = findall(map(<,newsize, oldsize)) + dimstoshorten = findall(map(<, newsize, oldsize)) for idim in dimstoshorten - delrange = (fld1(newsize[idim],chunks[idim])+1):(fld1(oldsize[idim],chunks[idim])) - allchunkranges = map(i->1:fld1(oldsize[i],chunks[i]),1:length(oldsize)) - r = (allchunkranges[1:idim-1]..., delrange, allchunkranges[idim+1:end]...) + delrange = (fld1(newsize[idim], chunks[idim])+1):(fld1(oldsize[idim], chunks[idim])) + allchunkranges = map(i->1:fld1(oldsize[i], chunks[i]), 1:length(oldsize)) + r = (allchunkranges[1:(idim-1)]..., delrange, allchunkranges[(idim+1):end]...) for cI in CartesianIndices(r) store_deletechunk(s, path, cI, chunk_key_encoding) end diff --git a/lib/ZarrCore/src/metadata.jl b/lib/ZarrCore/src/metadata.jl index 2e17326a..a3afc310 100644 --- a/lib/ZarrCore/src/metadata.jl +++ b/lib/ZarrCore/src/metadata.jl @@ -1,5 +1,5 @@ import Dates: Date, DateTime -using DateTimes64: DateTime64, pydatetime_string, datetime_from_pystring +using DateTimes64: DateTime64, pydatetime_string, datetime_from_pystring """NumPy array protocol type string (typestr) format @@ -22,7 +22,7 @@ Base.codepoint(x::ASCIIChar) = UInt8(x) Base.show(io::IO, x::ASCIIChar) = print(io, Char(x)) Base.zero(::Union{ASCIIChar,Type{ASCIIChar}}) = ASCIIChar(Base.zero(UInt8)) -Base.zero(t::Union{String, Type{String}}) = "" +Base.zero(t::Union{String,Type{String}}) = "" typestr(t::Type) = string('<', 'V', sizeof(t)) typestr(t::Type{>:Missing}) = typestr(Base.nonmissingtype(t)) @@ -40,7 +40,7 @@ typestr(t::Type{<:DateTime64}) = pydatetime_string(t) typestr(::Type{<:AbstractString}) = "|O" const typestr_regex = r"^([<|>])([tbiufcmMOSUV])(\d*)(\[\w+\])?$" -const typemap = Dict{Tuple{Char, Int}, DataType}( +const typemap = Dict{Tuple{Char,Int},DataType}( ('b', 1) => Bool, ('S', 1) => ASCIIChar, ) @@ -49,10 +49,10 @@ typecharf(::Type{<:Signed}) = 'i' typecharf(::Type{<:Unsigned}) = 'u' typecharf(::Type{<:AbstractFloat}) = 'f' typecharf(::Type{<:Complex}) = 'c' -foreach([Float16,Float32,Float64,Int8,Int16,Int32,Int64,Int128, - UInt8,UInt16,UInt32,UInt64,UInt128, - Complex{Float16},Complex{Float32},Complex{Float64}]) do t - typemap[(typecharf(t),sizemapf(t))] = t +foreach([Float16, Float32, Float64, Int8, Int16, Int32, Int64, Int128, + UInt8, UInt16, UInt32, UInt64, UInt128, + Complex{Float16}, Complex{Float32}, Complex{Float64}]) do t + typemap[(typecharf(t), sizemapf(t))] = t end function typestr(s::AbstractString, filterlist=nothing) @@ -81,7 +81,7 @@ function typestr(s::AbstractString, filterlist=nothing) return datetime_from_pystring(s) end # convert typecode to Char and typesize to Int - typemap[(tc,ts)] + typemap[(tc, ts)] end end @@ -102,7 +102,7 @@ value of the ".zarray" key within an array store. https://zarr.readthedocs.io/en/stable/spec/v2.html#metadata """ -abstract type AbstractMetadata{T,N,E <: AbstractChunkKeyEncoding} end +abstract type AbstractMetadata{T,N,E<:AbstractChunkKeyEncoding} end Base.ndims(::AbstractMetadata{<:Any,N}) where N = N @@ -110,11 +110,11 @@ Base.ndims(::AbstractMetadata{<:Any,N}) where N = N struct MetadataV2{T,N,C,F} <: AbstractMetadata{T,N,ChunkKeyEncoding} zarr_format::Int node_type::String - shape::Base.RefValue{NTuple{N, Int}} - chunks::NTuple{N, Int} + shape::Base.RefValue{NTuple{N,Int}} + chunks::NTuple{N,Int} dtype::String # structured data types not yet supported compressor::C - fill_value::Union{T, Nothing} + fill_value::Union{T,Nothing} order::Char filters::F # not yet supported chunk_key_encoding::ChunkKeyEncoding @@ -134,29 +134,29 @@ const Metadata = AbstractMetadata #To make unit tests pass with ref shape function Base.:(==)(m1::MetadataV2, m2::MetadataV2) - m1.zarr_format == m2.zarr_format && - m1.node_type == m2.node_type && - m1.shape[] == m2.shape[] && - m1.chunks == m2.chunks && - m1.dtype == m2.dtype && - m1.compressor == m2.compressor && - m1.fill_value == m2.fill_value && - m1.order == m2.order && - m1.filters == m2.filters && - m1.chunk_key_encoding == m2.chunk_key_encoding + m1.zarr_format == m2.zarr_format && + m1.node_type == m2.node_type && + m1.shape[] == m2.shape[] && + m1.chunks == m2.chunks && + m1.dtype == m2.dtype && + m1.compressor == m2.compressor && + m1.fill_value == m2.fill_value && + m1.order == m2.order && + m1.filters == m2.filters && + m1.chunk_key_encoding == m2.chunk_key_encoding end "Construct Metadata based on your data" -function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, zarr_format=DV; - node_type::String="array", - compressor::C=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - filters=nothing, - fill_as_missing = false, - dimension_separator::Char = '.' - ) where {T, N, C} +function Metadata(A::AbstractArray{T,N}, chunks, zarr_format=DV; + node_type::String="array", + compressor::C=BloscCompressor(), + fill_value::Union{T,Nothing}=nothing, + order::Char=('C'), + filters=nothing, + fill_as_missing=false, + dimension_separator::Char=('.') +) where {T,N,C} return Metadata(A, chunks, ZarrFormat(zarr_format); node_type=node_type, compressor=compressor, @@ -170,14 +170,14 @@ end # V2 constructor function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{2}; - node_type::String="array", - compressor::C=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - filters::F=nothing, - fill_as_missing = false, + node_type::String="array", + compressor::C=BloscCompressor(), + fill_value::Union{T,Nothing}=nothing, + order::Char=('C'), + filters::F=nothing, + fill_as_missing=false, chunk_key_encoding=ChunkKeyEncoding('.', false) - ) where {T, N, C, F} +) where {T,N,C,F} T2 = (fill_value === nothing || !fill_as_missing) ? T : Union{T,Missing} MetadataV2{T2,N,C,typeof(filters)}( 2, @@ -193,7 +193,7 @@ function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{2}; ) end -Metadata(s::Union{AbstractString, IO}, fill_as_missing) = Metadata(JSON.parse(s; dicttype=Dict{String,Any}), fill_as_missing) +Metadata(s::Union{AbstractString,IO}, fill_as_missing) = Metadata(JSON.parse(s; dicttype=Dict{String,Any}), fill_as_missing) "Construct Metadata from Dict" function Metadata(d::AbstractDict, fill_as_missing) @@ -232,8 +232,8 @@ function Metadata(d::AbstractDict, fill_as_missing, ::ZarrFormat{2}) MetadataV2{TU,N,C,F}( d["zarr_format"], node_type, - NTuple{N, Int}(d["shape"]) |> reverse, - NTuple{N, Int}(d["chunks"]) |> reverse, + NTuple{N,Int}(d["shape"]) |> reverse, + NTuple{N,Int}(d["chunks"]) |> reverse, d["dtype"], compressor, fv, @@ -246,7 +246,7 @@ end "Describes how to lower Metadata to JSON, used in json(::Metadata)" function JSON.lower(md::MetadataV2) - Dict{String, Any}( + Dict{String,Any}( "zarr_format" => Int(md.zarr_format), "node_type" => md.node_type, "shape" => md.shape[] |> reverse, @@ -265,7 +265,7 @@ end # https://zarr.readthedocs.io/en/stable/spec/v2.html#fill-value-encoding fill_value_encoding(v) = v -fill_value_encoding(::Nothing)=nothing +fill_value_encoding(::Nothing) = nothing function fill_value_encoding(v::AbstractFloat) if isnan(v) "NaN" diff --git a/lib/ZarrCore/src/metadata3.jl b/lib/ZarrCore/src/metadata3.jl index 9101e977..5e49676b 100644 --- a/lib/ZarrCore/src/metadata3.jl +++ b/lib/ZarrCore/src/metadata3.jl @@ -1,8 +1,10 @@ +using DiskArrays: GridChunks, RegularChunks, IrregularChunks + """ Prototype Zarr version 3 support """ -const typemap3 = Dict{String, DataType}() +const typemap3 = Dict{String,DataType}() foreach([Bool, Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64]) do t typemap3[lowercase(string(t))] = t end @@ -14,11 +16,11 @@ function typestr3(t::Type) return lowercase(string(t)) end -function typestr3(::Type{MaxLengthString{N, UInt32}}) where {N} - return Dict{String, Any}( +function typestr3(::Type{MaxLengthString{N,UInt32}}) where {N} + return Dict{String,Any}( "name" => "fixed_length_utf32", - "configuration" => Dict{String, Any}("length_bytes" => N * 4) - ) + "configuration" => Dict{String,Any}("length_bytes" => N * 4) + ) end # TODO: Check raw types @@ -49,7 +51,7 @@ function parse_datatype3(d) name = get(d, "name", nothing) if name == "fixed_length_utf32" - return MaxLengthString{d["configuration"]["length_bytes"] ÷ 4, UInt32} + return MaxLengthString{d["configuration"]["length_bytes"] ÷ 4,UInt32} end throw(ArgumentError("Unsupported Zarr v3 data_type: $d")) end @@ -63,21 +65,20 @@ function check_keys(d::AbstractDict, keys) end """Metadata for Zarr version 3 arrays""" -struct MetadataV3{T,N,P<:AbstractCodecPipeline,E<:AbstractChunkKeyEncoding} <: AbstractMetadata{T,N,E} +struct MetadataV3{T,N,P<:AbstractCodecPipeline,E<:AbstractChunkKeyEncoding,CT<:GridChunks{N}} <: AbstractMetadata{T,N,E} zarr_format::Int node_type::String - shape::Base.RefValue{NTuple{N, Int}} - chunks::NTuple{N, Int} - dtype::Union{String, Dict{String, Any}} # data_type in v3 + shape::Base.RefValue{NTuple{N,Int}} + chunks::CT + dtype::Union{String,Dict{String,Any}} # data_type in v3 pipeline::P - fill_value::Union{T, Nothing} + fill_value::Union{T,Nothing} chunk_key_encoding::E - function MetadataV3{T2,N,P,E}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) where {T2,N,P,E} + function MetadataV3{T2,N,P,E,CT}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) where {T2,N,P,E,CT} zarr_format == 3 || throw(ArgumentError("MetadataV3 only functions if zarr_format == 3")) #Do some sanity checks to make sure we have a sane array any(<(0), shape) && throw(ArgumentError("Size must be positive")) - any(<(1), chunks) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) - new{T2,N,P,E}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, pipeline, fill_value, chunk_key_encoding) + new{T2,N,P,E,CT}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, pipeline, fill_value, chunk_key_encoding) end end MetadataV3{T2,N,P}(args...) where {T2,N,P} = MetadataV3{T2,N,P,ChunkKeyEncoding}(args...) @@ -88,13 +89,13 @@ Convenience constructor for MetadataV3 that builds the codec pipeline from `order` (translated to a TransposeCodec), `endian` (translated to a BytesCodec), and `compressor` (translated to bytes->bytes codecs). """ -function MetadataV3{T2,N}(zarr_format, node_type, shape::NTuple{N,Int}, chunks::NTuple{N,Int}, - dtype, fill_value; - order::Char='C', - endian::Symbol=:little, - compressor=BloscCompressor(), - chunk_key_encoding::E=ChunkKeyEncoding('/', true) - ) where {T2, N, E} +function MetadataV3{T2,N}(zarr_format, node_type, shape::NTuple{N,Int}, chunks, + dtype, fill_value; + order::Char=('C'), + endian::Symbol=:little, + compressor=BloscCompressor(), + chunk_key_encoding::E=ChunkKeyEncoding('/', true) +) where {T2,N,E} T_base = Base.nonmissingtype(T2) array_array_codecs = if order == 'F' (Codecs.V3Codecs.TransposeCodec(ntuple(i -> N - i + 1, N)),) @@ -108,6 +109,9 @@ function MetadataV3{T2,N}(zarr_format, node_type, shape::NTuple{N,Int}, chunks:: array_bytes_codec = Codecs.V3Codecs.BytesCodec(endian) typesize = sizeof(T_base) end + if chunks isa Tuple + chunks = GridChunks(size(A), chunks) + end bytes_bytes_codecs = if compressor isa NoCompressor () elseif compressor isa BloscCompressor @@ -122,18 +126,18 @@ function MetadataV3{T2,N}(zarr_format, node_type, shape::NTuple{N,Int}, chunks:: throw(ArgumentError("Unsupported compressor type for v3: $(typeof(compressor))")) end pipeline = V3Pipeline(array_array_codecs, array_bytes_codec, bytes_bytes_codecs) - return MetadataV3{T2,N,typeof(pipeline),E}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) + return MetadataV3{T2,N,typeof(pipeline),E,typeof(chunks)}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) end function Base.:(==)(m1::MetadataV3, m2::MetadataV3) - m1.zarr_format == m2.zarr_format && - m1.node_type == m2.node_type && - m1.shape[] == m2.shape[] && - m1.chunks == m2.chunks && - m1.dtype == m2.dtype && - m1.fill_value == m2.fill_value && - m1.pipeline == m2.pipeline && - m1.chunk_key_encoding == m2.chunk_key_encoding + m1.zarr_format == m2.zarr_format && + m1.node_type == m2.node_type && + m1.shape[] == m2.shape[] && + m1.chunks == m2.chunks && + m1.dtype == m2.dtype && + m1.fill_value == m2.fill_value && + m1.pipeline == m2.pipeline && + m1.chunk_key_encoding == m2.chunk_key_encoding end """ @@ -164,8 +168,8 @@ function get_order(md::MetadataV3) )) end N = ndims(md) - c_perm = ntuple(identity, N) - f_perm = ntuple(i -> N - i + 1, N) + c_perm = ntuple(identity, N) + f_perm = ntuple(i -> N - i + 1, N) if codec.order == c_perm return 'C' elseif codec.order == f_perm @@ -246,6 +250,28 @@ function Metadata3(d::AbstractDict, fill_as_missing) if length(shape) != length(chunks) throw(ArgumentError("Shape has rank $(length(shape)) which does not match the chunk_shape rank of $(length(chunks))")) end + elseif chunk_grid["name"] == "rectilinear" + chunk_grid["kind"] == "inline" || throw(ArgumentError("Only chunk grid descriptors of kind \"inline\" are allowed")) + cshapes = chunk_grid["chunk_shapes"] + chunkspecs = map(shapes, shape) do csh, s + if shape isa Integer + RegularChunks(csh, 0, s) + else + chunksizes = Int[] + for spec in csh + if spec isa Int + push!(chunksizes, spec) + else + value, n = spec + for _ in 1:n + push!(chunksizes, value) + end + end + end + IrregularChunks(; chunksizes) + end + end + chunks = ChunkGrid(reverse(chunkspecs)) else throw(ArgumentError("Unknown chunk_grid of name, $(chunk_grid["name"])")) end @@ -257,7 +283,7 @@ function Metadata3(d::AbstractDict, fill_as_missing) T = typestr3(data_type) N = length(shape) - codec_ctx = (shape = shape, elsize = _sizeof(Base.nonmissingtype(T))) + codec_ctx = (shape=shape, elsize=_sizeof(Base.nonmissingtype(T))) pipeline = Codecs.V3Codecs.getCodec(d["codecs"], codec_ctx) fv = fill_value_decoding(d["fill_value"], T)::T @@ -267,11 +293,11 @@ function Metadata3(d::AbstractDict, fill_as_missing) chunk_key_encoding = parse_chunk_key_encoding(chunk_key_encoding) E = typeof(chunk_key_encoding) - MetadataV3{TU, N, typeof(pipeline), E}( + MetadataV3{TU,N,typeof(pipeline),E}( zarr_format, node_type, - NTuple{N, Int}(shape) |> reverse, - NTuple{N, Int}(chunks) |> reverse, + NTuple{N,Int}(shape) |> reverse, + chunks, typestr3(T), pipeline, fv, @@ -280,21 +306,24 @@ function Metadata3(d::AbstractDict, fill_as_missing) end "Construct MetadataV3 based on your data" -function Metadata3(A::AbstractArray{T, N}, chunks::NTuple{N, Int}; - node_type::String="array", - compressor=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - endian::Symbol=:little, - filters=nothing, - fill_as_missing = false, - dimension_separator::Char = '/' - ) where {T, N} +function Metadata3(A::AbstractArray{T,N}, chunks; + node_type::String="array", + compressor=BloscCompressor(), + fill_value::Union{T,Nothing}=nothing, + order::Char=('C'), + endian::Symbol=:little, + filters=nothing, + fill_as_missing=false, + dimension_separator::Char=('/') +) where {T,N} T2 = (fill_value === nothing || !fill_as_missing) ? T : Union{T,Missing} if fill_value === nothing fill_value = zero(T) end - return MetadataV3{T2, N}( + if chunks isa Tuple + chunks = GridChunks(size(A), chunks) + end + return MetadataV3{T2,N}( 3, node_type, size(A), @@ -309,19 +338,38 @@ function Metadata3(A::AbstractArray{T, N}, chunks::NTuple{N, Int}; end function lower3(md::MetadataV3{T}) where T - chunk_grid = Dict{String,Any}( - "name" => "regular", - "configuration" => Dict{String,Any}( - "chunk_shape" => md.chunks |> reverse + + + chunk_grid = if any(i->isa(i, IrregularChunks), md.chunks.chunks) + Dict{String,Any}( + "name" => "rectilinear", + "configuration" => Dict{String,Any}( + "kind" => "inline", + "chunk_shapes" => map(md.chunks.chunks) do c + if c isa RegularChunks + c.chunksize + else + length.(c) + end + end + ) ) - ) + else + Dict{String,Any}( + "name" => "regular", + "configuration" => Dict{String,Any}( + "chunk_shape" => md.chunks |> reverse + ) + ) + end + # chunk_key_encoding chunk_key_encoding = lower_chunk_key_encoding(md.chunk_key_encoding) codecs = Codecs.V3Codecs._pipeline_to_codec_list(md.pipeline) - Dict{String, Any}( + Dict{String,Any}( "zarr_format" => Int(md.zarr_format), "node_type" => md.node_type, "shape" => md.shape[] |> reverse, @@ -333,16 +381,16 @@ function lower3(md::MetadataV3{T}) where T ) end -function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{3}; - node_type::String="array", - compressor::C=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - endian::Symbol=:little, - filters::F=nothing, - fill_as_missing = false, - chunk_key_encoding::E=ChunkKeyEncoding('/', true) - ) where {T, N, C, F, E} +function Metadata(A::AbstractArray{T,N}, chunks, ::ZarrFormat{3}; + node_type::String="array", + compressor::C=BloscCompressor(), + fill_value::Union{T,Nothing}=nothing, + order::Char=('C'), + endian::Symbol=:little, + filters::F=nothing, + fill_as_missing=false, + chunk_key_encoding::E=ChunkKeyEncoding('/', true) +) where {T,N,C,F,E} return Metadata3(A, chunks; node_type=node_type, compressor=compressor, From 30fe3c43211548a7bc449104fe7d86d20067907c Mon Sep 17 00:00:00 2001 From: Felix Cremer Date: Tue, 18 Aug 2026 11:34:24 +0200 Subject: [PATCH 20/39] Remove `@public` macro and split user and dev public names The user facing interface is public in Zarr while the developer facing interface is only public in ZarrCore. --- lib/ZarrCore/src/ZarrCore.jl | 82 +-------------------------- lib/ZarrCore/src/public_names_core.jl | 39 +++++++++++++ src/Zarr.jl | 16 +++--- src/public_names_zarr.jl | 23 ++++++++ 4 files changed, 73 insertions(+), 87 deletions(-) create mode 100644 lib/ZarrCore/src/public_names_core.jl create mode 100644 src/public_names_zarr.jl diff --git a/lib/ZarrCore/src/ZarrCore.jl b/lib/ZarrCore/src/ZarrCore.jl index 402b02f1..84b13319 100644 --- a/lib/ZarrCore/src/ZarrCore.jl +++ b/lib/ZarrCore/src/ZarrCore.jl @@ -5,46 +5,6 @@ import Blosc import Unicode using OrderedCollections: OrderedDict -""" - PUBLIC_NAMES::Vector{Symbol} - -Every name this module declares with [`@public`](@ref), in declaration order. - -On Julia 1.11+ this duplicates what `names(ZarrCore)` already reports, but on -1.10 the `public` keyword does not exist, so this registry is the only record -of the public-but-not-exported API. The `Zarr` facade uses it to re-export the -same set of names on every supported Julia version. - -Like `public` itself, this is per-module: every module that uses `@public` owns -a `PUBLIC_NAMES`, and a submodule's public names live in *its* list, not here. -""" -const PUBLIC_NAMES = Symbol[] - -""" - @public name - @public name1, name2, ... - -Mark `name`s as public API without exporting them, i.e. `Base.ispublic` returns -`true` and they show up in `names(ZarrCore)`, but `using ZarrCore` does not -bring them into scope. - -This is the `public` keyword, which only exists from Julia 1.11 onwards. On -Julia 1.10 the declaration itself expands to nothing, so the names are recorded -in the calling module's [`PUBLIC_NAMES`](@ref) as well -- `names()` cannot -report them there. A module using `@public` must therefore define its own -`const PUBLIC_NAMES = Symbol[]`. -""" -macro public(ex) - syms = ex isa Symbol ? [ex] : ex.args - all(s -> s isa Symbol, syms) || - throw(ArgumentError("@public expects one or more symbols, got $ex")) - decl = VERSION >= v"1.11" ? esc(Expr(:public, syms...)) : nothing - quote - append!($(esc(:PUBLIC_NAMES)), $(QuoteNode(syms))) - $decl - end -end - struct ZarrFormat{V} version::Val{V} end @@ -88,43 +48,7 @@ export ZArray, ZGroup, zopen, zzeros, zcreate, zgroup, zarrcache, storagesize, storageratio, zinfo, DirectoryStore, S3Store, GCStore -@public zname, zopen_noerr - -# Stores. Every store type is public; `DirectoryStore`, `S3Store` and `GCStore` -# are additionally exported for backwards compatibility. -@public AbstractStore, DictStore, HTTPStore, ZipStore, CachingStore, - ConsolidatedStore, PermanentZarrCache -@public consolidate_metadata, writezip - -# The interface a new store backend has to implement, see `?AbstractStore`. -# (`storagesize` is part of it too, but is exported above.) -@public subdirs, subkeys, isinitialized, storefromstring, - store_read_strategy, SequentialRead, ConcurrentRead, storageregexlist, - cloud_list_objects, concurrent_io_tasks - -# Chunk key encodings and the registry used to add new ones. -@public AbstractChunkKeyEncoding, ChunkKeyEncoding, SuffixChunkKeyEncoding, - citostring, register_chunk_key_encoding, parse_chunk_key_encoding, - lower_chunk_key_encoding - -# Filters and the interface a new filter has to implement, see `?Filter`. -@public Filter, VLenArrayFilter, VLenUTF8Filter, Fletcher32Filter, - FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter -@public zencode, zdecode, getfilter, sourcetype, desttype, filterdict - -# Compressors and the interface a new compressor has to implement. -@public Compressor, NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor -@public zcompress, zcompress!, zuncompress, zuncompress!, getCompressor, - compressortypes - -# v3 codecs and the interface a new codec has to implement, see `?Codec`. -@public Codecs, Codec, V3Codec, BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, - ShardingCodec, TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, - CRC32cV3Codec, VLenUTF8V3Codec - -# Data type and fill value encoding, needed to map Zarr dtypes to Julia types. -# `DateTime64` is re-exported from DateTimes64.jl because it shows up as the -# `eltype` of datetime arrays. -@public typestr, fill_value_encoding, fill_value_decoding, DateTime64 - +@static if VERSION >= v"1.11" + include("public_names.jl") + end end # module diff --git a/lib/ZarrCore/src/public_names_core.jl b/lib/ZarrCore/src/public_names_core.jl new file mode 100644 index 00000000..538ee1e6 --- /dev/null +++ b/lib/ZarrCore/src/public_names_core.jl @@ -0,0 +1,39 @@ +public zname, zopen_noerr + +# Stores. Every store type is public; `DirectoryStore`, `S3Store` and `GCStore` +# are additionally exported for backwards compatibility. +public AbstractStore, DictStore, HTTPStore, ZipStore, CachingStore, + ConsolidatedStore + +public consolidate_metadata, writezip + +# The interface a new store backend has to implement, see `?AbstractStore`. +# (`storagesize` is part of it too, but is exported above.) +public subdirs, subkeys, isinitialized, storefromstring, + store_read_strategy, SequentialRead, ConcurrentRead, storageregexlist, + cloud_list_objects, concurrent_io_tasks + +# Chunk key encodings and the registry used to add new ones. +public AbstractChunkKeyEncoding, ChunkKeyEncoding, SuffixChunkKeyEncoding, + citostring, register_chunk_key_encoding, parse_chunk_key_encoding, + lower_chunk_key_encoding + +# Filters and the interface a new filter has to implement, see `?Filter`. +public Filter, VLenArrayFilter, VLenUTF8Filter, Fletcher32Filter, + FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter +public zencode, zdecode, getfilter, sourcetype, desttype, filterdict + +# Compressors and the interface a new compressor has to implement. +public Compressor, NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor +public zcompress, zcompress!, zuncompress, zuncompress!, getCompressor, + compressortypes + +# v3 codecs and the interface a new codec has to implement, see `?Codec`. +public Codecs, Codec, V3Codec, BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, + ShardingCodec, TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, + CRC32cV3Codec, VLenUTF8V3Codec + +# Data type and fill value encoding, needed to map Zarr dtypes to Julia types. +# `DateTime64` is re-exported from DateTimes64.jl because it shows up as the +# `eltype` of datetime arrays. +public typestr, fill_value_encoding, fill_value_decoding \ No newline at end of file diff --git a/src/Zarr.jl b/src/Zarr.jl index 084150e5..50edefc5 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -9,17 +9,17 @@ import ZarrCore # name in `PUBLIC_NAMES`. Taking the union keeps the surface identical on every # supported version; without it, every public-but-not-exported name would be # missing from `Zarr` on LTS. -for name in union(names(ZarrCore; all = false), ZarrCore.PUBLIC_NAMES) + +for name in names(ZarrCore) if name !== :ZarrCore @eval import ZarrCore: $name - end - if Base.isexported(ZarrCore, name) && name !== :ZarrCore - @eval export $name - else - @static if VERSION >= v"1.11" - Core.eval(@__MODULE__, Expr(:public, name)) + + if Base.isexported(ZarrCore, name) + @eval export $name end end end - +@static if VERSION >= v"1.11" + include("public_names_zarr.jl") +end end diff --git a/src/public_names_zarr.jl b/src/public_names_zarr.jl new file mode 100644 index 00000000..c5bfc63a --- /dev/null +++ b/src/public_names_zarr.jl @@ -0,0 +1,23 @@ +public zname + +# Stores. Every store type is public; `DirectoryStore`, `S3Store` and `GCStore` +# are additionally exported for backwards compatibility. +public DictStore, HTTPStore, ZipStore, CachingStore, + ConsolidatedStore + +public consolidate_metadata, writezip + +# Chunk key encodings and the registry used to add new ones. +public ChunkKeyEncoding, SuffixChunkKeyEncoding, + +# Filters and the interface a new filter has to implement, see `?Filter`. +public Filter, VLenArrayFilter, VLenUTF8Filter, Fletcher32Filter, + FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter + +# Compressors and the interface a new compressor has to implement. +public Compressor, NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor + +# v3 codecs and the interface a new codec has to implement, see `?Codec`. +public Codecs, Codec, V3Codec, BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, + ShardingCodec, TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, + CRC32cV3Codec, VLenUTF8V3Codec \ No newline at end of file From 62af4bbce330daee96ac8c9505c5dd4191a05ad2 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 18 Aug 2026 06:17:49 -0400 Subject: [PATCH 21/39] Fix public --- Project.toml | 3 --- lib/ZarrCore/src/Codecs/Codecs.jl | 6 +++--- lib/ZarrCore/src/Codecs/V3/V3.jl | 12 +++--------- lib/ZarrCore/src/Codecs/V3/public_names_v3.jl | 0 lib/ZarrCore/src/Codecs/public_names_codecs.jl | 1 + lib/ZarrCore/src/ZarrCore.jl | 2 +- src/public_names_zarr.jl | 2 +- 7 files changed, 9 insertions(+), 17 deletions(-) create mode 100644 lib/ZarrCore/src/Codecs/V3/public_names_v3.jl create mode 100644 lib/ZarrCore/src/Codecs/public_names_codecs.jl diff --git a/Project.toml b/Project.toml index 60cdfb53..b1c1ec14 100644 --- a/Project.toml +++ b/Project.toml @@ -3,9 +3,6 @@ uuid = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" version = "0.10.1" authors = ["Fabian Gans "] -[sources] -ZarrCore = { path = "lib/ZarrCore" } - [workspace] projects = ["test", "lib/ZarrCore"] diff --git a/lib/ZarrCore/src/Codecs/Codecs.jl b/lib/ZarrCore/src/Codecs/Codecs.jl index 3c767162..2ed2b495 100644 --- a/lib/ZarrCore/src/Codecs/Codecs.jl +++ b/lib/ZarrCore/src/Codecs/Codecs.jl @@ -46,8 +46,8 @@ getCodec(::Type{<:Codec}, d::Dict) = error("Unimplemented") include("V3/V3.jl") -import ..ZarrCore: @public -const PUBLIC_NAMES = Symbol[] # `@public` records here; see `?ZarrCore.@public` -@public Codec, V3Codecs, zencode, zencode!, zdecode, zdecode!, getCodec +@static if VERSION ≥ v"1.11" + include("public_names_codecs.jl") +end end diff --git a/lib/ZarrCore/src/Codecs/V3/V3.jl b/lib/ZarrCore/src/Codecs/V3/V3.jl index b957e154..18417e36 100644 --- a/lib/ZarrCore/src/Codecs/V3/V3.jl +++ b/lib/ZarrCore/src/Codecs/V3/V3.jl @@ -823,14 +823,8 @@ function codec_decode(::VLenUTF8V3Codec, encoded::Vector{UInt8}, ::Type{T}, shap out end -# The codec types themselves are re-exported (as public names) from `ZarrCore`; -# these are the extension points for defining and registering new v3 codecs. -import ...ZarrCore: @public -const PUBLIC_NAMES = Symbol[] # `@public` records here; see `?ZarrCore.@public` -@public V3Codec, getCodec, register_codec, codec_parsers, codec_encode, - codec_decode, is_fixed_size, name -@public BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, ShardingCodec, - TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, CRC32cV3Codec, - VLenUTF8V3Codec +@static if VERSION ≥ v"1.11" + include("public_names_v3.jl") +end end diff --git a/lib/ZarrCore/src/Codecs/V3/public_names_v3.jl b/lib/ZarrCore/src/Codecs/V3/public_names_v3.jl new file mode 100644 index 00000000..e69de29b diff --git a/lib/ZarrCore/src/Codecs/public_names_codecs.jl b/lib/ZarrCore/src/Codecs/public_names_codecs.jl new file mode 100644 index 00000000..78297f8c --- /dev/null +++ b/lib/ZarrCore/src/Codecs/public_names_codecs.jl @@ -0,0 +1 @@ +public Codec, V3Codecs, zencode, zencode!, zdecode, zdecode!, getCodec diff --git a/lib/ZarrCore/src/ZarrCore.jl b/lib/ZarrCore/src/ZarrCore.jl index 84b13319..5fa78a5d 100644 --- a/lib/ZarrCore/src/ZarrCore.jl +++ b/lib/ZarrCore/src/ZarrCore.jl @@ -49,6 +49,6 @@ export ZArray, ZGroup, zopen, zzeros, zcreate, zgroup, zarrcache, DirectoryStore, S3Store, GCStore @static if VERSION >= v"1.11" - include("public_names.jl") + include("public_names_core.jl") end end # module diff --git a/src/public_names_zarr.jl b/src/public_names_zarr.jl index c5bfc63a..e955bd8e 100644 --- a/src/public_names_zarr.jl +++ b/src/public_names_zarr.jl @@ -8,7 +8,7 @@ public DictStore, HTTPStore, ZipStore, CachingStore, public consolidate_metadata, writezip # Chunk key encodings and the registry used to add new ones. -public ChunkKeyEncoding, SuffixChunkKeyEncoding, +public ChunkKeyEncoding, SuffixChunkKeyEncoding # Filters and the interface a new filter has to implement, see `?Filter`. public Filter, VLenArrayFilter, VLenUTF8Filter, Fletcher32Filter, From 9f4bb983422812e1451f14816cc9de24fc750d6e Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 18 Aug 2026 07:47:14 -0400 Subject: [PATCH 22/39] Import "public" symbols in Zarr from ZarrCore for 1.10 --- lib/ZarrCore/src/Codecs/V3/public_names_v3.jl | 7 +++++++ src/Zarr.jl | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/lib/ZarrCore/src/Codecs/V3/public_names_v3.jl b/lib/ZarrCore/src/Codecs/V3/public_names_v3.jl index e69de29b..a685bebc 100644 --- a/lib/ZarrCore/src/Codecs/V3/public_names_v3.jl +++ b/lib/ZarrCore/src/Codecs/V3/public_names_v3.jl @@ -0,0 +1,7 @@ +# The codec types themselves are re-exported (as public names) from `ZarrCore`; +# these are the extension points for defining and registering new v3 codecs. +public V3Codec, getCodec, register_codec, codec_parsers, codec_encode, + codec_decode, is_fixed_size, name +public BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, ShardingCodec, + TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, CRC32cV3Codec, + VLenUTF8V3Codec diff --git a/src/Zarr.jl b/src/Zarr.jl index 50edefc5..26158f31 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -19,7 +19,16 @@ for name in names(ZarrCore) end end end + @static if VERSION >= v"1.11" include("public_names_zarr.jl") +else + # For Julia 1.10, we have to parse the public names from the source file, since + # `public` is not a keyword and `names(ZarrCore)` only returns exported names. + let public_names = read(joinpath(@__DIR__, "public_names_zarr.jl"), String) + public_names = replace(public_names, "public" => "using ZarrCore: ") + eval(Meta.parseall(public_names)) + end end + end From b7b86c26d0cc5e7edc5500c5a03827ec4537684e Mon Sep 17 00:00:00 2001 From: Fabian Gans Date: Tue, 18 Aug 2026 14:56:15 +0200 Subject: [PATCH 23/39] Fixes towards passing tests --- lib/ZarrCore/src/ZArray.jl | 20 ++++++++++++-------- lib/ZarrCore/src/metadata.jl | 29 ++++++++++++++++++----------- lib/ZarrCore/src/metadata3.jl | 31 ++++++++++++++++++------------- 3 files changed, 48 insertions(+), 32 deletions(-) diff --git a/lib/ZarrCore/src/ZArray.jl b/lib/ZarrCore/src/ZArray.jl index 543d0e99..74f29740 100644 --- a/lib/ZarrCore/src/ZArray.jl +++ b/lib/ZarrCore/src/ZArray.jl @@ -145,7 +145,7 @@ end function getchunkarray(z::ZArray{>:Missing}) # temporary workaround to use strings as data values - inner = fill(z.metadata.fill_value, z.metadata.chunks) + inner = fill(z.metadata.fill_value, DiskArrays.max_chunksize.(z.metadata.chunks.chunks)) a = SenMissArray(inner, z.metadata.fill_value) end _zero(T) = zero(T) @@ -180,7 +180,7 @@ resetbuffer!(_, a::SenMissArray) = fill!(a, missing) # coverage check is needed. function singlechunk_fastpath(arr, z::ZArray{T,N}, blockr::CartesianIndices{N}) where {T,N} arr isa Array{T,N} && !(Missing <: T) && - size(arr) == z.metadata.chunks && length(blockr) == 1 || return nothing + size(arr) == DiskArrays.max_chunksize.(z.metadata.chunks.chunks) && length(blockr) == 1 || return nothing return first(blockr) end @@ -210,7 +210,7 @@ end # fastpath guarantees `ain` is the caller's input array, not the shared # scratch buffer the multi-chunk path reuses across iterations. function write_singlechunk_fastpath!( - z::ZArray{T,N,<:AbstractStore,<:MetadataV2{T,N,NoCompressor,Nothing}}, + z::ZArray{T,N,<:AbstractStore,<:MetadataV2{T,N,NoCompressor,Nothing,<:DiskArrays.GridChunks{N}}}, ain::Array{T,N}, bI::CartesianIndex, ) where {T,N} fv = z.metadata.fill_value @@ -232,7 +232,6 @@ function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::Cartesi output_base_offsets = map(i->first(i)-1, r.indices) # Determines which chunks are affected blockr = CartesianIndices(map(DiskArrays.findchunk, z.metadata.chunks.chunks, r.indices)) - @show blockr # Fast path: single-chunk full-read decodes directly into `aout`, skipping the readtask channel and scratch buffer. bI = singlechunk_fastpath(aout, z, blockr) if bI !== nothing @@ -510,7 +509,7 @@ end Returns the Cartesian Indices of the chunks of a given ZArray """ -chunkindices(z::ZArray) = CartesianIndices(map((s, c) -> 1:ceil(Int, s/c), z.metadata.shape[], z.metadata.chunks)) +chunkindices(z::ZArray) = CartesianIndices(map(length, z.metadata.chunks.chunks)) """ zzeros(T, dims...; kwargs... ) @@ -587,11 +586,16 @@ function Base.append!(z::ZArray{<:Any,N}, a; dims=N) where N nothing end -function prune_oob_chunks(s::AbstractStore, path, oldsize, newsize, chunks, chunk_key_encoding) +function prune_oob_chunks(s::AbstractStore, path, oldsize, newsize, chunks::DiskArrays.GridChunks, chunk_key_encoding) + # Resizing/pruning is only well-defined for regular chunk grids. + any(c -> c isa DiskArrays.IrregularChunks, chunks.chunks) && + throw(ArgumentError("Resizing arrays with irregular chunk grids is not supported")) dimstoshorten = findall(map(<, newsize, oldsize)) for idim in dimstoshorten - delrange = (fld1(newsize[idim], chunks[idim])+1):(fld1(oldsize[idim], chunks[idim])) - allchunkranges = map(i->1:fld1(oldsize[i], chunks[i]), 1:length(oldsize)) + nchunks_new = DiskArrays.findchunk(chunks.chunks[idim], max(newsize[idim], 1)) + nchunks_old = DiskArrays.findchunk(chunks.chunks[idim], oldsize[idim]) + delrange = (nchunks_new + 1):nchunks_old + allchunkranges = map(i -> 1:DiskArrays.findchunk(chunks.chunks[i], oldsize[i]), 1:length(oldsize)) r = (allchunkranges[1:(idim-1)]..., delrange, allchunkranges[(idim+1):end]...) for cI in CartesianIndices(r) store_deletechunk(s, path, cI, chunk_key_encoding) diff --git a/lib/ZarrCore/src/metadata.jl b/lib/ZarrCore/src/metadata.jl index a3afc310..03832616 100644 --- a/lib/ZarrCore/src/metadata.jl +++ b/lib/ZarrCore/src/metadata.jl @@ -1,5 +1,6 @@ import Dates: Date, DateTime using DateTimes64: DateTime64, pydatetime_string, datetime_from_pystring +using DiskArrays: GridChunks """NumPy array protocol type string (typestr) format @@ -107,24 +108,24 @@ Base.ndims(::AbstractMetadata{<:Any,N}) where N = N """Metadata for Zarr version 2 arrays""" -struct MetadataV2{T,N,C,F} <: AbstractMetadata{T,N,ChunkKeyEncoding} +struct MetadataV2{T,N,C,F,CT<:GridChunks{N}} <: AbstractMetadata{T,N,ChunkKeyEncoding} zarr_format::Int node_type::String shape::Base.RefValue{NTuple{N,Int}} - chunks::NTuple{N,Int} + chunks::CT dtype::String # structured data types not yet supported compressor::C fill_value::Union{T,Nothing} order::Char filters::F # not yet supported chunk_key_encoding::ChunkKeyEncoding - function MetadataV2{T2,N,C,F}(zarr_format, node_type, shape, chunks, dtype, compressor, fill_value, order, filters, chunk_key_encoding) where {T2,N,C,F} + function MetadataV2{T2,N,C,F,CT}(zarr_format, node_type, shape, chunks::CT, dtype, compressor, fill_value, order, filters, chunk_key_encoding) where {T2,N,C,F,CT<:GridChunks{N}} zarr_format == 2 || throw(ArgumentError("MetadataV2 only functions if zarr_format == 2")) #Do some sanity checks to make sure we have a sane array any(<(0), shape) && throw(ArgumentError("Size must be positive")) - any(<(1), chunks) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) + any(<(1), DiskArrays.max_chunksize.(chunks.chunks)) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) order === 'C' || throw(ArgumentError("Currently only 'C' storage order is supported")) - new{T2,N,C,F}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, compressor, fill_value, order, filters, chunk_key_encoding) + new{T2,N,C,F,CT}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, compressor, fill_value, order, filters, chunk_key_encoding) end end zarr_format(::MetadataV2) = ZarrFormat(Val(2)) @@ -169,7 +170,7 @@ function Metadata(A::AbstractArray{T,N}, chunks, zarr_format=DV; end # V2 constructor -function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{2}; +function Metadata(A::AbstractArray{T,N}, chunks::Union{NTuple{N,Int},GridChunks{N}}, ::ZarrFormat{2}; node_type::String="array", compressor::C=BloscCompressor(), fill_value::Union{T,Nothing}=nothing, @@ -179,7 +180,10 @@ function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{2}; chunk_key_encoding=ChunkKeyEncoding('.', false) ) where {T,N,C,F} T2 = (fill_value === nothing || !fill_as_missing) ? T : Union{T,Missing} - MetadataV2{T2,N,C,typeof(filters)}( + if chunks isa NTuple + chunks = GridChunks(size(A), chunks) + end + MetadataV2{T2,N,C,typeof(filters),typeof(chunks)}( 2, node_type, size(A), @@ -229,11 +233,14 @@ function Metadata(d::AbstractDict, fill_as_missing, ::ZarrFormat{2}) dim_sep = only(get(d, "dimension_separator", '.')) - MetadataV2{TU,N,C,F}( + shape_jl = NTuple{N,Int}(d["shape"]) |> reverse + chunks_tuple = NTuple{N,Int}(d["chunks"]) |> reverse + chunks_jl = GridChunks(shape_jl, chunks_tuple) + MetadataV2{TU,N,C,F,typeof(chunks_jl)}( d["zarr_format"], node_type, - NTuple{N,Int}(d["shape"]) |> reverse, - NTuple{N,Int}(d["chunks"]) |> reverse, + shape_jl, + chunks_jl, d["dtype"], compressor, fv, @@ -250,7 +257,7 @@ function JSON.lower(md::MetadataV2) "zarr_format" => Int(md.zarr_format), "node_type" => md.node_type, "shape" => md.shape[] |> reverse, - "chunks" => md.chunks |> reverse, + "chunks" => reverse(DiskArrays.max_chunksize.(md.chunks.chunks)), "dtype" => md.dtype, "compressor" => md.compressor, "fill_value" => fill_value_encoding(md.fill_value), diff --git a/lib/ZarrCore/src/metadata3.jl b/lib/ZarrCore/src/metadata3.jl index 5e49676b..85eaf63e 100644 --- a/lib/ZarrCore/src/metadata3.jl +++ b/lib/ZarrCore/src/metadata3.jl @@ -78,6 +78,7 @@ struct MetadataV3{T,N,P<:AbstractCodecPipeline,E<:AbstractChunkKeyEncoding,CT<:G zarr_format == 3 || throw(ArgumentError("MetadataV3 only functions if zarr_format == 3")) #Do some sanity checks to make sure we have a sane array any(<(0), shape) && throw(ArgumentError("Size must be positive")) + all(((c, s),)->DiskArrays.arraysize_from_chunksize(c) == s, zip(chunks.chunks, shape)) || error("Chunk array does not have the same size as the array shape") new{T2,N,P,E,CT}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, pipeline, fill_value, chunk_key_encoding) end end @@ -210,7 +211,7 @@ function Metadata3(d::AbstractDict, fill_as_missing) end group_pipeline = V3Pipeline((), Codecs.V3Codecs.BytesCodec(), ()) - return MetadataV3{Int,0,typeof(group_pipeline),ChunkKeyEncoding}(zarr_format, node_type, (), (), "", group_pipeline, 0, ChunkKeyEncoding('/', true)) + return MetadataV3{Int,0,typeof(group_pipeline),ChunkKeyEncoding,GridChunks{0,Tuple{}}}(zarr_format, node_type, (), GridChunks{0,Tuple{}}(()), "", group_pipeline, 0, ChunkKeyEncoding('/', true)) end # Array keys @@ -246,17 +247,22 @@ function Metadata3(d::AbstractDict, fill_as_missing) # Chunk Grid chunk_grid = d["chunk_grid"] if chunk_grid["name"] == "regular" - chunks = Int.(chunk_grid["configuration"]["chunk_shape"]) - if length(shape) != length(chunks) - throw(ArgumentError("Shape has rank $(length(shape)) which does not match the chunk_shape rank of $(length(chunks))")) + cshape = Int.(chunk_grid["configuration"]["chunk_shape"]) + if length(shape) != length(cshape) + throw(ArgumentError("Shape has rank $(length(shape)) which does not match the chunk_shape rank of $(length(cshape))")) end + shape_jl = NTuple{length(shape),Int}(shape) |> reverse + chunks = GridChunks(shape_jl, NTuple{length(cshape),Int}(cshape) |> reverse) elseif chunk_grid["name"] == "rectilinear" - chunk_grid["kind"] == "inline" || throw(ArgumentError("Only chunk grid descriptors of kind \"inline\" are allowed")) - cshapes = chunk_grid["chunk_shapes"] - chunkspecs = map(shapes, shape) do csh, s - if shape isa Integer + chunk_grid["configuration"]["kind"] == "inline" || throw(ArgumentError("Only chunk grid descriptors of kind \"inline\" are allowed")) + cshapes = chunk_grid["configuration"]["chunk_shapes"] + length(cshapes) == length(shape) || throw(ArgumentError("chunk_shapes rank does not match the array shape rank")) + chunkspecs = map(cshapes, shape) do csh, s + if csh isa Integer + # a single integer declares a regular grid along this axis RegularChunks(csh, 0, s) else + # a list of edge lengths, possibly with [V, n] run-length encoding chunksizes = Int[] for spec in csh if spec isa Int @@ -271,11 +277,10 @@ function Metadata3(d::AbstractDict, fill_as_missing) IrregularChunks(; chunksizes) end end - chunks = ChunkGrid(reverse(chunkspecs)) + chunks = GridChunks(reverse(chunkspecs)...) else throw(ArgumentError("Unknown chunk_grid of name, $(chunk_grid["name"])")) end - # Chunk Key Encoding chunk_key_encoding = d["chunk_key_encoding"] @@ -293,7 +298,7 @@ function Metadata3(d::AbstractDict, fill_as_missing) chunk_key_encoding = parse_chunk_key_encoding(chunk_key_encoding) E = typeof(chunk_key_encoding) - MetadataV3{TU,N,typeof(pipeline),E}( + MetadataV3{TU,N,typeof(pipeline),E,typeof(chunks)}( zarr_format, node_type, NTuple{N,Int}(shape) |> reverse, @@ -345,7 +350,7 @@ function lower3(md::MetadataV3{T}) where T "name" => "rectilinear", "configuration" => Dict{String,Any}( "kind" => "inline", - "chunk_shapes" => map(md.chunks.chunks) do c + "chunk_shapes" => map(reverse(md.chunks.chunks)) do c if c isa RegularChunks c.chunksize else @@ -358,7 +363,7 @@ function lower3(md::MetadataV3{T}) where T Dict{String,Any}( "name" => "regular", "configuration" => Dict{String,Any}( - "chunk_shape" => md.chunks |> reverse + "chunk_shape" => reverse(DiskArrays.max_chunksize.(md.chunks.chunks)) ) ) end From dc0d6985ed0d4cb35eb4c27d7a76565076cd5982 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 18 Aug 2026 09:06:54 -0400 Subject: [PATCH 24/39] Fix public tests and DateTime64 tests --- .gitignore | 1 + src/Zarr.jl | 6 ------ test/Filters.jl | 1 + test/Project.toml | 1 + test/python.jl | 9 +++++---- test/runtests.jl | 29 ++++++++++++++++------------- 6 files changed, 24 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 85903261..4abf6567 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ Manifest.toml +Manifest-v1.12.toml docs/build *.zarr .CondaPkg diff --git a/src/Zarr.jl b/src/Zarr.jl index 26158f31..366ea656 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -3,12 +3,6 @@ module Zarr import ZarrCore # Mirror ZarrCore's export/public split. Internals stay at Zarr.ZarrCore. -# -# `names` reports exported *and* public names, but only from Julia 1.11 on -- -# 1.10 has no `public` keyword, so ZarrCore additionally records every `@public` -# name in `PUBLIC_NAMES`. Taking the union keeps the surface identical on every -# supported version; without it, every public-but-not-exported name would be -# missing from `Zarr` on LTS. for name in names(ZarrCore) if name !== :ZarrCore diff --git a/test/Filters.jl b/test/Filters.jl index 3168ca5b..d808a2cc 100644 --- a/test/Filters.jl +++ b/test/Filters.jl @@ -60,6 +60,7 @@ end end @testset "ShuffleFilter" begin + using DateTimes64: DateTime64 codecs = [ ShuffleFilter(), diff --git a/test/Project.toml b/test/Project.toml index e69e73ce..6f5cfb53 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,6 +1,7 @@ [deps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" +DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Minio = "4281f0d9-7ae0-406e-9172-b7277c1efa20" diff --git a/test/python.jl b/test/python.jl index d50b41de..082e6af7 100644 --- a/test/python.jl +++ b/test/python.jl @@ -270,12 +270,13 @@ end @testset "Python datetime types" begin using Dates, Test, Zarr, PythonCall +using DateTimes64: DateTime64 vd = Date(1970,1,1):Day(1):Date(1970,6,30) |> collect vt = DateTime(1970,1,1):Second(1):DateTime(1970,1,1,2,0,0)|> collect ad = ZArray(vd) at = ZArray(vt) -@test eltype(ad)==Zarr.DateTime64{Day} -@test eltype(at)==Zarr.DateTime64{Millisecond} +@test eltype(ad)==DateTime64{Day} +@test eltype(at)==DateTime64{Millisecond} @test DateTime.(at[:]) == vt[:] @test Date.(ad[:]) == vd[:] @@ -286,11 +287,11 @@ for pt in [Week, Day, Hour, Minute, Second, if pt <: DatePeriod vd = range(Date(1970,1,1),step = pt(1), length=100) - a = zcreate(Zarr.DateTime64{pt},g,string(pt),100) + a = zcreate(DateTime64{pt},g,string(pt),100) a[:] = vd else vd = range(DateTime(1970,1,1),step = pt(1), length=100) - a = zcreate(Zarr.DateTime64{pt},g,string(pt),100) + a = zcreate(DateTime64{pt},g,string(pt),100) a[:] = vd end end diff --git a/test/runtests.jl b/test/runtests.jl index c0606e1c..f39a0f0a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -31,7 +31,6 @@ using Dates # `Zarr` mirrors `ZarrCore`'s API surface exactly, split intact. @test exported(Zarr) == exported(ZarrCore) - @test publiconly(Zarr) == publiconly(ZarrCore) # The specific failure mode: a name that is only `public` in `ZarrCore` must # not become an export of `Zarr`, and vice versa. @@ -41,16 +40,19 @@ using Dates # Version-independent: the two assertions above compare `names` against # `names`, so on 1.10 -- where `@public` expands to nothing and both # public-only sets are empty -- they pass no matter what `Zarr` re-exports. - # `PUBLIC_NAMES` is populated on every version, so this catches a facade - # that silently drops the entire public API on LTS. - @test !isempty(ZarrCore.PUBLIC_NAMES) - @test isempty(filter(n -> !isdefined(Zarr, n), ZarrCore.PUBLIC_NAMES)) - - # Where both sources exist, they must agree -- otherwise LTS and 1.11+ would - # drift apart again, which is exactly what the registry is there to prevent. - @static if VERSION >= v"1.11" - @test Set(ZarrCore.PUBLIC_NAMES) == publiconly(ZarrCore) - end + #@test !isempty(ZarrCore.PUBLIC_NAMES) + #@test isempty(filter(n -> !isdefined(Zarr, n), ZarrCore.PUBLIC_NAMES)) + @test all(isdefined.(Ref(Zarr), [:zname]) ) + @test all(isdefined.(Ref(Zarr), [:DictStore, :HTTPStore, :ZipStore, :CachingStore, :ConsolidatedStore])) + @test all(isdefined.(Ref(Zarr), [:consolidate_metadata, :writezip])) + @test all(isdefined.(Ref(Zarr), [:ChunkKeyEncoding, :SuffixChunkKeyEncoding])) + @test all(isdefined.(Ref(Zarr), [:Filter, :VLenArrayFilter, :VLenUTF8Filter, :Fletcher32Filter, + :FixedScaleOffsetFilter, :ShuffleFilter, :QuantizeFilter, :DeltaFilter])) + @test all(isdefined.(Ref(Zarr), [:Compressor, :NoCompressor, :BloscCompressor, :ZlibCompressor, :ZstdCompressor])) + @test all(isdefined.(Ref(Zarr), [:Codecs, :Codec, :V3Codec, :BloscCodec, :BytesCodec, :CRC32cCodec, :GzipCodec, + :ShardingCodec, :TransposeCodec, :GzipV3Codec, :BloscV3Codec, :ZstdV3Codec, + :CRC32cV3Codec, :VLenUTF8V3Codec])) + end @testset "ZArray" begin @@ -265,6 +267,7 @@ end @testset "Metadata" begin @testset "Data type encoding" begin + using DateTimes64: DateTime64 @test Zarr.typestr(Bool) === "|b1" @test Zarr.typestr(Int8) === "|i1" @test Zarr.typestr(Int64) === " Date: Tue, 18 Aug 2026 09:08:36 -0400 Subject: [PATCH 25/39] Remove commented out tests --- test/runtests.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index f39a0f0a..747f80ca 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -40,8 +40,6 @@ using Dates # Version-independent: the two assertions above compare `names` against # `names`, so on 1.10 -- where `@public` expands to nothing and both # public-only sets are empty -- they pass no matter what `Zarr` re-exports. - #@test !isempty(ZarrCore.PUBLIC_NAMES) - #@test isempty(filter(n -> !isdefined(Zarr, n), ZarrCore.PUBLIC_NAMES)) @test all(isdefined.(Ref(Zarr), [:zname]) ) @test all(isdefined.(Ref(Zarr), [:DictStore, :HTTPStore, :ZipStore, :CachingStore, :ConsolidatedStore])) @test all(isdefined.(Ref(Zarr), [:consolidate_metadata, :writezip])) From e08569e64206ccb558205c0a1c59e28f164d7531 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 18 Aug 2026 09:31:02 -0400 Subject: [PATCH 26/39] Remove DateTime64 import from Zarr --- test/Filters.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Filters.jl b/test/Filters.jl index d808a2cc..7d4e6d5a 100644 --- a/test/Filters.jl +++ b/test/Filters.jl @@ -1,5 +1,5 @@ using Test -using Zarr: DateTime64 # for datetime reinterpret +using DateTime64s: DateTime64 # for datetime reinterpret using Zarr: zencode, zdecode using Zarr: Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter From 417917d2ce6f678628aefcd55d44b38e6e04921a Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 18 Aug 2026 10:35:51 -0400 Subject: [PATCH 27/39] Fix package name to DateTimes64 --- test/Filters.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Filters.jl b/test/Filters.jl index 7d4e6d5a..0325850c 100644 --- a/test/Filters.jl +++ b/test/Filters.jl @@ -1,5 +1,5 @@ using Test -using DateTime64s: DateTime64 # for datetime reinterpret +using DateTimes64: DateTime64 # for datetime reinterpret using Zarr: zencode, zdecode using Zarr: Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter From 5d00d077b022531709a64af31ef07544d39f09c5 Mon Sep 17 00:00:00 2001 From: Felix Cremer Date: Wed, 19 Aug 2026 10:20:01 +0200 Subject: [PATCH 28/39] Test ZarrCore.typestr instead of Zarr.typestr --- lib/ZarrCore/src/ZarrCore.jl | 2 +- test/runtests.jl | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/ZarrCore/src/ZarrCore.jl b/lib/ZarrCore/src/ZarrCore.jl index 5fa78a5d..ec2b5f4a 100644 --- a/lib/ZarrCore/src/ZarrCore.jl +++ b/lib/ZarrCore/src/ZarrCore.jl @@ -37,7 +37,7 @@ import .Codecs.V3Codecs: V3Codec, BloscCodec, BytesCodec, CRC32cCodec, GzipCodec # The rule of thumb is: anything a downstream consumer could need, and every # documented extension point, is part of the public API. `export` is reserved # for the handful of names that are convenient to have in scope after -# `using Zarr`; everything else is marked `@public` and must be qualified. +# `using Zarr`; everything else is marked `public` and must be qualified. # # Things that are deliberately *not* public (and may change without notice): # `Metadata`/`MetadataV2`/`MetadataV3`, `ZarrFormat`, `is_zarray`, `is_zgroup`, diff --git a/test/runtests.jl b/test/runtests.jl index 747f80ca..b55694bc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -266,22 +266,22 @@ end @testset "Metadata" begin @testset "Data type encoding" begin using DateTimes64: DateTime64 - @test Zarr.typestr(Bool) === "|b1" - @test Zarr.typestr(Int8) === "|i1" - @test Zarr.typestr(Int64) === " Date: Wed, 19 Aug 2026 10:51:55 +0200 Subject: [PATCH 29/39] Add typrstr, fill_value_* to public_names_zarr.jl Fix LTS CI --- src/public_names_zarr.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/public_names_zarr.jl b/src/public_names_zarr.jl index e955bd8e..18c0075f 100644 --- a/src/public_names_zarr.jl +++ b/src/public_names_zarr.jl @@ -20,4 +20,7 @@ public Compressor, NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor # v3 codecs and the interface a new codec has to implement, see `?Codec`. public Codecs, Codec, V3Codec, BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, ShardingCodec, TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, - CRC32cV3Codec, VLenUTF8V3Codec \ No newline at end of file + CRC32cV3Codec, VLenUTF8V3Codec + +# Data type and fill value encoding, needed to map Zarr dtypes to Julia types. +public typestr, fill_value_encoding, fill_value_decoding \ No newline at end of file From 4cf520ae1152779d057fe1e98b585410b7a89eb0 Mon Sep 17 00:00:00 2001 From: Felix Cremer Date: Wed, 19 Aug 2026 11:13:27 +0200 Subject: [PATCH 30/39] Use public names from ZarrCore not Zarr This is necessary because in 1.10 the names function behaves differently and the public names are not imported into Zarr. --- src/Zarr.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Zarr.jl b/src/Zarr.jl index 366ea656..dee5af73 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -19,7 +19,7 @@ end else # For Julia 1.10, we have to parse the public names from the source file, since # `public` is not a keyword and `names(ZarrCore)` only returns exported names. - let public_names = read(joinpath(@__DIR__, "public_names_zarr.jl"), String) + let public_names = read(joinpath(@__DIR__, "..", "lib", "ZarrCore", "public_names_zarr.jl"), String) public_names = replace(public_names, "public" => "using ZarrCore: ") eval(Meta.parseall(public_names)) end From 4c4bef0e95d4800251f4ce6eb636f58946d4591b Mon Sep 17 00:00:00 2001 From: Felix Cremer Date: Wed, 19 Aug 2026 11:26:48 +0200 Subject: [PATCH 31/39] Fix path typo --- src/Zarr.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Zarr.jl b/src/Zarr.jl index dee5af73..c5d44032 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -19,7 +19,7 @@ end else # For Julia 1.10, we have to parse the public names from the source file, since # `public` is not a keyword and `names(ZarrCore)` only returns exported names. - let public_names = read(joinpath(@__DIR__, "..", "lib", "ZarrCore", "public_names_zarr.jl"), String) + let public_names = read(joinpath(@__DIR__, "..", "lib", "ZarrCore", "public_names_core.jl"), String) public_names = replace(public_names, "public" => "using ZarrCore: ") eval(Meta.parseall(public_names)) end From e69220ff54a4aca519d576cde664e6b4bd18a8b5 Mon Sep 17 00:00:00 2001 From: Felix Cremer Date: Wed, 19 Aug 2026 11:39:24 +0200 Subject: [PATCH 32/39] Fix path typo again --- src/Zarr.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Zarr.jl b/src/Zarr.jl index c5d44032..1dd4b8a9 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -19,7 +19,7 @@ end else # For Julia 1.10, we have to parse the public names from the source file, since # `public` is not a keyword and `names(ZarrCore)` only returns exported names. - let public_names = read(joinpath(@__DIR__, "..", "lib", "ZarrCore", "public_names_core.jl"), String) + let public_names = read(joinpath(@__DIR__, "..", "lib", "ZarrCore", "src", "public_names_core.jl"), String) public_names = replace(public_names, "public" => "using ZarrCore: ") eval(Meta.parseall(public_names)) end From f142f74a79211ac7b85905e1767adf83859a1ad4 Mon Sep 17 00:00:00 2001 From: Fabian Gans Date: Thu, 20 Aug 2026 10:52:06 +0200 Subject: [PATCH 33/39] Make tests almost pass --- lib/ZarrCore/src/ZArray.jl | 48 ++++++++++++++++++++--------------- lib/ZarrCore/src/metadata.jl | 8 +++--- lib/ZarrCore/src/metadata3.jl | 15 +++++------ test/Project.toml | 1 + test/http_sharded.jl | 8 +++--- test/runtests.jl | 9 ++++--- test/v3_codecs.jl | 21 +++++++++------ 7 files changed, 62 insertions(+), 48 deletions(-) diff --git a/lib/ZarrCore/src/ZArray.jl b/lib/ZarrCore/src/ZArray.jl index 74f29740..b9318e44 100644 --- a/lib/ZarrCore/src/ZArray.jl +++ b/lib/ZarrCore/src/ZArray.jl @@ -99,7 +99,7 @@ function zinfo(io::IO, z::ZArray) "Type" => "ZArray", "Data type" => eltype(z), "Shape" => size(z), - "Chunk Shape" => z.metadata.chunks, + "Chunk Shape" => eachchunk(z), "Order" => try get_order(z.metadata) catch e @@ -145,7 +145,7 @@ end function getchunkarray(z::ZArray{>:Missing}) # temporary workaround to use strings as data values - inner = fill(z.metadata.fill_value, DiskArrays.max_chunksize.(z.metadata.chunks.chunks)) + inner = fill(z.metadata.fill_value, DiskArrays.max_chunksize.(eachchunk(z).chunks)) a = SenMissArray(inner, z.metadata.fill_value) end _zero(T) = zero(T) @@ -153,7 +153,7 @@ _zero(T::Type{<:MaxLengthString}) = zero(T) _zero(T::Type{ASCIIChar}) = ASCIIChar(0) _zero(::Type{<:Vector{T}}) where T = T[] _zero(::Type{Char}) = Char(0) -getchunkarray(z::ZArray) = fill(_zero(eltype(z)), DiskArrays.max_chunksize.(z.metadata.chunks.chunks)) +getchunkarray(z::ZArray) = fill(_zero(eltype(z)), DiskArrays.max_chunksize.(eachchunk(z).chunks)) # Same as `getchunkarray` but skips the zero/fill_value-fill. Use only when # the caller guarantees the buffer will be fully overwritten before any read @@ -165,7 +165,7 @@ getchunkarray(z::ZArray) = fill(_zero(eltype(z)), DiskArrays.max_chunksize.(z.me # friends reject non-isbits eltypes). function getchunkarray_undef(z::ZArray{T}) where {T} Missing <: T && return getchunkarray(z) - return Array{T}(undef, DiskArrays.max_chunksize.(z.metadata.chunks.chunks)) + return Array{T}(undef, DiskArrays.max_chunksize.(eachchunk(z).chunks)) end maybeinner(a::Array) = a @@ -180,7 +180,7 @@ resetbuffer!(_, a::SenMissArray) = fill!(a, missing) # coverage check is needed. function singlechunk_fastpath(arr, z::ZArray{T,N}, blockr::CartesianIndices{N}) where {T,N} arr isa Array{T,N} && !(Missing <: T) && - size(arr) == DiskArrays.max_chunksize.(z.metadata.chunks.chunks) && length(blockr) == 1 || return nothing + size(arr) == DiskArrays.max_chunksize.(eachchunk(z).chunks) && length(blockr) == 1 || return nothing return first(blockr) end @@ -231,7 +231,7 @@ function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::Cartesi output_base_offsets = map(i->first(i)-1, r.indices) # Determines which chunks are affected - blockr = CartesianIndices(map(DiskArrays.findchunk, z.metadata.chunks.chunks, r.indices)) + blockr = CartesianIndices(map(DiskArrays.findchunk, eachchunk(z).chunks, r.indices)) # Fast path: single-chunk full-read decodes directly into `aout`, skipping the readtask channel and scratch buffer. bI = singlechunk_fastpath(aout, z, blockr) if bI !== nothing @@ -276,7 +276,7 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::Cartesi z.writeable || error("Can not write to read-only ZArray") input_base_offsets = map(i->first(i)-1, r.indices) # Determines which chunks are affected - blockr = CartesianIndices(map(DiskArrays.findchunk, z.metadata.chunks.chunks, r.indices)) + blockr = CartesianIndices(map(DiskArrays.findchunk, eachchunk(z).chunks, r.indices)) # Fast path: single-chunk full-overwrite skips the readtask/writetask channels and the scratch buffer. bI = singlechunk_fastpath(ain, z, blockr) if bI !== nothing @@ -342,7 +342,7 @@ end DiskArrays.readblock!(a::ZArray, aout, i::AbstractUnitRange...) = readblock!(aout, a, CartesianIndices(i)) DiskArrays.writeblock!(a::ZArray, v, i::AbstractUnitRange...) = writeblock!(v, a, CartesianIndices(i)) DiskArrays.haschunks(::ZArray) = DiskArrays.Chunked() -DiskArrays.eachchunk(a::ZArray) = a.metadata.chunks +DiskArrays.eachchunk(a::ZArray) = a.metadata.chunks[] """ uncompress_raw!(a::DenseArray{T},z::ZArray{T,N},i::CartesianIndex{N}) @@ -376,7 +376,7 @@ function uncompress_to_output!(aout, output_base_offsets, z, chunk_compressed, c end function compress_raw(a, z) - length(a) == prod(DiskArrays.max_chunksize.(z.metadata.chunks.chunks)) || throw(DimensionMismatch("Array size does not equal chunk size")) + length(a) == prod(DiskArrays.max_chunksize.(eachchunk(z).chunks)) || throw(DimensionMismatch("Array size does not equal chunk size")) pipeline_encode(get_pipeline(z.metadata), a, z.metadata.fill_value) end @@ -509,7 +509,7 @@ end Returns the Cartesian Indices of the chunks of a given ZArray """ -chunkindices(z::ZArray) = CartesianIndices(map(length, z.metadata.chunks.chunks)) +chunkindices(z::ZArray) = CartesianIndices(map(length, eachchunk(z).chunks)) """ zzeros(T, dims...; kwargs... ) @@ -518,15 +518,18 @@ Creates a zarr array and initializes all values with zero. Accepts the same keyw """ function zzeros(T, dims...; kwargs...) z = zcreate(T, dims...; kwargs...) - # as = zeros(T, DiskArrays.max_chunksize.(z.metadata.chunks.chunks)) - # data_encoded = compress_raw(as, z) - # p = z.path - # if data_encoded !== nothing - # for i in chunkindices(z) - # store_writechunk(z.storage, data_encoded, p, i, z.metadata.chunk_key_encoding) - # end - # end - z .= zero(T) + if all(i->isa(i, RegularChunks), eachchunk(z).chunks) + as = zeros(T, DiskArrays.max_chunksize.(eachchunk(z).chunks)) + data_encoded = compress_raw(as, z) + p = z.path + if data_encoded !== nothing + for i in chunkindices(z) + store_writechunk(z.storage, data_encoded, p, i, z.metadata.chunk_key_encoding) + end + end + else + z .= zero(T) + end z end @@ -541,9 +544,14 @@ function Base.resize!(z::ZArray{T,N}, newsize::NTuple{N}) where {T,N} any(<(0), newsize) && throw(ArgumentError("Size must be positive")) oldsize = z.metadata.shape[] z.metadata.shape[] = newsize + gc = z.metadata.chunks[].chunks + new_chunks = map(gc, newsize) do c, s + c isa DiskArrays.RegularChunks ? DiskArrays.RegularChunks(c.chunksize, c.offset, s) : c + end + z.metadata.chunks[] = DiskArrays.GridChunks(new_chunks) #Check if array was shrunk if any(map(<, newsize, oldsize)) - prune_oob_chunks(z.storage, z.path, oldsize, newsize, z.metadata.chunks, z.metadata.chunk_key_encoding) + prune_oob_chunks(z.storage, z.path, oldsize, newsize, eachchunk(z), z.metadata.chunk_key_encoding) end writemetadata(zarr_format(z), z.storage, z.path, z.metadata) nothing diff --git a/lib/ZarrCore/src/metadata.jl b/lib/ZarrCore/src/metadata.jl index 03832616..e8ee6b09 100644 --- a/lib/ZarrCore/src/metadata.jl +++ b/lib/ZarrCore/src/metadata.jl @@ -112,7 +112,7 @@ struct MetadataV2{T,N,C,F,CT<:GridChunks{N}} <: AbstractMetadata{T,N,ChunkKeyEnc zarr_format::Int node_type::String shape::Base.RefValue{NTuple{N,Int}} - chunks::CT + chunks::Base.RefValue{CT} dtype::String # structured data types not yet supported compressor::C fill_value::Union{T,Nothing} @@ -125,7 +125,7 @@ struct MetadataV2{T,N,C,F,CT<:GridChunks{N}} <: AbstractMetadata{T,N,ChunkKeyEnc any(<(0), shape) && throw(ArgumentError("Size must be positive")) any(<(1), DiskArrays.max_chunksize.(chunks.chunks)) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) order === 'C' || throw(ArgumentError("Currently only 'C' storage order is supported")) - new{T2,N,C,F,CT}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, compressor, fill_value, order, filters, chunk_key_encoding) + new{T2,N,C,F,CT}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), Ref(chunks), dtype, compressor, fill_value, order, filters, chunk_key_encoding) end end zarr_format(::MetadataV2) = ZarrFormat(Val(2)) @@ -138,7 +138,7 @@ function Base.:(==)(m1::MetadataV2, m2::MetadataV2) m1.zarr_format == m2.zarr_format && m1.node_type == m2.node_type && m1.shape[] == m2.shape[] && - m1.chunks == m2.chunks && + m1.chunks[] == m2.chunks[] && m1.dtype == m2.dtype && m1.compressor == m2.compressor && m1.fill_value == m2.fill_value && @@ -257,7 +257,7 @@ function JSON.lower(md::MetadataV2) "zarr_format" => Int(md.zarr_format), "node_type" => md.node_type, "shape" => md.shape[] |> reverse, - "chunks" => reverse(DiskArrays.max_chunksize.(md.chunks.chunks)), + "chunks" => reverse(DiskArrays.max_chunksize.(md.chunks[].chunks)), "dtype" => md.dtype, "compressor" => md.compressor, "fill_value" => fill_value_encoding(md.fill_value), diff --git a/lib/ZarrCore/src/metadata3.jl b/lib/ZarrCore/src/metadata3.jl index 85eaf63e..aa05c90d 100644 --- a/lib/ZarrCore/src/metadata3.jl +++ b/lib/ZarrCore/src/metadata3.jl @@ -69,7 +69,7 @@ struct MetadataV3{T,N,P<:AbstractCodecPipeline,E<:AbstractChunkKeyEncoding,CT<:G zarr_format::Int node_type::String shape::Base.RefValue{NTuple{N,Int}} - chunks::CT + chunks::Base.RefValue{CT} dtype::Union{String,Dict{String,Any}} # data_type in v3 pipeline::P fill_value::Union{T,Nothing} @@ -79,10 +79,9 @@ struct MetadataV3{T,N,P<:AbstractCodecPipeline,E<:AbstractChunkKeyEncoding,CT<:G #Do some sanity checks to make sure we have a sane array any(<(0), shape) && throw(ArgumentError("Size must be positive")) all(((c, s),)->DiskArrays.arraysize_from_chunksize(c) == s, zip(chunks.chunks, shape)) || error("Chunk array does not have the same size as the array shape") - new{T2,N,P,E,CT}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, pipeline, fill_value, chunk_key_encoding) + new{T2,N,P,E,CT}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), Ref(chunks), dtype, pipeline, fill_value, chunk_key_encoding) end end -MetadataV3{T2,N,P}(args...) where {T2,N,P} = MetadataV3{T2,N,P,ChunkKeyEncoding}(args...) zarr_format(::MetadataV3) = ZarrFormat(Val(3)) """ @@ -111,7 +110,7 @@ function MetadataV3{T2,N}(zarr_format, node_type, shape::NTuple{N,Int}, chunks, typesize = sizeof(T_base) end if chunks isa Tuple - chunks = GridChunks(size(A), chunks) + chunks = GridChunks(shape, chunks) end bytes_bytes_codecs = if compressor isa NoCompressor () @@ -134,7 +133,7 @@ function Base.:(==)(m1::MetadataV3, m2::MetadataV3) m1.zarr_format == m2.zarr_format && m1.node_type == m2.node_type && m1.shape[] == m2.shape[] && - m1.chunks == m2.chunks && + m1.chunks[] == m2.chunks[] && m1.dtype == m2.dtype && m1.fill_value == m2.fill_value && m1.pipeline == m2.pipeline && @@ -345,12 +344,12 @@ end function lower3(md::MetadataV3{T}) where T - chunk_grid = if any(i->isa(i, IrregularChunks), md.chunks.chunks) + chunk_grid = if any(i->isa(i, IrregularChunks), md.chunks[].chunks) Dict{String,Any}( "name" => "rectilinear", "configuration" => Dict{String,Any}( "kind" => "inline", - "chunk_shapes" => map(reverse(md.chunks.chunks)) do c + "chunk_shapes" => map(reverse(md.chunks[].chunks)) do c if c isa RegularChunks c.chunksize else @@ -363,7 +362,7 @@ function lower3(md::MetadataV3{T}) where T Dict{String,Any}( "name" => "regular", "configuration" => Dict{String,Any}( - "chunk_shape" => reverse(DiskArrays.max_chunksize.(md.chunks.chunks)) + "chunk_shape" => reverse(DiskArrays.max_chunksize.(md.chunks[].chunks)) ) ) end diff --git a/test/Project.toml b/test/Project.toml index e69e73ce..25241841 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,6 +2,7 @@ AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Minio = "4281f0d9-7ae0-406e-9172-b7277c1efa20" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" diff --git a/test/http_sharded.jl b/test/http_sharded.jl index 2f567633..1a5aeb69 100644 --- a/test/http_sharded.jl +++ b/test/http_sharded.jl @@ -45,7 +45,7 @@ const TESSERA_BASE = "https://dl2.geotessera.org/zarr/v1/2024.zarr" # Julia reverses to column-major: (128, 66560, 1355776) @test size(z) == (128, 66560, 1355776) # Outer shard shape [256, 256, 128] → Julia (128, 256, 256) - @test z.metadata.chunks == (128, 256, 256) + @test z.metadata.chunks[] == (128, 256, 256) # Verify the codec is sharding_indexed with the expected inner chunk shape pipeline = z.metadata.pipeline @@ -144,7 +144,7 @@ const FLAMINGO_BASE = "https://radosgw.public.os.wwu.de/n4bi-goe" @test eltype(z) == UInt16 @test ndims(z) == 3 @test size(z) == (1024, 1024, 192) - @test z.metadata.chunks == (512, 512, 512) + @test z.metadata.chunks[] == (512, 512, 512) sharding = z.metadata.pipeline.array_bytes @test sharding isa Zarr.Codecs.V3Codecs.ShardingCodec @@ -173,7 +173,7 @@ const MUENSTER_BASE = "https://radosgw.public.os2.wwu.de/ngff" @test eltype(z) == UInt8 @test ndims(z) == 3 @test size(z) == (6000, 6000, 6000) - @test z.metadata.chunks == (8192, 8192, 1) + @test z.metadata.chunks[] == (8192, 8192, 1) sharding = z.metadata.pipeline.array_bytes @test sharding isa Zarr.Codecs.V3Codecs.ShardingCodec @@ -234,7 +234,7 @@ end # @testset "Remote HTTP sharded arrays (SSBD — RIKEN)" @test eltype(z) == UInt16 @test ndims(z) == 5 @test size(z) == (522693, 244215, 1, 3, 1) - @test z.metadata.chunks == (2048, 2048, 1, 1, 1) + @test z.metadata.chunks[] == (2048, 2048, 1, 1, 1) sharding = z.metadata.pipeline.array_bytes @test sharding isa Zarr.Codecs.V3Codecs.ShardingCodec diff --git a/test/runtests.jl b/test/runtests.jl index c0606e1c..64eda4fc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,6 +5,7 @@ using JSON using JSON: json using Pkg using Dates +using DiskArrays: GridChunks, DiskArrays, RegularChunks @testset "Zarr" begin @@ -56,7 +57,7 @@ end @testset "ZArray" begin @testset "fields" begin z = zzeros(Int64, 2, 3) - @test z isa ZArray{Int64,2,ZarrCore.DictStore,ZarrCore.MetadataV2{Int64,2,ZarrCore.BloscCompressor,Nothing}} + @test z isa ZArray{Int64,2,ZarrCore.DictStore,ZarrCore.MetadataV2{Int64,2,ZarrCore.BloscCompressor,Nothing,GridChunks{2,Tuple{RegularChunks,RegularChunks}}}} @test :a ∈ propertynames(z.storage) @test length(z.storage.a) === 3 @test length(z.storage.a["0.0"]) === 64 @@ -65,7 +66,7 @@ end @test z.metadata.node_type === "array" @test z.metadata.shape[] === (2, 3) @test z.metadata.order === 'C' - @test z.metadata.chunks === (2, 3) + @test z.metadata.chunks[] === GridChunks((2, 3), (2, 3)) @test z.metadata.fill_value === nothing @test z.metadata.compressor isa Zarr.BloscCompressor @test z.metadata.compressor.blocksize === 0 @@ -82,7 +83,7 @@ end @testset "methods" begin z = zzeros(Int64, 2, 3) - @test z isa ZArray{Int64,2,Zarr.DictStore,ZarrCore.MetadataV2{Int64,2,ZarrCore.BloscCompressor,Nothing}} + @test z isa ZArray{Int64,2,ZarrCore.DictStore,ZarrCore.MetadataV2{Int64,2,ZarrCore.BloscCompressor,Nothing,GridChunks{2,Tuple{RegularChunks,RegularChunks}}}} @test eltype(z) === Int64 @test ndims(z) === 2 @test size(z) === (2, 3) @@ -290,7 +291,7 @@ end @test metadata isa ZarrCore.Metadata @test metadata.zarr_format === 2 @test metadata.shape[] === size(A) - @test metadata.chunks === chunks + @test metadata.chunks[] === GridChunks(size(A), chunks) @test metadata.dtype === " Date: Fri, 21 Aug 2026 10:37:27 +0200 Subject: [PATCH 34/39] Fix remaining unit tests --- test/consolidated.jl | 4 +-- test/http_sharded.jl | 8 +++--- test/v3_codecs.jl | 67 +++++++++++++++++++++++++++----------------- test/v3_julia.jl | 9 ++++-- 4 files changed, 53 insertions(+), 35 deletions(-) diff --git a/test/consolidated.jl b/test/consolidated.jl index 65c8fc73..03d95b38 100644 --- a/test/consolidated.jl +++ b/test/consolidated.jl @@ -85,7 +85,7 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") # getmetadata v3 reads from cons["metadata"][key] meta = ZarrCore.getmetadata(ZarrCore.ZarrFormat(3), cs.storage, "1d.chunked.i2", false) @test eltype(meta) == Int16 - @test meta.chunks == (2,) + @test meta.chunks[] == ZarrCore.DiskArrays.GridChunks((4,), (2,)) end @testset "is_zarray / is_zgroup v3 on ConsolidatedStore" begin @@ -232,7 +232,7 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") V2 = ZarrCore.ZarrFormat(2) meta = ZarrCore.getmetadata(V2, cs, "arr", false) @test meta.dtype == Int32 || eltype(meta) == Int32 - @test meta.chunks == (5, 5) + @test meta.chunks[] == ZarrCore.DiskArrays.GridChunks((10, 20), (5, 5)) end @testset "v3 getattrs" begin diff --git a/test/http_sharded.jl b/test/http_sharded.jl index 1a5aeb69..fa6ff83a 100644 --- a/test/http_sharded.jl +++ b/test/http_sharded.jl @@ -45,7 +45,7 @@ const TESSERA_BASE = "https://dl2.geotessera.org/zarr/v1/2024.zarr" # Julia reverses to column-major: (128, 66560, 1355776) @test size(z) == (128, 66560, 1355776) # Outer shard shape [256, 256, 128] → Julia (128, 256, 256) - @test z.metadata.chunks[] == (128, 256, 256) + @test DiskArrays.max_chunksize.(z.metadata.chunks[].chunks) == (128, 256, 256) # Verify the codec is sharding_indexed with the expected inner chunk shape pipeline = z.metadata.pipeline @@ -144,7 +144,7 @@ const FLAMINGO_BASE = "https://radosgw.public.os.wwu.de/n4bi-goe" @test eltype(z) == UInt16 @test ndims(z) == 3 @test size(z) == (1024, 1024, 192) - @test z.metadata.chunks[] == (512, 512, 512) + @test DiskArrays.max_chunksize.(z.metadata.chunks[].chunks) == (512, 512, 512) sharding = z.metadata.pipeline.array_bytes @test sharding isa Zarr.Codecs.V3Codecs.ShardingCodec @@ -173,7 +173,7 @@ const MUENSTER_BASE = "https://radosgw.public.os2.wwu.de/ngff" @test eltype(z) == UInt8 @test ndims(z) == 3 @test size(z) == (6000, 6000, 6000) - @test z.metadata.chunks[] == (8192, 8192, 1) + @test DiskArrays.max_chunksize.(z.metadata.chunks[].chunks) == (8192, 8192, 1) sharding = z.metadata.pipeline.array_bytes @test sharding isa Zarr.Codecs.V3Codecs.ShardingCodec @@ -234,7 +234,7 @@ end # @testset "Remote HTTP sharded arrays (SSBD — RIKEN)" @test eltype(z) == UInt16 @test ndims(z) == 5 @test size(z) == (522693, 244215, 1, 3, 1) - @test z.metadata.chunks[] == (2048, 2048, 1, 1, 1) + @test DiskArrays.max_chunksize.(z.metadata.chunks[].chunks) == (2048, 2048, 1, 1, 1) sharding = z.metadata.pipeline.array_bytes @test sharding isa Zarr.Codecs.V3Codecs.ShardingCodec diff --git a/test/v3_codecs.jl b/test/v3_codecs.jl index 8d3e80f3..58c1de3d 100644 --- a/test/v3_codecs.jl +++ b/test/v3_codecs.jl @@ -68,39 +68,41 @@ end @testset "get_order" begin bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() + ch = DiskArrays.GridChunks((3, 3, 3), (3, 3, 3)) + cke = Zarr.ChunkKeyEncoding('/', true) # No array->array codecs → 'C' p = ZarrCore.V3Pipeline((), bytes_codec, ()) - md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) @test ZarrCore.get_order(md) == 'C' # Single TransposeCodec with identity permutation → 'C' tc_c = Zarr.Codecs.V3Codecs.TransposeCodec((1,2,3)) p = ZarrCore.V3Pipeline((tc_c,), bytes_codec, ()) - md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) @test ZarrCore.get_order(md) == 'C' # Single TransposeCodec with reverse permutation → 'F' tc_f = Zarr.Codecs.V3Codecs.TransposeCodec((3,2,1)) p = ZarrCore.V3Pipeline((tc_f,), bytes_codec, ()) - md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) @test ZarrCore.get_order(md) == 'F' # Single TransposeCodec with arbitrary (non-C, non-F) permutation → ArgumentError tc_other = Zarr.Codecs.V3Codecs.TransposeCodec((2,1,3)) p = ZarrCore.V3Pipeline((tc_other,), bytes_codec, ()) - md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) @test_throws ArgumentError ZarrCore.get_order(md) # Multiple array->array codecs → ArgumentError p = ZarrCore.V3Pipeline((tc_f, tc_f), bytes_codec, ()) - md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) @test_throws ArgumentError ZarrCore.get_order(md) # Unrecognized array->array codec type → ArgumentError struct _FakeCodec <: Zarr.Codecs.V3Codecs.V3Codec{:array,:array} end p = ZarrCore.V3Pipeline((_FakeCodec(),), bytes_codec, ()) - md = ZarrCore.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) @test_throws ArgumentError ZarrCore.get_order(md) end @@ -440,7 +442,7 @@ end md = ZarrCore.Metadata(json_str, false) @test md isa ZarrCore.MetadataV3 @test md.shape[] == (4,) - @test md.chunks == (4,) + @test md.chunks[] == GridChunks((4,), (4,)) @test md.fill_value == Int32(0) pipeline = ZarrCore.get_pipeline(md) @@ -631,7 +633,8 @@ end pipeline = ZarrCore.V3Pipeline((), bytes_codec, ()) P = typeof(pipeline) E = typeof(cke) - md = ZarrCore.MetadataV3{Int32,2,P,E}(3, "array", (4,4), (2,2), "int32", pipeline, Int32(0), cke) + ch = DiskArrays.GridChunks((4, 4), (2, 2)) + md = ZarrCore.MetadataV3{Int32,2,P,E,typeof(ch)}(3, "array", (4,4), ch, "int32", pipeline, Int32(0), cke) z = Zarr.ZArray(md, store, "", Dict(), true) z[:,:] = reshape(Int32.(1:16), 4, 4) @test z[:,:] == reshape(Int32.(1:16), 4, 4) @@ -1141,9 +1144,11 @@ end ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) pipeline = ZarrCore.V3Pipeline((), sharding, ()) - md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline)}( - 3, "array", (3,), (3,), "int16", pipeline, Int16(0), - Zarr.ChunkKeyEncoding('/', true) + ch = DiskArrays.GridChunks((3,), (3,)) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline),typeof(cke),typeof(ch)}( + 3, "array", (3,), ch, "int16", pipeline, Int16(0), + cke ) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) @@ -1165,9 +1170,11 @@ end ) sharding2 = Zarr.Codecs.V3Codecs.ShardingCodec((2,2), inner_pipeline2, index_pipeline2, :end) pipeline2 = ZarrCore.V3Pipeline((), sharding2, ()) - md2 = ZarrCore.MetadataV3{Int32,2,typeof(pipeline2)}( - 3, "array", (3,3), (3,3), "int32", pipeline2, Int32(0), - Zarr.ChunkKeyEncoding('/', true) + ch2 = DiskArrays.GridChunks((3, 3), (3, 3)) + cke2 = Zarr.ChunkKeyEncoding('/', true) + md2 = ZarrCore.MetadataV3{Int32,2,typeof(pipeline2),typeof(cke2),typeof(ch2)}( + 3, "array", (3,3), ch2, "int32", pipeline2, Int32(0), + cke2 ) store2 = Zarr.DictStore() z2 = Zarr.ZArray(md2, store2, "", Dict(), true) @@ -1192,9 +1199,11 @@ end ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) pipeline = ZarrCore.V3Pipeline((), sharding, ()) - md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline)}( - 3, "array", (4,), (4,), "int16", pipeline, Int16(0), - Zarr.ChunkKeyEncoding('/', true) + ch = DiskArrays.GridChunks((4,), (4,)) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline),typeof(cke),typeof(ch)}( + 3, "array", (4,), ch, "int16", pipeline, Int16(0), + cke ) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) @@ -1221,9 +1230,11 @@ end ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :start) pipeline = ZarrCore.V3Pipeline((), sharding, ()) - md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline)}( - 3, "array", (4,), (4,), "int16", pipeline, Int16(0), - Zarr.ChunkKeyEncoding('/', true) + ch = DiskArrays.GridChunks((4,), (4,)) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline),typeof(cke),typeof(ch)}( + 3, "array", (4,), ch, "int16", pipeline, Int16(0), + cke ) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) @@ -1300,9 +1311,11 @@ end ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) pipeline = ZarrCore.V3Pipeline((), sharding, ()) - md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline)}( - 3, "array", (8,), (4,), "int16", pipeline, Int16(0), - Zarr.ChunkKeyEncoding('/', true) + ch = DiskArrays.GridChunks((8,), (4,)) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline),typeof(cke),typeof(ch)}( + 3, "array", (8,), ch, "int16", pipeline, Int16(0), + cke ) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) @@ -1329,9 +1342,11 @@ end ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) pipeline = ZarrCore.V3Pipeline((), sharding, ()) - md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline)}( - 3, "array", (4,), (4,), "int16", pipeline, Int16(99), - Zarr.ChunkKeyEncoding('/', true) + ch = DiskArrays.GridChunks((4,), (4,)) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline),typeof(cke),typeof(ch)}( + 3, "array", (4,), ch, "int16", pipeline, Int16(99), + cke ) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) diff --git a/test/v3_julia.jl b/test/v3_julia.jl index 46ab75f5..a3a37299 100644 --- a/test/v3_julia.jl +++ b/test/v3_julia.jl @@ -3,6 +3,7 @@ using Zarr import Zarr: ZarrCore +using DiskArrays: GridChunks using JSON # Paths @@ -230,9 +231,11 @@ function create_sharded(store, name, data, outer_chunk_shape, inner_chunk_shape; ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec(inner_chunk_shape, inner_pipeline, index_pipeline, index_location) pipeline = ZarrCore.V3Pipeline((), sharding, ()) - md = ZarrCore.MetadataV3{T, N, typeof(pipeline)}( - 3, "array", size(data), outer_chunk_shape, ZarrCore.typestr3(T), pipeline, zero(T), - Zarr.ChunkKeyEncoding('/', true), + ch = GridChunks(size(data), outer_chunk_shape) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{T, N, typeof(pipeline), typeof(cke), typeof(ch)}( + 3, "array", size(data), ch, ZarrCore.typestr3(T), pipeline, zero(T), + cke, ) z = Zarr.ZArray(md, store, name, Dict(), true) ZarrCore.writemetadata(ZarrCore.zarr_format(md), store, name, md) From 00ad67096cc615554e78b5ad5b38a6590b61b927 Mon Sep 17 00:00:00 2001 From: Fabian Gans Date: Fri, 21 Aug 2026 09:34:02 +0200 Subject: [PATCH 35/39] towards passing round trip --- lib/ZarrCore/src/ZArray.jl | 49 +++++++++++++++++++++++------- test/python.jl | 61 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 11 deletions(-) diff --git a/lib/ZarrCore/src/ZArray.jl b/lib/ZarrCore/src/ZArray.jl index b9318e44..7d4c1470 100644 --- a/lib/ZarrCore/src/ZArray.jl +++ b/lib/ZarrCore/src/ZArray.jl @@ -252,16 +252,25 @@ function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::Cartesi end bind(c, task) + chunks = eachchunk(z) + try for i in 1:length(blockr) bI, chunk_compressed = take!(c) - current_chunk_offsets = map((s, i)->s*(i-1), size(a), Tuple(bI)) + current_chunks_size = length.(chunks[bI]) - indranges = map(boundint, r.indices, size(a), current_chunk_offsets) + current_chunk_offsets = first.(chunks[bI]) .- 1 - uncompress_to_output!(aout, output_base_offsets, z, chunk_compressed, current_chunk_offsets, a, indranges) + indranges = map(boundint, r.indices, current_chunks_size, current_chunk_offsets) + #If the chunk size is smaller than the buffer size we need to write a smaller buffer + if current_chunks_size != size(a) + inds_reduced = Base.OneTo.(current_chunks_size) + uncompress_to_output!(aout, output_base_offsets, z, chunk_compressed, current_chunk_offsets, view(a, inds_reduced...), indranges) + else + uncompress_to_output!(aout, output_base_offsets, z, chunk_compressed, current_chunk_offsets, a, indranges) + end nothing end finally @@ -303,12 +312,16 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::Cartesi end bind(writechannel, writetask) + chunks = eachchunk(z) + try for i in 1:length(blockr) bI, chunk_compressed = take!(readchannel) - current_chunk_offsets = map((s, i)->s*(i-1), size(a), Tuple(bI)) + current_chunk_offsets = first.(chunks[bI]) .- 1 + + current_chunks_size = length.(chunks[bI]) indranges = map(boundint, r.indices, size(a), current_chunk_offsets) @@ -322,13 +335,29 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::Cartesi a end - if chunk_compressed !== nothing - uncompress_raw!(a, z, chunk_compressed) - end - curchunk .= view(ain, dotminus.(indranges, input_base_offsets)...) + if current_chunks_size != size(a) + inds_reduced = Base.OneTo.(length.(chunks[bI])) + + if chunk_compressed !== nothing + uncompress_raw!(view(a, inds_reduced...), z, chunk_compressed) + end - put!(writechannel, bI=>compress_raw(maybeinner(a), z)) + curchunk .= view(ain, dotminus.(indranges, input_base_offsets)...) + + #If the chunk size is smaller than the buffer size we need to write a smaller buffer + + inds_reduced = Base.OneTo.(length.(chunks[bI])) + put!(writechannel, bI=>compress_raw(maybeinner(a)[inds_reduced...], z)) + + else + + if chunk_compressed !== nothing + uncompress_raw!(a, z, chunk_compressed) + end + curchunk .= view(ain, dotminus.(indranges, input_base_offsets)...) + put!(writechannel, bI=>compress_raw(maybeinner(a), z)) + end nothing end finally @@ -366,7 +395,6 @@ dotminus(x, y) = x .- y function uncompress_to_output!(aout, output_base_offsets, z, chunk_compressed, current_chunk_offsets, a, indranges) uncompress_raw!(a, z, chunk_compressed) - if length.(indranges) == size(a) aout[dotminus.(indranges, output_base_offsets)...] = ndims(a) == 0 ? a[1] : a else @@ -376,7 +404,6 @@ function uncompress_to_output!(aout, output_base_offsets, z, chunk_compressed, c end function compress_raw(a, z) - length(a) == prod(DiskArrays.max_chunksize.(eachchunk(z).chunks)) || throw(DimensionMismatch("Array size does not equal chunk size")) pipeline_encode(get_pipeline(z.metadata), a, z.metadata.fill_value) end diff --git a/test/python.jl b/test/python.jl index d50b41de..644e891b 100644 --- a/test/python.jl +++ b/test/python.jl @@ -308,3 +308,64 @@ for unit in ["Week", "Day", "Hour", "Minute", "Second", end end + +@testset "Python zarr v3 irregular (rectilinear) chunks" begin + # Irregular (rectilinear) chunks are an experimental python-zarr feature, + # gated behind a config flag that must be enabled before opening/creating + # any rectilinear store. Irregular chunk grids only exist in zarr v3. + zarr = pyimport("zarr") + numpy = pyimport("numpy") + pybuiltins = pyimport("builtins") + zarr.config.set(pydict(Dict("array.rectilinear_chunks" => true))) + + import Zarr: ZarrCore + using DiskArrays: GridChunks, RegularChunks, IrregularChunks + + # Julia-side storage for irregular chunks pads every chunk to the global + # max chunk size, whereas python-zarr stores each rectilinear chunk at its + # exact data extent. The two on-disk layouts are therefore incompatible: + # - python cannot reshape Julia's padded chunks back to their extents, + # - Julia cannot decode python's exact-size chunks into its padded buffer. + # The metadata round-trips correctly in both directions; only the *data* + # read is broken. Those data assertions are marked `@test_broken` until the + # Julia chunk I/O writes exact-size chunks per grid position. + + pjulia = tempname() + ppython = tempname() + + # ---- Direction A: Julia writes an irregular grid, python reads it ---- + za = zcreate(Int32, 20, 20; zarr_format=3, path=pjulia, + chunks=GridChunks(RegularChunks(4, 0, 20), + IrregularChunks(chunksizes=[3, 4, 5, 6, 2]))) + data = reshape(Int32.(1:400), 20, 20) + za[:, :] = data + + # Metadata round-trips: python sees a rectilinear grid. + b = zarr.open_array(pjulia, mode="r") + grid = pyconvert(Dict, b.metadata.chunk_grid.to_dict()) + @test grid["name"] == "rectilinear" + @test grid["configuration"]["kind"] == "inline" + # The irregular dimension serializes as a bare int (regular) + a list. + cshapes = grid["configuration"]["chunk_shapes"] + @test cshapes[1] isa AbstractVector + @test collect(cshapes[1]) == [3, 4, 5, 6, 2] + @test cshapes[2] == 4 + + # Data read: currently broken (python cannot reshape Julia's padded chunks). + @test_broken pyconvert(Bool, (b[pybuiltins.Ellipsis] == numpy.array(data)).all()) + + # ---- Direction B: python writes an irregular grid, Julia reads it ---- + pd = reshape(Int32.(0:399), 20, 20) + a = zarr.create_array(ppython; zarr_format=3, shape=(20, 20), + chunks=pylist([pylist([3, 4, 5, 6, 2]), pylist([4, 4, 4, 4, 4])]), + dtype="int32") + a.__setitem__(pybuiltins.Ellipsis, numpy.array(pd)) + + # Metadata round-trips: Julia parses the rectilinear grid. + z = zopen(ppython) + @test any(c -> c isa IrregularChunks, ZarrCore.eachchunk(z).chunks) + + # Data read: currently broken (Julia cannot decode exact-size chunks into + # its padded buffer). + @test_broken z[:, :] == pd +end From ad2a84700b53da257f90e77fed825084f9af8d00 Mon Sep 17 00:00:00 2001 From: Fabian Gans Date: Fri, 21 Aug 2026 14:02:23 +0200 Subject: [PATCH 36/39] AI-supported update of docs and Changelog --- CHANGELOG.md | 1 + docs/src/.vitepress/config.mts | 1 + docs/src/UserGuide/chunking.md | 110 +++++++++++++++++++++++++++++++++ docs/src/get_started.md | 14 +++++ lib/ZarrCore/src/ZArray.jl | 2 +- 5 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 docs/src/UserGuide/chunking.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bf54a38..d8d92170 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased - Drop HTTP.jl 2.x support +- Support irregular (rectilinear) chunking: `zcreate`, `zzeros` and the `ZArray(a; chunks=...)` constructor now accept a `DiskArrays.GridChunks` object for the `chunks` keyword in addition to a tuple of chunk sizes. Irregular grids round-trip through Zarr v3's `rectilinear` chunk grid; Zarr v2 persists only the maximum chunk size per axis [#326](https://github.com/JuliaIO/Zarr.jl/pull/326) - Move code to ZarrCore.jl with low dependencies - Declare an explicit public API [#317](https://github.com/JuliaIO/Zarr.jl/pull/317). Every store, codec, filter and compressor type, and every documented extension point, is now `public`; the set of exported names is unchanged. Internals (`Metadata`, `ZarrFormat`, `is_zarray`, `is_zgroup`, `normalize_path`, `MaxLengthString`, ...) are no longer reachable as `Zarr.x` and must be accessed via `Zarr.ZarrCore.x` - Add logo and favicon to docs [#307](https://github.com/JuliaIO/Zarr.jl/pull/307) diff --git a/docs/src/.vitepress/config.mts b/docs/src/.vitepress/config.mts index 89761cb2..39242097 100644 --- a/docs/src/.vitepress/config.mts +++ b/docs/src/.vitepress/config.mts @@ -24,6 +24,7 @@ const userGuideItems = [ // { text: 'Data Types', link: '/UserGuide/data_types' }, // { text: 'Codecs & Performance', link: '/UserGuide/performance' }, { text: 'Operations', link: '/UserGuide/operations'}, + { text: 'Chunking', link: '/UserGuide/chunking' }, // { text: 'Sharding', link: '/UserGuide/sharding' }, { text: 'Missing Values', link: '/UserGuide/missings' }, ] diff --git a/docs/src/UserGuide/chunking.md b/docs/src/UserGuide/chunking.md new file mode 100644 index 00000000..ab51a5c3 --- /dev/null +++ b/docs/src/UserGuide/chunking.md @@ -0,0 +1,110 @@ +# Chunking and Irregular Chunk Grids + +A Zarr array is divided into chunks, each of which is compressed and stored (and +transferred) independently. The chunk size along every axis is therefore one of +the main knobs controlling read/write performance and storage granularity. + +By default chunks have a uniform size along each axis (regular chunking), which +is what the plain tuple form of the `chunks` keyword gives you: + +````jldoctest regular +julia> using Zarr + +julia> z = zzeros(Int, 100, 100; chunks=(10, 20)) +ZArray{Int64} of size 100 x 100 +```` + +Here the array is divided into chunks of 10×20 elements, evenly tiling the +array. + +## Controlling the chunk grid with `GridChunks` + +Passing a tuple only fixes the chunk size; to fully control the chunk grid you +can instead pass a `DiskArrays.GridChunks` object as the `chunks` keyword to +[`zcreate`](@ref), [`zzeros`](@ref) or the `ZArray(a::AbstractArray, ...; chunks=...)` +constructor. This is also what makes *irregular* (rectilinear) chunking +possible, where chunk sizes vary along an axis instead of being uniform. + +The chunk types and the `GridChunks` wrapper come from the +[DiskArrays.jl](https://github.com/JuliaIO/DiskArrays.jl) package, which Zarr +builds on. Bring them into scope with: + +```julia +using DiskArrays: GridChunks, IrregularChunks, RegularChunks +``` + +A `GridChunks` holds one chunk specification per axis, either a +`RegularChunks` or an `IrregularChunks`. Irregular chunks along an axis are +described by their edge lengths: + +````jldoctest irregular +julia> using Zarr + +julia> using DiskArrays: GridChunks, IrregularChunks, RegularChunks + +julia> chunks = GridChunks(RegularChunks(2, 0, 5), IrregularChunks(chunksizes=[3, 4, 5, 6, 2])); + +julia> z = zcreate(Int, 5, 20; zarr_format=3, chunks=chunks) +ZArray{Int64} of size 5 x 20 + +julia> eachchunk(z) +GridChunks( +RegularChunks(2, 0, 5)IrregularChunks([0, 3, 7, 12, 18, 20])) + +julia> z[:, :] = reshape(1:100, 5, 20); + +julia> z[:, :] == reshape(1:100, 5, 20) +true +```` + +The second axis is split into chunks of 3, 4, 5, 6 and 2 elements (20 in total), +while the first axis keeps regular chunks of size 2. + +`zzeros` works the same way; for grids that are not uniformly regular it fills +the array element by element rather than reusing a single encoded chunk: + +````jldoctest irregular +julia> zz = zzeros(Int, 10, 10; zarr_format=3, chunks=GridChunks(RegularChunks(5, 0, 10), IrregularChunks(chunksizes=[3, 3, 4]))); + +julia> zz[1, 1] +0 +```` + +## Persistence and interoperability + +Irregular chunk grids are stored in the array metadata, so they survive +round-tripping through a store. Zarr v3 records them as a `rectilinear` chunk +grid with `kind: "inline"`, where each axis is either a single chunk size +(regular) or a list of edge lengths. Reopening the array restores the exact +grid: + +````jldoctest irregular +julia> dir = joinpath(mktempdir(), "irregular.zarr"); + +julia> z = zcreate(Int, 5, 20; zarr_format=3, chunks=chunks, path=dir); + +julia> z2 = zopen(dir); + +julia> eachchunk(z2) +GridChunks( +RegularChunks(2, 0, 5)IrregularChunks([0, 3, 7, 12, 18, 20])) +```` + +::: warning + +Zarr v2 has no concept of irregular chunk grids. If you create an array with an +irregular grid and persist it as v2, the metadata only records the *maximum* +chunk size along each axis, and the array is reopened as a regular grid of that +size. Irregular chunking is therefore only preserved end-to-end with Zarr v3 +(`zarr_format=3`). + +::: + +::: warning + +`resize!` and `append!` are not supported for arrays whose grid contains +`IrregularChunks`; shrinking such an array throws an `ArgumentError`. Resizing +an irregular grid would require redefining the per-axis edge lengths, which is +not implemented. + +::: diff --git a/docs/src/get_started.md b/docs/src/get_started.md index 26dd93be..1c1805e5 100644 --- a/docs/src/get_started.md +++ b/docs/src/get_started.md @@ -80,6 +80,20 @@ z = ZArray(rand(Float64, 100, 100)) zinfo(z) ``` +For irregular chunking — where chunk sizes vary along an axis instead of being +uniform — pass a `GridChunks` object (from DiskArrays) as the `chunks` keyword +instead of a tuple of sizes: + +```@example irregular-chunks +using Zarr +using DiskArrays: GridChunks, IrregularChunks, RegularChunks +chunks = GridChunks(RegularChunks(2, 0, 5), IrregularChunks(chunksizes=[3, 4, 5, 6, 2])); +z = zcreate(Int, 5, 20; zarr_format=3, chunks=chunks) +``` + +Irregular grids are preserved end-to-end with Zarr v3. See the +[Chunking](./UserGuide/chunking) page for details and caveats. + ## Reading and Writing ```@example rw using Zarr diff --git a/lib/ZarrCore/src/ZArray.jl b/lib/ZarrCore/src/ZArray.jl index 7d4c1470..95f6be1c 100644 --- a/lib/ZarrCore/src/ZArray.jl +++ b/lib/ZarrCore/src/ZArray.jl @@ -418,7 +418,7 @@ Creates a new empty zarr array with element type `T` and array dimensions `dims` * `name=""` name of the zarr array, defaults to the directory name * `zarr_format`=$(DV) Zarr format version (2 or 3) * `storagetype` determines the storage to use, current options are `DirectoryStore` or `DictStore` -* `chunks=dims` size of the individual array chunks, must be a tuple of length `length(dims)` +* `chunks=dims` size of the individual array chunks. Either a tuple of length `length(dims)` specifying the chunk size along each axis (regular chunking), or a `DiskArrays.GridChunks` object for full control over the chunk grid, which also enables irregular (rectilinear) chunking where chunk sizes vary along an axis. See [Chunking](../UserGuide/chunking) for details. * `fill_value=nothing` value to represent missing values * `fill_as_missing=false` set to `true` shall fillvalue s be converted to `missing`s * `filters`=filters to be applied From c51e91beafb05072890671d453c32bd793b56ef2 Mon Sep 17 00:00:00 2001 From: Fabian Gans Date: Fri, 21 Aug 2026 16:22:18 +0200 Subject: [PATCH 37/39] Attempt at fixing last python interop incompatibility --- lib/ZarrCore/src/ZArray.jl | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/ZarrCore/src/ZArray.jl b/lib/ZarrCore/src/ZArray.jl index 95f6be1c..68f6bd1e 100644 --- a/lib/ZarrCore/src/ZArray.jl +++ b/lib/ZarrCore/src/ZArray.jl @@ -143,6 +143,10 @@ function boundint(r1, s2, o2) UnitRange(f1 > f2 ? f1 : f2, l1 < l2 ? l1 : l2) end +chunk_unbounded(gc::GridChunks, i) = map(chunk_unbounded, gc.chunks, i.I) +chunk_unbounded(r::RegularChunks, i) = ((i-1)*r.chunksize+1-r.offset):(i*r.chunksize-r.offset) +chunk_unbounded(chunks::IrregularChunks, i) = (chunks.offsets[i]+1):chunks.offsets[i+1] + function getchunkarray(z::ZArray{>:Missing}) # temporary workaround to use strings as data values inner = fill(z.metadata.fill_value, DiskArrays.max_chunksize.(eachchunk(z).chunks)) @@ -259,9 +263,11 @@ function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::Cartesi bI, chunk_compressed = take!(c) - current_chunks_size = length.(chunks[bI]) + current_chunk_inds = chunk_unbounded(chunks, bI) + + current_chunks_size = length.(current_chunk_inds) - current_chunk_offsets = first.(chunks[bI]) .- 1 + current_chunk_offsets = first.(current_chunk_inds) .- 1 indranges = map(boundint, r.indices, current_chunks_size, current_chunk_offsets) #If the chunk size is smaller than the buffer size we need to write a smaller buffer @@ -319,9 +325,11 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::Cartesi bI, chunk_compressed = take!(readchannel) - current_chunk_offsets = first.(chunks[bI]) .- 1 + current_chunk_inds = chunk_unbounded(chunks, bI) + + current_chunk_offsets = first.(current_chunk_inds) .- 1 - current_chunks_size = length.(chunks[bI]) + current_chunks_size = length.(current_chunk_inds) indranges = map(boundint, r.indices, size(a), current_chunk_offsets) @@ -337,7 +345,7 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::Cartesi if current_chunks_size != size(a) - inds_reduced = Base.OneTo.(length.(chunks[bI])) + inds_reduced = Base.OneTo.(length.(current_chunk_inds)) if chunk_compressed !== nothing uncompress_raw!(view(a, inds_reduced...), z, chunk_compressed) @@ -347,7 +355,7 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::Cartesi #If the chunk size is smaller than the buffer size we need to write a smaller buffer - inds_reduced = Base.OneTo.(length.(chunks[bI])) + inds_reduced = Base.OneTo.(length.(current_chunk_inds)) put!(writechannel, bI=>compress_raw(maybeinner(a)[inds_reduced...], z)) else @@ -629,7 +637,7 @@ function prune_oob_chunks(s::AbstractStore, path, oldsize, newsize, chunks::Disk for idim in dimstoshorten nchunks_new = DiskArrays.findchunk(chunks.chunks[idim], max(newsize[idim], 1)) nchunks_old = DiskArrays.findchunk(chunks.chunks[idim], oldsize[idim]) - delrange = (nchunks_new + 1):nchunks_old + delrange = (nchunks_new+1):nchunks_old allchunkranges = map(i -> 1:DiskArrays.findchunk(chunks.chunks[i], oldsize[i]), 1:length(oldsize)) r = (allchunkranges[1:(idim-1)]..., delrange, allchunkranges[(idim+1):end]...) for cI in CartesianIndices(r) From e06f9558ccf1bc1a3fac503a6c3d424899f19149 Mon Sep 17 00:00:00 2001 From: Fabian Gans Date: Fri, 21 Aug 2026 16:26:43 +0200 Subject: [PATCH 38/39] fix one broken test --- test/python.jl | 564 ++++++++++++++++++++++++------------------------- 1 file changed, 282 insertions(+), 282 deletions(-) diff --git a/test/python.jl b/test/python.jl index bb240130..580626c0 100644 --- a/test/python.jl +++ b/test/python.jl @@ -12,309 +12,309 @@ CondaPkg.add([ @testset "Python zarr implementation" begin -import Mmap -using PythonCall -#If we are on conda, import zarr -zarr = pyimport("zarr") -zarr_storage = pyimport("zarr.storage") - -#Create some directories -proot = tempname() -mkpath(proot) -pjulia = joinpath(proot,"julia") -ppython = joinpath(proot,"python") - -#First create an array in Julia and read with python zarr -groupattrs = Dict("String attribute"=>"One", "Int attribute"=>5, "Float attribute"=>10.5) -g = zgroup(pjulia,attrs=groupattrs) - -# Test all supported data types and compressors -import Zarr: NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor, - Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter -import Zarr: ZarrCore -using Random: randstring -numeric_dtypes = (UInt8, UInt16, UInt32, UInt64, - Int8, Int16, Int32, Int64, - Float16, Float32, Float64, - Complex{Float32}, Complex{Float64}, - Bool,) -dtypes = (numeric_dtypes..., - ZarrCore.MaxLengthString{10,UInt8},ZarrCore.MaxLengthString{10,UInt32}, - String) -dtypesp = ("uint8","uint16","uint32","uint64", - "int8","int16","int32","int64", - "float16","float32","float64", - "complex64", "complex128","bool","S10","U10", "O") -compressors = ( - "no"=>NoCompressor(), - "blosc"=>BloscCompressor(cname="zstd"), - "blosc_autoshuffle"=>BloscCompressor(cname="zstd",shuffle=-1), - "blosc_noshuffle"=>BloscCompressor(cname="zstd",shuffle=0), - "blosc_bitshuffle"=>BloscCompressor(cname="zstd",shuffle=2), - "zlib"=>ZlibCompressor(), - "zlib_2"=>ZlibCompressor(;clevel=2), - "zstd"=>ZstdCompressor(), -) -filters = ( - "fletcher32"=>Fletcher32Filter(), - "scale_offset"=>FixedScaleOffsetFilter(offset=1000, scale=10^6, T=Float64, Tenc=Int32), - "shuffle"=>ShuffleFilter(elementsize=4), - "quantize"=>QuantizeFilter{Float64,Float32}(digits=5), - "delta"=>DeltaFilter{Int32}() -) -testarrays = Dict(t=>(t<:AbstractString) ? [randstring(maximum(i.I)) for i in CartesianIndices((1:10,1:6,1:2))] : rand(t,10,6,2) for t in dtypes) -testzerodimarrays = Dict(t=>(t<:AbstractString) ? randstring(10) : rand(t) for t in dtypes) - -# Test arrays with compressors -for t in dtypes, co in compressors - compstr, comp = co - att = Dict("This is a nested attribute"=>Dict("a"=>5)) - a = zcreate(t, g,string("a",t,compstr),10,6,2,attrs=att, chunks = (5,2,2),compressor=comp) - a[:,:,:] = testarrays[t] - - a = zcreate(t, g,string("azerodim",t,compstr), compressor=comp) - a[] = testzerodimarrays[t] -end + import Mmap + using PythonCall + #If we are on conda, import zarr + zarr = pyimport("zarr") + zarr_storage = pyimport("zarr.storage") -# Test arrays with filters -for (filterstr, filter) in filters - t = eltype(filter) == Any ? Float64 : eltype(filter) - att = Dict("Filter test attribute"=>Dict("b"=>6)) - a = zcreate(t, g,string("filter_",filterstr),10,6,2,attrs=att, chunks = (5,2,2),filters=[filter]) - testdata = rand(t,10,6,2) - a[:,:,:] = testdata - - # Test zero-dimensional array - a = zcreate(t, g,string("filter_zerodim_",filterstr), filters=[filter]) - testzerodim = rand(t) - a[] = testzerodim -end + #Create some directories + proot = tempname() + mkpath(proot) + pjulia = joinpath(proot, "julia") + ppython = joinpath(proot, "python") -#Also save as zip file. -open(pjulia*".zip";write=true) do io - Zarr.writezip(io, g) -end + #First create an array in Julia and read with python zarr + groupattrs = Dict("String attribute"=>"One", "Int attribute"=>5, "Float attribute"=>10.5) + g = zgroup(pjulia, attrs=groupattrs) -@testset "reading in julia" begin - g = zopen(pjulia) - #Test group attributes - @test g.attrs == groupattrs - for (t, co) in Iterators.product(dtypes, compressors) - compstr,comp = co - arname = string("a",t,compstr) - ar = g[arname] - @test ar.attrs == Dict("This is a nested attribute"=>Dict("a"=>5)) - @test ar == testarrays[t] + # Test all supported data types and compressors + import Zarr: NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor, + Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter + import Zarr: ZarrCore + using Random: randstring + numeric_dtypes = (UInt8, UInt16, UInt32, UInt64, + Int8, Int16, Int32, Int64, + Float16, Float32, Float64, + Complex{Float32}, Complex{Float64}, + Bool,) + dtypes = (numeric_dtypes..., + ZarrCore.MaxLengthString{10,UInt8}, ZarrCore.MaxLengthString{10,UInt32}, + String) + dtypesp = ("uint8", "uint16", "uint32", "uint64", + "int8", "int16", "int32", "int64", + "float16", "float32", "float64", + "complex64", "complex128", "bool", "S10", "U10", "O") + compressors = ( + "no"=>NoCompressor(), + "blosc"=>BloscCompressor(cname="zstd"), + "blosc_autoshuffle"=>BloscCompressor(cname="zstd", shuffle=-1), + "blosc_noshuffle"=>BloscCompressor(cname="zstd", shuffle=0), + "blosc_bitshuffle"=>BloscCompressor(cname="zstd", shuffle=2), + "zlib"=>ZlibCompressor(), + "zlib_2"=>ZlibCompressor(; clevel=2), + "zstd"=>ZstdCompressor(), + ) + filters = ( + "fletcher32"=>Fletcher32Filter(), + "scale_offset"=>FixedScaleOffsetFilter(offset=1000, scale=10^6, T=Float64, Tenc=Int32), + "shuffle"=>ShuffleFilter(elementsize=4), + "quantize"=>QuantizeFilter{Float64,Float32}(digits=5), + "delta"=>DeltaFilter{Int32}() + ) + testarrays = Dict(t=>(t<:AbstractString) ? [randstring(maximum(i.I)) for i in CartesianIndices((1:10, 1:6, 1:2))] : rand(t, 10, 6, 2) for t in dtypes) + testzerodimarrays = Dict(t=>(t<:AbstractString) ? randstring(10) : rand(t) for t in dtypes) + + # Test arrays with compressors + for t in dtypes, co in compressors + compstr, comp = co + att = Dict("This is a nested attribute"=>Dict("a"=>5)) + a = zcreate(t, g, string("a", t, compstr), 10, 6, 2, attrs=att, chunks=(5, 2, 2), compressor=comp) + a[:, :, :] = testarrays[t] + + a = zcreate(t, g, string("azerodim", t, compstr), compressor=comp) + a[] = testzerodimarrays[t] end -end -# Test reading in python -for julia_path in (pjulia, pjulia*".zip") - # zarr-python 3.x requires explicitly opening zip files as ZipStore - if endswith(julia_path, ".zip") - store = zarr_storage.ZipStore(julia_path) - g = zarr.open_group(store=store, mode="r") - else - g = zarr.open_group(julia_path, mode="r") + # Test arrays with filters + for (filterstr, filter) in filters + t = eltype(filter) == Any ? Float64 : eltype(filter) + att = Dict("Filter test attribute"=>Dict("b"=>6)) + a = zcreate(t, g, string("filter_", filterstr), 10, 6, 2, attrs=att, chunks=(5, 2, 2), filters=[filter]) + testdata = rand(t, 10, 6, 2) + a[:, :, :] = testdata + + # Test zero-dimensional array + a = zcreate(t, g, string("filter_zerodim_", filterstr), filters=[filter]) + testzerodim = rand(t) + a[] = testzerodim end - gatts = pyconvert(Any, g.attrs) - - #Test group attributes - @test gatts["String attribute"] == "One" - @test gatts["Int attribute"] == 5 - @test gatts["Float attribute"] == 10.5 - - #Test accessing arrays from python and reading data - for i=1:length(dtypes), co in compressors - compstr,comp = co - t = dtypes[i] - tp = dtypesp[i] - # zarr-python 3.x does not support fixed-length (5) - @test pyeq(Bool, ar.dtype, tp) - @test pyconvert(Tuple, ar.shape) == (2,6,10) - @test PyArray(ar[pybuiltins.Ellipsis]) == permutedims(testarrays[t],(3,2,1)) + + #Also save as zip file. + open(pjulia*".zip"; write=true) do io + Zarr.writezip(io, g) end - # Test reading filtered arrays from python - for (filterstr, filter) in filters - t = eltype(filter) == Any ? Float64 : eltype(filter) - arname = string("filter_",filterstr) - local ar - try - ar=g[arname] - catch e - @error "Error loading group with filter $filterstr" exception=(e,catch_backtrace()) - @test false # test failed. + @testset "reading in julia" begin + g = zopen(pjulia) + #Test group attributes + @test g.attrs == groupattrs + for (t, co) in Iterators.product(dtypes, compressors) + compstr, comp = co + arname = string("a", t, compstr) + ar = g[arname] + @test ar.attrs == Dict("This is a nested attribute"=>Dict("a"=>5)) + @test ar == testarrays[t] end - - @test pyconvert(Any, ar.attrs["Filter test attribute"]) == Dict("b"=>6) - @test pyconvert(Tuple, ar.shape) == (2,6,10) - - # Test zero-dimensional filtered array - arname = string("filter_zerodim_",filterstr) - ar_zero=g[arname] - @test pyconvert(Tuple, ar_zero.shape) == () end - for i=1:length(dtypes), co in compressors - compstr,comp = co - t = dtypes[i] - tp = dtypesp[i] - if t == UInt64 - continue - # need to exclude UInt64: - # need explicit conversion because of https://github.com/JuliaPy/PyCall.jl/issues/744 - # but explicit conversion uses PyLong_AsLongLongAndOverflow, which converts everything - # to a signed 64-bit integer, which can error out if the UInt64 is too large. - # Adding an overload to PyCall for unsigned ints doesn't work with NumPy scalars because - # they are not subtypes of integer: https://stackoverflow.com/a/58816671 + # Test reading in python + for julia_path in (pjulia, pjulia*".zip") + # zarr-python 3.x requires explicitly opening zip files as ZipStore + if endswith(julia_path, ".zip") + store = zarr_storage.ZipStore(julia_path) + g = zarr.open_group(store=store, mode="r") + else + g = zarr.open_group(julia_path, mode="r") end - # zarr-python 3.x does not support fixed-length (5) + @test pyeq(Bool, ar.dtype, tp) + @test pyconvert(Tuple, ar.shape) == (2, 6, 10) + @test PyArray(ar[pybuiltins.Ellipsis]) == permutedims(testarrays[t], (3, 2, 1)) + end - @test pyeq(Bool, ar.dtype, tp) - @test pyconvert(Tuple, ar.shape) == () - @test pyconvert(Any, ar[()])[] == testzerodimarrays[t] - end - # Close ZipStore if opened - if endswith(julia_path, ".zip") - store.close() + # Test reading filtered arrays from python + for (filterstr, filter) in filters + t = eltype(filter) == Any ? Float64 : eltype(filter) + arname = string("filter_", filterstr) + local ar + try + ar=g[arname] + catch e + @error "Error loading group with filter $filterstr" exception=(e, catch_backtrace()) + @test false # test failed. + end + + @test pyconvert(Any, ar.attrs["Filter test attribute"]) == Dict("b"=>6) + @test pyconvert(Tuple, ar.shape) == (2, 6, 10) + + # Test zero-dimensional filtered array + arname = string("filter_zerodim_", filterstr) + ar_zero=g[arname] + @test pyconvert(Tuple, ar_zero.shape) == () + end + + for i=1:length(dtypes), co in compressors + compstr, comp = co + t = dtypes[i] + tp = dtypesp[i] + if t == UInt64 + continue + # need to exclude UInt64: + # need explicit conversion because of https://github.com/JuliaPy/PyCall.jl/issues/744 + # but explicit conversion uses PyLong_AsLongLongAndOverflow, which converts everything + # to a signed 64-bit integer, which can error out if the UInt64 is too large. + # Adding an overload to PyCall for unsigned ints doesn't work with NumPy scalars because + # they are not subtypes of integer: https://stackoverflow.com/a/58816671 + end + # zarr-python 3.x does not support fixed-length ("Hi"))) -z1 = g.require_array("a1", shape=(2,6,10), chunks=(1,2,3), dtype="i4") -z1[pybuiltins.Ellipsis] = numpy.array(data) -z1.update_attributes(pydict(Dict("test" => pydict(Dict("b" => 6))))) -z2 = g.require_array("a2", shape=(5,), chunks=(5,), dtype="S1", compressor=numcodecs.Zlib()) -z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"]) -z3 = g.require_array("a3", shape=(2,), dtype=pybuiltins.str) -z3[pybuiltins.Ellipsis]=numpy.asarray(["test1", "test234"], dtype="O") -z4 = g.require_array("a4", shape=(2,3), chunks=(2,3), dtype="c16", fill_value=cfill) -z4[pybuiltins.Ellipsis] = numpy.array(cdata) -zarr.consolidate_metadata(ppython) - -#Open in Julia -g = zopen(ppython) -@test g isa Zarr.ZGroup -@test g.attrs["groupatt"] == "Hi" -a1 = g["a1"] -@test a1 isa ZArray -@test a1[:,:,:]==permutedims(data,(3,2,1)) -@test a1.attrs["test"]==Dict("b"=>6) -# Test reading the string array -@test String(g["a2"][:])=="hallo" -@test g["a3"] == ["test1", "test234"] -a4 = g["a4"] -@test eltype(a4) === ComplexF64 -@test a4.metadata.fill_value === cfill -@test a4[:,:] == permutedims(cdata,(2,1)) - -# And test for consolidated metadata -# Delete files so we make sure they are not accessed -rm(joinpath(ppython,".zattrs")) -rm(joinpath(ppython,"a1",".zattrs")) -rm(joinpath(ppython,"a1",".zarray")) -rm(joinpath(ppython,"a2",".zarray")) -g = zopen(ppython, "w", consolidated=true) -@test g isa Zarr.ZGroup -@test g.attrs["groupatt"] == "Hi" -a1 = g["a1"] -@test a1 isa ZArray -@test a1[:,:,:]==permutedims(data,(3,2,1)) -@test a1.attrs["test"]==Dict("b"=>6) -@test storagesize(a1) == 960 -@test sort(Zarr.subkeys(a1.storage,"a1"))[1:5] == ["0.0.0","0.0.1","0.0.2","0.0.3","0.1.0"] -a1[:,1,1] = 1:10 -@test a1[:,1,1] == 1:10 -# Test reading the string array -@test String(g["a2"][:])=="hallo" - - -# Test zip file can be read -ppythonzip = ppython*".zip" -store = zarr_storage.ZipStore(ppythonzip, mode="w") -g = zarr.open_group(store=store, mode="w", zarr_format=2) -g.update_attributes(pydict(Dict("groupatt"=>"Hi"))) -z1 = g.require_array("a1", shape=(2,6,10), chunks=(1,2,3), dtype="i4") -z1[pybuiltins.Ellipsis] = numpy.array(data) -z1.update_attributes(pydict(Dict("test" => pydict(Dict("b" => 6))))) -z2 = g.require_array("a2", shape=(5,), chunks=(5,), dtype="S1", compressor=numcodecs.Zlib()) -z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"]) -z3 = g.require_array("a3", shape=(2,), dtype=pybuiltins.str) -z3[pybuiltins.Ellipsis] = numpy.asarray(["test1", "test234"], dtype="O") -store.close() - -g = zopen(Zarr.ZipStore(Mmap.mmap(ppythonzip))) -@test g isa Zarr.ZGroup -@test g.attrs["groupatt"] == "Hi" -a1 = g["a1"] -@test a1 isa ZArray -@test a1[:,:,:]==permutedims(data,(3,2,1)) -@test a1.attrs["test"]==Dict("b"=>6) -# Test reading the string array -@test String(g["a2"][:])=="hallo" -@test g["a3"] == ["test1", "test234"] + ## Now the other way around, we create a zarr array using the python lib and read back into julia + data = rand(Int32, 2, 6, 10) + cdata = rand(ComplexF64, 2, 3) + cfill = ComplexF64(1.5, -2.5) + + numpy = pyimport("numpy") + numcodecs = pyimport("numcodecs") + g = zarr.open_group(ppython, mode="w", zarr_format=2) + g.update_attributes(pydict(Dict("groupatt"=>"Hi"))) + z1 = g.require_array("a1", shape=(2, 6, 10), chunks=(1, 2, 3), dtype="i4") + z1[pybuiltins.Ellipsis] = numpy.array(data) + z1.update_attributes(pydict(Dict("test" => pydict(Dict("b" => 6))))) + z2 = g.require_array("a2", shape=(5,), chunks=(5,), dtype="S1", compressor=numcodecs.Zlib()) + z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"]) + z3 = g.require_array("a3", shape=(2,), dtype=pybuiltins.str) + z3[pybuiltins.Ellipsis]=numpy.asarray(["test1", "test234"], dtype="O") + z4 = g.require_array("a4", shape=(2, 3), chunks=(2, 3), dtype="c16", fill_value=cfill) + z4[pybuiltins.Ellipsis] = numpy.array(cdata) + zarr.consolidate_metadata(ppython) + + #Open in Julia + g = zopen(ppython) + @test g isa Zarr.ZGroup + @test g.attrs["groupatt"] == "Hi" + a1 = g["a1"] + @test a1 isa ZArray + @test a1[:, :, :]==permutedims(data, (3, 2, 1)) + @test a1.attrs["test"]==Dict("b"=>6) + # Test reading the string array + @test String(g["a2"][:])=="hallo" + @test g["a3"] == ["test1", "test234"] + a4 = g["a4"] + @test eltype(a4) === ComplexF64 + @test a4.metadata.fill_value === cfill + @test a4[:, :] == permutedims(cdata, (2, 1)) + + # And test for consolidated metadata + # Delete files so we make sure they are not accessed + rm(joinpath(ppython, ".zattrs")) + rm(joinpath(ppython, "a1", ".zattrs")) + rm(joinpath(ppython, "a1", ".zarray")) + rm(joinpath(ppython, "a2", ".zarray")) + g = zopen(ppython, "w", consolidated=true) + @test g isa Zarr.ZGroup + @test g.attrs["groupatt"] == "Hi" + a1 = g["a1"] + @test a1 isa ZArray + @test a1[:, :, :]==permutedims(data, (3, 2, 1)) + @test a1.attrs["test"]==Dict("b"=>6) + @test storagesize(a1) == 960 + @test sort(Zarr.subkeys(a1.storage, "a1"))[1:5] == ["0.0.0", "0.0.1", "0.0.2", "0.0.3", "0.1.0"] + a1[:, 1, 1] = 1:10 + @test a1[:, 1, 1] == 1:10 + # Test reading the string array + @test String(g["a2"][:])=="hallo" + + + # Test zip file can be read + ppythonzip = ppython*".zip" + store = zarr_storage.ZipStore(ppythonzip, mode="w") + g = zarr.open_group(store=store, mode="w", zarr_format=2) + g.update_attributes(pydict(Dict("groupatt"=>"Hi"))) + z1 = g.require_array("a1", shape=(2, 6, 10), chunks=(1, 2, 3), dtype="i4") + z1[pybuiltins.Ellipsis] = numpy.array(data) + z1.update_attributes(pydict(Dict("test" => pydict(Dict("b" => 6))))) + z2 = g.require_array("a2", shape=(5,), chunks=(5,), dtype="S1", compressor=numcodecs.Zlib()) + z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"]) + z3 = g.require_array("a3", shape=(2,), dtype=pybuiltins.str) + z3[pybuiltins.Ellipsis] = numpy.asarray(["test1", "test234"], dtype="O") + store.close() + + g = zopen(Zarr.ZipStore(Mmap.mmap(ppythonzip))) + @test g isa Zarr.ZGroup + @test g.attrs["groupatt"] == "Hi" + a1 = g["a1"] + @test a1 isa ZArray + @test a1[:, :, :]==permutedims(data, (3, 2, 1)) + @test a1.attrs["test"]==Dict("b"=>6) + # Test reading the string array + @test String(g["a2"][:])=="hallo" + @test g["a3"] == ["test1", "test234"] end @testset "Python datetime types" begin -using Dates, Test, Zarr, PythonCall -using DateTimes64: DateTime64 -vd = Date(1970,1,1):Day(1):Date(1970,6,30) |> collect -vt = DateTime(1970,1,1):Second(1):DateTime(1970,1,1,2,0,0)|> collect -ad = ZArray(vd) -at = ZArray(vt) -@test eltype(ad)==DateTime64{Day} -@test eltype(at)==DateTime64{Millisecond} -@test DateTime.(at[:]) == vt[:] -@test Date.(ad[:]) == vd[:] - -p = tempname() -g = zgroup(p) -for pt in [Week, Day, Hour, Minute, Second, + using Dates, Test, Zarr, PythonCall + using DateTimes64: DateTime64 + vd = Date(1970, 1, 1):Day(1):Date(1970, 6, 30) |> collect + vt = DateTime(1970, 1, 1):Second(1):DateTime(1970, 1, 1, 2, 0, 0) |> collect + ad = ZArray(vd) + at = ZArray(vt) + @test eltype(ad)==DateTime64{Day} + @test eltype(at)==DateTime64{Millisecond} + @test DateTime.(at[:]) == vt[:] + @test Date.(ad[:]) == vd[:] + + p = tempname() + g = zgroup(p) + for pt in [Week, Day, Hour, Minute, Second, Millisecond] - - if pt <: DatePeriod - vd = range(Date(1970,1,1),step = pt(1), length=100) - a = zcreate(DateTime64{pt},g,string(pt),100) - a[:] = vd - else - vd = range(DateTime(1970,1,1),step = pt(1), length=100) - a = zcreate(DateTime64{pt},g,string(pt),100) - a[:] = vd + + if pt <: DatePeriod + vd = range(Date(1970, 1, 1), step=pt(1), length=100) + a = zcreate(DateTime64{pt}, g, string(pt), 100) + a[:] = vd + else + vd = range(DateTime(1970, 1, 1), step=pt(1), length=100) + a = zcreate(DateTime64{pt}, g, string(pt), 100) + a[:] = vd + end end -end -zarr = pyimport("zarr") -numpy = pyimport("numpy") -g_julia = zopen(p) -g_python = zarr.open(p) + zarr = pyimport("zarr") + numpy = pyimport("numpy") + g_julia = zopen(p) + g_python = zarr.open(p) -for unit in ["Week", "Day", "Hour", "Minute", "Second", + for unit in ["Week", "Day", "Hour", "Minute", "Second", "Millisecond"] - for i in [0, 9, 99] - @test pyeq(Bool, numpy.datetime64(g_julia[unit][i+1] |> DateTime |> string), g_python[unit][i]) + for i in [0, 9, 99] + @test pyeq(Bool, numpy.datetime64(g_julia[unit][i+1] |> DateTime |> string), g_python[unit][i]) + end end -end end @@ -344,8 +344,8 @@ end # ---- Direction A: Julia writes an irregular grid, python reads it ---- za = zcreate(Int32, 20, 20; zarr_format=3, path=pjulia, - chunks=GridChunks(RegularChunks(4, 0, 20), - IrregularChunks(chunksizes=[3, 4, 5, 6, 2]))) + chunks=GridChunks(RegularChunks(4, 0, 20), + IrregularChunks(chunksizes=[3, 4, 5, 6, 2]))) data = reshape(Int32.(1:400), 20, 20) za[:, :] = data @@ -366,8 +366,8 @@ end # ---- Direction B: python writes an irregular grid, Julia reads it ---- pd = reshape(Int32.(0:399), 20, 20) a = zarr.create_array(ppython; zarr_format=3, shape=(20, 20), - chunks=pylist([pylist([3, 4, 5, 6, 2]), pylist([4, 4, 4, 4, 4])]), - dtype="int32") + chunks=pylist([pylist([3, 4, 5, 6, 2]), pylist([4, 4, 4, 4, 4])]), + dtype="int32") a.__setitem__(pybuiltins.Ellipsis, numpy.array(pd)) # Metadata round-trips: Julia parses the rectilinear grid. @@ -376,5 +376,5 @@ end # Data read: currently broken (Julia cannot decode exact-size chunks into # its padded buffer). - @test_broken z[:, :] == pd + @test z[:, :] == pd end From 545def036a4d8b27384f921176f2bd4a802e2eda Mon Sep 17 00:00:00 2001 From: Fabian Gans Date: Fri, 21 Aug 2026 16:41:38 +0200 Subject: [PATCH 39/39] Fix a broken test --- test/python.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/python.jl b/test/python.jl index 580626c0..b604c15f 100644 --- a/test/python.jl +++ b/test/python.jl @@ -376,5 +376,5 @@ end # Data read: currently broken (Julia cannot decode exact-size chunks into # its padded buffer). - @test z[:, :] == pd + @test permutedims(z[:, :], (2, 1)) == pd end