diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0e723bb5..690b2ee3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -19,10 +19,15 @@ jobs: - 'pre' os: - ubuntu-latest - - macOS-latest - - windows-latest arch: - 'default' + include: + - os: macOS-latest + version: '1' + arch: 'default' + - os: windows-latest + version: '1' + arch: 'default' steps: - uses: actions/checkout@v7 - uses: julia-actions/setup-julia@v3 @@ -31,6 +36,18 @@ jobs: arch: ${{ matrix.arch }} show-versioninfo: true - uses: julia-actions/cache@v3 + # Pkg only understands `[sources]`/`[workspace]` from Julia 1.11 on, so on + # LTS (1.10) the unregistered path dependency on ZarrCore is invisible and + # resolution fails with "expected package ZarrCore to be registered". + # Dev'ing it explicitly fixes that; on 1.11+ it is a no-op because + # `[sources]` already points at the same path. This only affects developing + # from a checkout -- once both packages are registered, users resolve + # ZarrCore normally on any supported Julia. + - name: Develop ZarrCore (Pkg < 1.11 ignores [sources]) + # bash on every platform: PowerShell mangles the embedded double quotes + # when forwarding them to a native executable. + shell: bash + run: julia --project=. -e 'using Pkg; Pkg.develop(path="lib/ZarrCore")' - uses: julia-actions/julia-buildpkg@v1 env: PYTHON: @@ -41,11 +58,25 @@ jobs: with: timeout_minutes: 5 max_attempts: 3 + # bash on every platform, for the same quoting reason as above (and so + # `rm`-style POSIX commands added here behave consistently). + shell: bash command: | - julia --project=test -e 'using Pkg; Pkg.develop(path=pwd()); Pkg.resolve(); Pkg.instantiate()' + julia --project=test -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path="lib/ZarrCore")]); Pkg.resolve(); Pkg.instantiate()' julia --project=test test/v3_julia.jl julia --project=test test/v3_python.jl - rm test/Manifest.toml + # The step above dev's ZarrCore into the test environment so the fixture + # scripts can `using Zarr` on LTS, which makes ZarrCore a *direct* dep of + # test/Project.toml. `Pkg.test` then builds its sandbox by merging the + # active manifest (where ZarrCore is also dev'd, hence fixed) with the + # fixed deps of the test manifest, and refuses to merge a package that + # appears in both: "ERROR: can not merge projects". Dropping the test + # manifest leaves nothing to merge; the sandbox resolves ZarrCore from the + # active manifest instead. On 1.11+ the workspace keeps the only manifest + # at the repo root, so this is a no-op there. + - name: Drop the test manifest so Pkg.test can build its sandbox + shell: bash + run: rm -f test/Manifest.toml - uses: julia-actions/julia-runtest@v1 env: PYTHON: diff --git a/.gitignore b/.gitignore index 85903261..4abf6567 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ Manifest.toml +Manifest-v1.12.toml docs/build *.zarr .CondaPkg diff --git a/CHANGELOG.md b/CHANGELOG.md index 291c1821..0da669a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog ## Unreleased +- Drop HTTP.jl 2.x support +- Support irregular (rectilinear) chunking: `zcreate`, `zzeros` and the `ZArray(a; chunks=...)` constructor now accept a `DiskArrays.GridChunks` object for the `chunks` keyword in addition to a tuple of chunk sizes. Irregular grids round-trip through Zarr v3's `rectilinear` chunk grid; Zarr v2 persists only the maximum chunk size per axis [#326](https://github.com/JuliaIO/Zarr.jl/pull/326) +- Move code to ZarrCore.jl with low dependencies +- Declare an explicit public API [#317](https://github.com/JuliaIO/Zarr.jl/pull/317). Every store, codec, filter and compressor type, and every documented extension point, is now `public`; the set of exported names is unchanged. Internals (`Metadata`, `ZarrFormat`, `is_zarray`, `is_zgroup`, `normalize_path`, `MaxLengthString`, ...) are no longer reachable as `Zarr.x` and must be accessed via `Zarr.ZarrCore.x` ## v0.10.2 - 2026-08-19 diff --git a/CLAUDE.md b/CLAUDE.md index dfbdb435..83c4956c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,11 +16,20 @@ julia --project -e 'using Pkg; Pkg.test()' julia --project=test -e 'using Test, Zarr, JSON; include("test/v3_codecs.jl")' # Instantiate test dependencies (after Julia version change or first time setup) -julia --project=test -e 'using Pkg; Pkg.instantiate()' +julia --project=test -e 'using Pkg; Pkg.develop(path=pwd()); Pkg.resolve(); Pkg.instantiate()' + +# Generate the v3 test fixtures (required before running the suite; CI does this +# in a separate step, `runtests.jl` does not do it for you) +julia --project=test test/v3_julia.jl +julia --project=test test/v3_python.jl ``` Julia version requirement: 1.10+. CI tests against Julia LTS, stable (`1`), nightly, and pre-release on Ubuntu, macOS, and Windows. +Because 1.10 is still supported, the `public` keyword (Julia 1.11+) cannot be used directly. Use the `ZarrCore.@public` macro instead — it expands to `public` on 1.11+ and to nothing on 1.10. + +Since there is no `public` on 1.10, `names()` cannot report public names there, so `@public` *also* appends to the calling module's `PUBLIC_NAMES::Vector{Symbol}`. **Any module that uses `@public` must define its own `const PUBLIC_NAMES = Symbol[]`** (per-module, exactly like `public` itself); forgetting it is a load-time `UndefVarError`. The `Zarr` facade unions `names(ZarrCore)` with `ZarrCore.PUBLIC_NAMES`, which is what keeps the public API present on LTS — without it, every public-but-not-exported name silently vanishes from `Zarr` on 1.10 while 1.11+ looks fine. + ## Architecture ### Core Type Hierarchy @@ -43,16 +52,35 @@ ZArray{T,N,S<:AbstractStore,M<:AbstractMetadata} <: AbstractDiskArray{T,N} ZGroup{S<:AbstractStore} ``` +### Package Layout + +The repo is a Pkg workspace with two packages: + +- `ZarrCore` (`lib/ZarrCore/`) — the entire implementation. Every type, method and docstring lives here. +- `Zarr` (`src/Zarr.jl`) — a thin facade that re-exports `ZarrCore`'s API. It mirrors the export/public split: names `ZarrCore` exports are re-exported, names it only marks `@public` stay public (not exported). Nothing else is forwarded, so internals must be reached as `Zarr.ZarrCore.foo`. + +### Public API Policy + +Set by [a maintainer comment on PR #317](https://github.com/JuliaIO/Zarr.jl/pull/317#issuecomment-5314176722). Anything a downstream consumer could need, plus every documented extension point, is part of the public API; everything else is internal and may change. + +- **Exported** (in scope after `using Zarr`): `ZArray`, `ZGroup`, `zopen`, `zzeros`, `zcreate`, `zgroup`, `zarrcache`, `storagesize`, `storageratio`, `zinfo`, `DirectoryStore`, `S3Store`, `GCStore`. +- **Public but not exported**: every store, codec, filter and compressor type; the store/filter/compressor/codec/chunk-key-encoding extension interfaces; `typestr`, `fill_value_encoding`, `fill_value_decoding`, `zname`, `writezip`, `consolidate_metadata`, `DateTime64`. +- **Internal**: `Metadata`/`MetadataV2`/`MetadataV3`, `ZarrFormat`, `DV`, `is_zarray`, `is_zgroup`, `normalize_path`, `MaxLengthString`, `ASCIIChar`, `ShapeOnlyArray` (should be removed), `getattrs`/`writeattrs`/`getmetadata`/`writemetadata`, `V2Pipeline`/`V3Pipeline`/`pipeline_encode`/`pipeline_decode!`, and the `store_*` helpers. + +Tests follow the same rule: public names are used as `Zarr.foo`, internals as `ZarrCore.foo` (each test file does `import Zarr: ZarrCore`). If a test needs `ZarrCore.` for something a downstream user would plausibly need, that is a signal the name should be made public rather than the test qualified. + ### Module/File Layout -- `src/Zarr.jl` — Module entry point, defines `ZarrFormat{V}` (Val-parameterized version tag, default `DV = ZarrFormat(Val(2))`) +All paths below are relative to `lib/ZarrCore/`. + +- `src/ZarrCore.jl` — Module entry point, defines `ZarrFormat{V}` (Val-parameterized version tag, default `DV = ZarrFormat(Val(2))`), the `@public` macro, and the export/public declarations - `src/metadata.jl` — `MetadataV2` struct, type string encoding (`typestr`), fill value encoding/decoding, `Metadata()` constructors for V2; dispatches V3 to `metadata3.jl` - `src/metadata3.jl` — All V3-specific code: `MetadataV3` struct and constructors, `Metadata3(dict)` parsing, `lower3` serialization, codec pipeline parsing, `get_order`, `JSON.lower(::MetadataV3)` -- `src/chunkencoding.jl` — `ChunkEncoding` struct (separator char + prefix bool), `citostring()` for chunk path generation. V2 default: `'.'` separator, no prefix. V3 default: `'/'` separator, `"c/"` prefix +- `src/chunkkeyencoding.jl` — `ChunkKeyEncoding` struct (separator char + prefix bool), `citostring()` for chunk path generation, plus the `register_chunk_key_encoding` registry. V2 default: `'.'` separator, no prefix. V3 default: `'/'` separator, `"c/"` prefix - `src/ZArray.jl` — Core array type, `readblock!`/`writeblock!` (DiskArrays interface), `zcreate`, `zzeros`, `zopen`, resize/append - `src/ZGroup.jl` — Hierarchical group support, `zopen`, `zgroup`, auto-detection of zarr version via `ZarrFormat(store, path)` -- `src/Compressors/` — `Compressor` abstract type, `compressortypes` registry (keyed by spec name string), implementations: `blosc.jl`, `zlib.jl`, `zstd.jl`, `v3.jl` (v3 wrapper `Compressor_v3{C}`) -- `src/Codecs/` — V3 codec system (`Codec` abstract type), `V3/V3.jl` defines `V3Codec{In,Out}` with `BloscV3Codec`, `BytesCodec`, `CRC32cV3Codec`, `GzipV3Codec`, `ShardingCodec`, `TransposeCodec`, `ZstdV3Codec` +- `src/Compressors/` — `Compressor` abstract type, `compressortypes` registry (keyed by spec name string), implementations: `blosc.jl`, `zlib.jl`, `zstd.jl` +- `src/Codecs/` — V3 codec system (`Codec` abstract type), `V3/V3.jl` defines `V3Codec{In,Out}` with `BloscV3Codec`, `BytesCodec`, `CRC32cV3Codec`, `GzipV3Codec`, `ShardingCodec`, `TransposeCodec`, `ZstdV3Codec`, and the `register_codec` registry - `src/Filters/` — `Filter{T,TENC}` abstract type, implementations for variable-length arrays, strings, Fletcher32, shuffle, delta, quantize - `src/Storage/Storage.jl` — `AbstractStore` interface, I/O strategy (`SequentialRead`/`ConcurrentRead`), chunk read/write/delete helpers, metadata read/write dispatched on `ZarrFormat{2}` vs `ZarrFormat{3}` @@ -73,14 +101,14 @@ New store backends must implement: `getindex(store, key)::Union{Vector{UInt8}, N V3 support is under active development. Current state: -**Codecs (`src/Codecs/V3/V3.jl`)** +**Codecs (`lib/ZarrCore/src/Codecs/V3/V3.jl`)** - `BytesCodec` — stores `endian::Symbol` (`:little` or `:big`); encode/decode byte-swap elements when the target endian differs from the system byte order (`Base.ENDIAN_BOM`). Default is `:little`. - `TransposeCodec` — array→array permutation codec (renamed from `TransposeCodecImpl`) - `BloscV3Codec` — shuffle stored as integer (0=noshuffle, 1=shuffle, 2=bitshuffle); parsed from spec strings (`"noshuffle"`, `"shuffle"`, `"bitshuffle"`) and serialized back to strings - Sharding codec (`sharding_indexed`) has struct definitions and encode/decode logic but is not yet wired into the main read/write pipeline (throws `ArgumentError` when encountered) - `crc32c` codec has encode/decode implementations and is parseable from metadata -**Metadata (`src/metadata3.jl`)** +**Metadata (`lib/ZarrCore/src/metadata3.jl`)** - `MetadataV3{T,N,P}` has no `order` field; storage order is encoded in the pipeline via `TransposeCodec` - Two constructors: - Primary inner constructor: `MetadataV3{T,N,P}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_encoding)` — takes a pre-built pipeline, no `order` argument diff --git a/Project.toml b/Project.toml index 59586ab6..e9579ac8 100644 --- a/Project.toml +++ b/Project.toml @@ -3,24 +3,11 @@ uuid = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" version = "0.10.2" authors = ["Fabian Gans "] +[workspace] +projects = ["test", "lib/ZarrCore"] + [deps] -Blosc = "a74b3585-a348-5f62-a45c-50e91977d574" -CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" -ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" -ChunkCodecLibZlib = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" -ChunkCodecLibZstd = "55437552-ac27-4d47-9aa3-63184e8fd398" -DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" -HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" -JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -ZipArchives = "49080126-0e18-4c2a-b176-c102e4b3760c" +ZarrCore = "77f5b75c-4c08-499f-ba13-550b0a0af171" [weakdeps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" @@ -30,19 +17,4 @@ ZarrAWSS3Ext = "AWSS3" [compat] AWSS3 = "0.10, 0.11" -Blosc = "0.5, 0.6, 0.7" -CRC32c = "1.10, 1.11" -ChunkCodecCore = "1" -ChunkCodecLibZlib = "1" -ChunkCodecLibZstd = "1" -DataStructures = "0.17, 0.18, 0.19" -DateTimes64 = "1" -DiskArrays = "0.4.21" -HTTP = "2" -JSON = "0.21, 1" -OffsetArrays = "0.11, 1.0" -OrderedCollections = "1.8.2, 2" -URIs = "1" -Unicode = "1.10, 1.11.0" -ZipArchives = "2" julia = "1.10" diff --git a/docs/make.jl b/docs/make.jl index f13ec745..15a27701 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,11 +1,14 @@ using DocumenterVitepress using Documenter, Zarr +# All docstrings live in the `ZarrCore` implementation package; `Zarr` is only +# the facade that re-exports its API. +using Zarr: ZarrCore cp(joinpath(@__DIR__, "..", "CHANGELOG.md"), joinpath(@__DIR__, "src", "changelog.md"), force = true) cp(joinpath(@__DIR__, "..", "CONTRIBUTING.md"), joinpath(@__DIR__, "src", "contributing.md"), force = true) makedocs( - modules = [Zarr], + modules = [Zarr, ZarrCore], clean = false, doctest = true, format = DocumenterVitepress.MarkdownVitepress( diff --git a/docs/src/.vitepress/config.mts b/docs/src/.vitepress/config.mts index 89761cb2..39242097 100644 --- a/docs/src/.vitepress/config.mts +++ b/docs/src/.vitepress/config.mts @@ -24,6 +24,7 @@ const userGuideItems = [ // { text: 'Data Types', link: '/UserGuide/data_types' }, // { text: 'Codecs & Performance', link: '/UserGuide/performance' }, { text: 'Operations', link: '/UserGuide/operations'}, + { text: 'Chunking', link: '/UserGuide/chunking' }, // { text: 'Sharding', link: '/UserGuide/sharding' }, { text: 'Missing Values', link: '/UserGuide/missings' }, ] diff --git a/docs/src/UserGuide/caching.md b/docs/src/UserGuide/caching.md index 8d3aac1d..0d7fdc4f 100644 --- a/docs/src/UserGuide/caching.md +++ b/docs/src/UserGuide/caching.md @@ -23,7 +23,7 @@ and wrap it into a `CachedDiskArray`: ````jldoctest cache julia> a_lrucache = DiskArrays.cache(a,maxsize=1) -10000×10000 DiskArrays.CachedDiskArray{Float64, 2, ZArray{Float64, 2, DirectoryStore, Zarr.MetadataV2{Float64, 2, Zarr.BloscCompressor, Nothing}}, LRUCache.LRU{ChunkIndex{2, DiskArrays.OffsetChunks}, OffsetArrays.OffsetMatrix{Float64, Matrix{Float64}}}} +10000×10000 DiskArrays.CachedDiskArray{Float64, 2, ZArray{Float64, 2, DirectoryStore, ZarrCore.MetadataV2{Float64, 2, ZarrCore.BloscCompressor, Nothing}}, LRUCache.LRU{ChunkIndex{2, DiskArrays.OffsetChunks}, OffsetArrays.OffsetMatrix{Float64, Matrix{Float64}}}} Chunked: ( [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000] @@ -55,7 +55,7 @@ So, if we want to hold an incomplete local copy of any `AbstractDiskArray` you c ````jldoctest cache julia> a_arraycache = zarrcache(a,"./my_persistent_store.zarr") -10000×10000 DiskArrays.CachedDiskArray{Float64, 2, ZArray{Float64, 2, DirectoryStore, Zarr.MetadataV2{Float64, 2, Zarr.BloscCompressor, Nothing}}, Zarr.PermanentZarrCache{Float64, 2, ZArray{Float64, 2, DirectoryStore, Zarr.MetadataV2{Float64, 2, Zarr.BloscCompressor, Nothing}}}} +10000×10000 DiskArrays.CachedDiskArray{Float64, 2, ZArray{Float64, 2, DirectoryStore, ZarrCore.MetadataV2{Float64, 2, ZarrCore.BloscCompressor, Nothing}}, ZarrCore.PermanentZarrCache{Float64, 2, ZArray{Float64, 2, DirectoryStore, ZarrCore.MetadataV2{Float64, 2, ZarrCore.BloscCompressor, Nothing}}}} Chunked: ( [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000] diff --git a/docs/src/UserGuide/chunking.md b/docs/src/UserGuide/chunking.md new file mode 100644 index 00000000..ab51a5c3 --- /dev/null +++ b/docs/src/UserGuide/chunking.md @@ -0,0 +1,110 @@ +# Chunking and Irregular Chunk Grids + +A Zarr array is divided into chunks, each of which is compressed and stored (and +transferred) independently. The chunk size along every axis is therefore one of +the main knobs controlling read/write performance and storage granularity. + +By default chunks have a uniform size along each axis (regular chunking), which +is what the plain tuple form of the `chunks` keyword gives you: + +````jldoctest regular +julia> using Zarr + +julia> z = zzeros(Int, 100, 100; chunks=(10, 20)) +ZArray{Int64} of size 100 x 100 +```` + +Here the array is divided into chunks of 10×20 elements, evenly tiling the +array. + +## Controlling the chunk grid with `GridChunks` + +Passing a tuple only fixes the chunk size; to fully control the chunk grid you +can instead pass a `DiskArrays.GridChunks` object as the `chunks` keyword to +[`zcreate`](@ref), [`zzeros`](@ref) or the `ZArray(a::AbstractArray, ...; chunks=...)` +constructor. This is also what makes *irregular* (rectilinear) chunking +possible, where chunk sizes vary along an axis instead of being uniform. + +The chunk types and the `GridChunks` wrapper come from the +[DiskArrays.jl](https://github.com/JuliaIO/DiskArrays.jl) package, which Zarr +builds on. Bring them into scope with: + +```julia +using DiskArrays: GridChunks, IrregularChunks, RegularChunks +``` + +A `GridChunks` holds one chunk specification per axis, either a +`RegularChunks` or an `IrregularChunks`. Irregular chunks along an axis are +described by their edge lengths: + +````jldoctest irregular +julia> using Zarr + +julia> using DiskArrays: GridChunks, IrregularChunks, RegularChunks + +julia> chunks = GridChunks(RegularChunks(2, 0, 5), IrregularChunks(chunksizes=[3, 4, 5, 6, 2])); + +julia> z = zcreate(Int, 5, 20; zarr_format=3, chunks=chunks) +ZArray{Int64} of size 5 x 20 + +julia> eachchunk(z) +GridChunks( +RegularChunks(2, 0, 5)IrregularChunks([0, 3, 7, 12, 18, 20])) + +julia> z[:, :] = reshape(1:100, 5, 20); + +julia> z[:, :] == reshape(1:100, 5, 20) +true +```` + +The second axis is split into chunks of 3, 4, 5, 6 and 2 elements (20 in total), +while the first axis keeps regular chunks of size 2. + +`zzeros` works the same way; for grids that are not uniformly regular it fills +the array element by element rather than reusing a single encoded chunk: + +````jldoctest irregular +julia> zz = zzeros(Int, 10, 10; zarr_format=3, chunks=GridChunks(RegularChunks(5, 0, 10), IrregularChunks(chunksizes=[3, 3, 4]))); + +julia> zz[1, 1] +0 +```` + +## Persistence and interoperability + +Irregular chunk grids are stored in the array metadata, so they survive +round-tripping through a store. Zarr v3 records them as a `rectilinear` chunk +grid with `kind: "inline"`, where each axis is either a single chunk size +(regular) or a list of edge lengths. Reopening the array restores the exact +grid: + +````jldoctest irregular +julia> dir = joinpath(mktempdir(), "irregular.zarr"); + +julia> z = zcreate(Int, 5, 20; zarr_format=3, chunks=chunks, path=dir); + +julia> z2 = zopen(dir); + +julia> eachchunk(z2) +GridChunks( +RegularChunks(2, 0, 5)IrregularChunks([0, 3, 7, 12, 18, 20])) +```` + +::: warning + +Zarr v2 has no concept of irregular chunk grids. If you create an array with an +irregular grid and persist it as v2, the metadata only records the *maximum* +chunk size along each axis, and the array is reopened as a regular grid of that +size. Irregular chunking is therefore only preserved end-to-end with Zarr v3 +(`zarr_format=3`). + +::: + +::: warning + +`resize!` and `append!` are not supported for arrays whose grid contains +`IrregularChunks`; shrinking such an array throws an `ArgumentError`. Resizing +an irregular grid would require redefining the per-axis edge lengths, which is +not implemented. + +::: diff --git a/docs/src/get_started.md b/docs/src/get_started.md index 501b186d..1c1805e5 100644 --- a/docs/src/get_started.md +++ b/docs/src/get_started.md @@ -80,6 +80,20 @@ z = ZArray(rand(Float64, 100, 100)) zinfo(z) ``` +For irregular chunking — where chunk sizes vary along an axis instead of being +uniform — pass a `GridChunks` object (from DiskArrays) as the `chunks` keyword +instead of a tuple of sizes: + +```@example irregular-chunks +using Zarr +using DiskArrays: GridChunks, IrregularChunks, RegularChunks +chunks = GridChunks(RegularChunks(2, 0, 5), IrregularChunks(chunksizes=[3, 4, 5, 6, 2])); +z = zcreate(Int, 5, 20; zarr_format=3, chunks=chunks) +``` + +Irregular grids are preserved end-to-end with Zarr v3. See the +[Chunking](./UserGuide/chunking) page for details and caveats. + ## Reading and Writing ```@example rw using Zarr @@ -232,7 +246,7 @@ Zarr allows you to create hierarchical groups, similar to directories: using Zarr store = Zarr.DirectoryStore("experiment.zarr") -g = zgroup(store, "", Zarr.ZarrFormat(3)) +g = zgroup(store, "", 3) # 3 selects the Zarr v3 format zcreate(Float64, g, "temperature", 100, 100; chunks=(50, 50), fill_value=0.0) zcreate(Float64, g, "precipitation", 100, 100; chunks=(50, 50), fill_value=0.0) diff --git a/docs/src/reference.md b/docs/src/reference.md index 7d0e31c7..91efb08c 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -10,13 +10,13 @@ zzeros ## Group hierarchy ```@autodocs -Modules = [Zarr] +Modules = [ZarrCore] Pages = ["ZGroup.jl"] ``` ## Compressors ```@autodocs -Modules = [Zarr] +Modules = [ZarrCore] Pages = ["Compressors/Compressors.jl", "Compressors/blosc.jl", "Compressors/zlib.jl", "Compressors/zstd.jl"] ``` diff --git a/docs/src/tutorials/tutorial.md b/docs/src/tutorials/tutorial.md index 00cdf96d..93d16408 100644 --- a/docs/src/tutorials/tutorial.md +++ b/docs/src/tutorials/tutorial.md @@ -173,7 +173,7 @@ A number of different compressors can be used with Zarr. In this Julia package w julia> using Zarr julia> compressor = Zarr.BloscCompressor(cname="zstd", clevel=3, shuffle=true) -Zarr.BloscCompressor(0, 3, "zstd", 1) +ZarrCore.BloscCompressor(0, 3, "zstd", 1) julia> data = Int32(1):Int32(100000000) 1:100000000 @@ -197,7 +197,7 @@ Shape : (10000, 10000) Chunk Shape : (1000, 1000) Order : C Read-Only : false -Compressor : Zarr.BloscCompressor(0, 3, "zstd", 1) +Compressor : ZarrCore.BloscCompressor(0, 3, "zstd", 1) Filters : nothing Store type : Dictionary Storage No. bytes : 400000000 @@ -227,7 +227,7 @@ julia> z = zcreate(Vector{Int}, 4) ZArray{Vector{Int64}} of size 4 julia> z.metadata.filters -(Zarr.VLenArrayFilter{Int64}(),) +(ZarrCore.VLenArrayFilter{Int64}(),) julia> z[1:3] = [[1,3,5],[4],[7,9,14]]; diff --git a/ext/ZarrAWSS3Ext.jl b/ext/ZarrAWSS3Ext.jl index 151f9e0d..b7f5f303 100644 --- a/ext/ZarrAWSS3Ext.jl +++ b/ext/ZarrAWSS3Ext.jl @@ -1,14 +1,14 @@ module ZarrAWSS3Ext import Zarr -import Zarr: +import ZarrCore: S3Store, AbstractStore, cloud_list_objects, ConcurrentRead, storageregexlist, concurrent_io_tasks, - zopen + zopen, ZarrCore using AWSS3: AWSS3, s3_put, s3_get, s3_delete, s3_list_objects, s3_exists, S3Path, get_config diff --git a/ext/s3store.jl b/ext/s3store.jl index bf967c32..00f58725 100644 --- a/ext/s3store.jl +++ b/ext/s3store.jl @@ -1,4 +1,4 @@ -function Zarr.S3Store(bucket::String; +function ZarrCore.S3Store(bucket::String; aws = nothing, ) if aws === nothing @@ -31,7 +31,7 @@ end Base.delete!(s::S3Store, d::String) = s3_delete(s.aws,s.bucket,d) -function Zarr.storagesize(s::S3Store,p) +function ZarrCore.storagesize(s::S3Store,p) prefix = (isempty(p) || endswith(p,"/")) ? p : string(p,"/") r = s3_list_objects(s.aws,s.bucket,prefix) s = 0 @@ -44,12 +44,12 @@ function Zarr.storagesize(s::S3Store,p) s end -function Zarr.isinitialized(s::S3Store, i::String) +function ZarrCore.isinitialized(s::S3Store, i::String) s3_exists(s.aws,s.bucket,i) end -function Zarr.cloud_list_objects(s::S3Store,p) +function ZarrCore.cloud_list_objects(s::S3Store,p) prefix = (isempty(p) || endswith(p,"/")) ? p : string(p,"/") s3_list_objects_delim(s.aws, s.bucket, prefix) end @@ -75,12 +75,12 @@ function s3_list_objects_delim(aws, bucket, prefix, delimiter="/") end result end -function Zarr.subdirs(s::S3Store, p) +function ZarrCore.subdirs(s::S3Store, p) s3_resp = cloud_list_objects(s, p) !haskey(s3_resp,"CommonPrefixes") && return String[] allstrings(s3_resp["CommonPrefixes"],"Prefix") end -function Zarr.subkeys(s::S3Store, p) +function ZarrCore.subkeys(s::S3Store, p) s3_resp = cloud_list_objects(s, p) !haskey(s3_resp,"Contents") && return String[] r = allstrings(s3_resp["Contents"],"Key") @@ -91,16 +91,16 @@ allstrings(v,prefixkey) = [rstrip(String(v[prefixkey]),'/')] # push!(storageregexlist,r"^s3://"=>S3Store) -function Zarr.storefromstring(::Type{<:S3Store}, s, _) +function ZarrCore.storefromstring(::Type{<:S3Store}, s, _) decomp = split(s,"/",keepempty=false) bucket = decomp[2] path = join(decomp[3:end],"/") S3Store(String(bucket),aws=AWSS3.AWS.current_aws_config()),path end -Zarr.store_read_strategy(::S3Store) = ConcurrentRead(concurrent_io_tasks[]) +ZarrCore.store_read_strategy(::S3Store) = ConcurrentRead(concurrent_io_tasks[]) -function Zarr.zopen(s::S3Path, mode="r"; kwargs...) +function ZarrCore.zopen(s::S3Path, mode="r"; kwargs...) decomp = split(string(s),"/",keepempty=false) bucket = decomp[2] path = join(decomp[3:end],"/") diff --git a/lib/ZarrCore/Project.toml b/lib/ZarrCore/Project.toml new file mode 100644 index 00000000..51d1ca1d --- /dev/null +++ b/lib/ZarrCore/Project.toml @@ -0,0 +1,41 @@ +name = "ZarrCore" +uuid = "77f5b75c-4c08-499f-ba13-550b0a0af171" +version = "0.10.1" +authors = ["Fabian Gans "] + +[deps] +Blosc = "a74b3585-a348-5f62-a45c-50e91977d574" +CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" +ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" +ChunkCodecLibZlib = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" +ChunkCodecLibZstd = "55437552-ac27-4d47-9aa3-63184e8fd398" +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +ZipArchives = "49080126-0e18-4c2a-b176-c102e4b3760c" + +[compat] +Blosc = "0.5, 0.6, 0.7" +CRC32c = "1.10, 1.11" +ChunkCodecCore = "1" +ChunkCodecLibZlib = "1" +ChunkCodecLibZstd = "1" +DataStructures = "0.17, 0.18, 0.19" +DateTimes64 = "1" +DiskArrays = "0.4.21" +HTTP = "2" +JSON = "0.21, 1" +OffsetArrays = "0.11, 1.0" +OrderedCollections = "1.8.2, 2" +URIs = "1" +Unicode = "1.10, 1.11.0" +ZipArchives = "2" +julia = "1.10" diff --git a/src/Codecs/Codecs.jl b/lib/ZarrCore/src/Codecs/Codecs.jl similarity index 96% rename from src/Codecs/Codecs.jl rename to lib/ZarrCore/src/Codecs/Codecs.jl index ec6e6205..2ed2b495 100644 --- a/src/Codecs/Codecs.jl +++ b/lib/ZarrCore/src/Codecs/Codecs.jl @@ -46,4 +46,8 @@ getCodec(::Type{<:Codec}, d::Dict) = error("Unimplemented") include("V3/V3.jl") +@static if VERSION ≥ v"1.11" + include("public_names_codecs.jl") +end + end diff --git a/src/Codecs/V3/V3.jl b/lib/ZarrCore/src/Codecs/V3/V3.jl similarity index 98% rename from src/Codecs/V3/V3.jl rename to lib/ZarrCore/src/Codecs/V3/V3.jl index d3c4fa41..18417e36 100644 --- a/src/Codecs/V3/V3.jl +++ b/lib/ZarrCore/src/Codecs/V3/V3.jl @@ -2,9 +2,9 @@ module V3Codecs import ..Codecs: zencode, zdecode, zencode!, zdecode! # Import compressor types and functions from Zarr (grandparent module) -import ...Zarr: ZlibCompressor, ZstdCompressor, zcompress, zuncompress -import ...Zarr: BloscCompressor as ZarrBloscCompressor -import ...Zarr: AbstractCodecPipeline, V3Pipeline, pipeline_encode, pipeline_decode! +import ...ZarrCore: ZlibCompressor, ZstdCompressor, zcompress, zuncompress +import ...ZarrCore: BloscCompressor as ZarrBloscCompressor +import ...ZarrCore: AbstractCodecPipeline, V3Pipeline, pipeline_encode, pipeline_decode! using CRC32c: CRC32c using JSON: JSON using ChunkCodecLibZlib: GzipCodec as LibZGzipCodec, GzipEncodeOptions @@ -823,4 +823,8 @@ function codec_decode(::VLenUTF8V3Codec, encoded::Vector{UInt8}, ::Type{T}, shap out end +@static if VERSION ≥ v"1.11" + include("public_names_v3.jl") +end + end diff --git a/lib/ZarrCore/src/Codecs/V3/public_names_v3.jl b/lib/ZarrCore/src/Codecs/V3/public_names_v3.jl new file mode 100644 index 00000000..a685bebc --- /dev/null +++ b/lib/ZarrCore/src/Codecs/V3/public_names_v3.jl @@ -0,0 +1,7 @@ +# The codec types themselves are re-exported (as public names) from `ZarrCore`; +# these are the extension points for defining and registering new v3 codecs. +public V3Codec, getCodec, register_codec, codec_parsers, codec_encode, + codec_decode, is_fixed_size, name +public BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, ShardingCodec, + TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, CRC32cV3Codec, + VLenUTF8V3Codec diff --git a/lib/ZarrCore/src/Codecs/public_names_codecs.jl b/lib/ZarrCore/src/Codecs/public_names_codecs.jl new file mode 100644 index 00000000..78297f8c --- /dev/null +++ b/lib/ZarrCore/src/Codecs/public_names_codecs.jl @@ -0,0 +1 @@ +public Codec, V3Codecs, zencode, zencode!, zdecode, zdecode!, getCodec diff --git a/src/Compressors/Compressors.jl b/lib/ZarrCore/src/Compressors/Compressors.jl similarity index 100% rename from src/Compressors/Compressors.jl rename to lib/ZarrCore/src/Compressors/Compressors.jl diff --git a/src/Compressors/blosc.jl b/lib/ZarrCore/src/Compressors/blosc.jl similarity index 97% rename from src/Compressors/blosc.jl rename to lib/ZarrCore/src/Compressors/blosc.jl index 789a298f..72d37b79 100644 --- a/src/Compressors/blosc.jl +++ b/lib/ZarrCore/src/Compressors/blosc.jl @@ -67,4 +67,4 @@ end JSON.lower(c::BloscCompressor) = Dict("id"=>"blosc", "cname"=>c.cname, "clevel"=>c.clevel, "shuffle"=>c.shuffle, "blocksize"=>c.blocksize) -Zarr.compressortypes["blosc"] = BloscCompressor \ No newline at end of file +ZarrCore.compressortypes["blosc"] = BloscCompressor \ No newline at end of file diff --git a/src/Compressors/zlib.jl b/lib/ZarrCore/src/Compressors/zlib.jl similarity index 96% rename from src/Compressors/zlib.jl rename to lib/ZarrCore/src/Compressors/zlib.jl index 6b82feef..92f2371a 100644 --- a/src/Compressors/zlib.jl +++ b/lib/ZarrCore/src/Compressors/zlib.jl @@ -42,4 +42,4 @@ end JSON.lower(z::ZlibCompressor) = Dict("id"=>"zlib", "level" => z.config.level) -Zarr.compressortypes["zlib"] = ZlibCompressor \ No newline at end of file +ZarrCore.compressortypes["zlib"] = ZlibCompressor \ No newline at end of file diff --git a/src/Compressors/zstd.jl b/lib/ZarrCore/src/Compressors/zstd.jl similarity index 97% rename from src/Compressors/zstd.jl rename to lib/ZarrCore/src/Compressors/zstd.jl index 6cd80a08..c6648ee2 100644 --- a/src/Compressors/zstd.jl +++ b/lib/ZarrCore/src/Compressors/zstd.jl @@ -52,4 +52,4 @@ function JSON.lower(z::ZstdCompressor) end end -Zarr.compressortypes["zstd"] = ZstdCompressor +ZarrCore.compressortypes["zstd"] = ZstdCompressor diff --git a/src/Filters/Filters.jl b/lib/ZarrCore/src/Filters/Filters.jl similarity index 100% rename from src/Filters/Filters.jl rename to lib/ZarrCore/src/Filters/Filters.jl diff --git a/src/Filters/delta.jl b/lib/ZarrCore/src/Filters/delta.jl similarity index 100% rename from src/Filters/delta.jl rename to lib/ZarrCore/src/Filters/delta.jl diff --git a/src/Filters/fixedscaleoffset.jl b/lib/ZarrCore/src/Filters/fixedscaleoffset.jl similarity index 100% rename from src/Filters/fixedscaleoffset.jl rename to lib/ZarrCore/src/Filters/fixedscaleoffset.jl diff --git a/src/Filters/fletcher32.jl b/lib/ZarrCore/src/Filters/fletcher32.jl similarity index 100% rename from src/Filters/fletcher32.jl rename to lib/ZarrCore/src/Filters/fletcher32.jl diff --git a/src/Filters/quantize.jl b/lib/ZarrCore/src/Filters/quantize.jl similarity index 100% rename from src/Filters/quantize.jl rename to lib/ZarrCore/src/Filters/quantize.jl diff --git a/src/Filters/shuffle.jl b/lib/ZarrCore/src/Filters/shuffle.jl similarity index 100% rename from src/Filters/shuffle.jl rename to lib/ZarrCore/src/Filters/shuffle.jl diff --git a/src/Filters/vlenfilters.jl b/lib/ZarrCore/src/Filters/vlenfilters.jl similarity index 100% rename from src/Filters/vlenfilters.jl rename to lib/ZarrCore/src/Filters/vlenfilters.jl diff --git a/src/MaxLengthStrings.jl b/lib/ZarrCore/src/MaxLengthStrings.jl similarity index 100% rename from src/MaxLengthStrings.jl rename to lib/ZarrCore/src/MaxLengthStrings.jl diff --git a/src/Storage/Storage.jl b/lib/ZarrCore/src/Storage/Storage.jl similarity index 100% rename from src/Storage/Storage.jl rename to lib/ZarrCore/src/Storage/Storage.jl diff --git a/src/Storage/cachingstore.jl b/lib/ZarrCore/src/Storage/cachingstore.jl similarity index 100% rename from src/Storage/cachingstore.jl rename to lib/ZarrCore/src/Storage/cachingstore.jl diff --git a/src/Storage/consolidated.jl b/lib/ZarrCore/src/Storage/consolidated.jl similarity index 100% rename from src/Storage/consolidated.jl rename to lib/ZarrCore/src/Storage/consolidated.jl diff --git a/src/Storage/dictstore.jl b/lib/ZarrCore/src/Storage/dictstore.jl similarity index 100% rename from src/Storage/dictstore.jl rename to lib/ZarrCore/src/Storage/dictstore.jl diff --git a/src/Storage/directorystore.jl b/lib/ZarrCore/src/Storage/directorystore.jl similarity index 100% rename from src/Storage/directorystore.jl rename to lib/ZarrCore/src/Storage/directorystore.jl diff --git a/src/Storage/gcstore.jl b/lib/ZarrCore/src/Storage/gcstore.jl similarity index 96% rename from src/Storage/gcstore.jl rename to lib/ZarrCore/src/Storage/gcstore.jl index 02a2051c..86499d46 100644 --- a/src/Storage/gcstore.jl +++ b/lib/ZarrCore/src/Storage/gcstore.jl @@ -5,7 +5,7 @@ const GOOGLE_STORAGE_REST_API = GOOGLE_STORAGE_API * "/storage/v1" const GOOGLE_STORAGE_CREDENTIALS = Dict{String,String}() """ - Zarr.gcs_credentials(user_project,access_token,token_type) + gcs_credentials(user_project,access_token,token_type) Set the user project, access token and and token type for the Google Cloud Store. @@ -18,7 +18,7 @@ function gcs_credentials(user_project,access_token,token_type) end """ - Zarr.gcs_credentials(; metadata_url = "http://metadata.google.internal/computeMetadata/v1/") + gcs_credentials(; metadata_url = "http://metadata.google.internal/computeMetadata/v1/") Set (or renew) the user project, access token and and token type for the Google Cloud Store from the Metadata server (assuming the function is executed from diff --git a/src/Storage/http.jl b/lib/ZarrCore/src/Storage/http.jl similarity index 97% rename from src/Storage/http.jl rename to lib/ZarrCore/src/Storage/http.jl index 636483e7..892f2062 100644 --- a/src/Storage/http.jl +++ b/lib/ZarrCore/src/Storage/http.jl @@ -27,7 +27,7 @@ if r.status >= 300 """Received error code $(r.status) when connecting to $(s.url) with message $(String(r.body)). This might be an actual error, or an indication that the server returns a different error code than 404 for missing chunks. In the latter case, you can run - `Zarr.missing_chunk_return_code!(a.storage,$(r.status))` where `a` is your Zarr array or group, + `ZarrCore.missing_chunk_return_code!(a.storage,$(r.status))` where `a` is your Zarr array or group, to fix the issue. """ throw(ErrorException(err_msg)) diff --git a/src/Storage/zipstore.jl b/lib/ZarrCore/src/Storage/zipstore.jl similarity index 100% rename from src/Storage/zipstore.jl rename to lib/ZarrCore/src/Storage/zipstore.jl diff --git a/src/ZArray.jl b/lib/ZarrCore/src/ZArray.jl similarity index 62% rename from src/ZArray.jl rename to lib/ZarrCore/src/ZArray.jl index 857b9a6b..68f6bd1e 100644 --- a/src/ZArray.jl +++ b/lib/ZarrCore/src/ZArray.jl @@ -10,23 +10,23 @@ Number of tasks to use for async reading of chunks. Warning: setting this to ver """ const concurrent_io_tasks = Ref(50) -getfillval(::Type{T}, t::String) where {T <: Number} = parse(T, t) +getfillval(::Type{T}, t::String) where {T<:Number} = parse(T, t) getfillval(::Type{T}, t::Union{T,Nothing}) where {T} = t struct SenMissArray{T,N} <: AbstractArray{Union{T,Missing},N} x::Array{T,N} senval::T end -SenMissArray(x::Array{T,N},v) where {T,N} = SenMissArray{T,N}(x,convert(T,v)) +SenMissArray(x::Array{T,N}, v) where {T,N} = SenMissArray{T,N}(x, convert(T, v)) Base.size(x::SenMissArray) = size(x.x) senval(x::SenMissArray) = x.senval -function Base.getindex(x::SenMissArray,i::Int) +function Base.getindex(x::SenMissArray, i::Int) v = x.x[i] - isequal(v,senval(x)) ? missing : v + isequal(v, senval(x)) ? missing : v end -Base.setindex!(x::SenMissArray,v,i::Int) = x.x[i] = v -Base.setindex!(x::SenMissArray,::Missing,i::Int) = x.x[i] = senval(x) -Base.IndexStyle(::Type{<:SenMissArray})=Base.IndexLinear() +Base.setindex!(x::SenMissArray, v, i::Int) = x.x[i] = v +Base.setindex!(x::SenMissArray, ::Missing, i::Int) = x.x[i] = senval(x) +Base.IndexStyle(::Type{<:SenMissArray}) = Base.IndexLinear() # Struct representing a Zarr Array in Julia, note that # chunks(chunk size) and size are always in Julia column-major order @@ -40,7 +40,7 @@ end Base.eltype(::ZArray{T}) where {T} = T Base.ndims(::ZArray{<:Any,N}) where {N} = N -Base.size(z::ZArray{<:Any,N}) where {N} = z.metadata.shape[]::NTuple{N, Int} +Base.size(z::ZArray{<:Any,N}) where {N} = z.metadata.shape[]::NTuple{N,Int} function Base.size(z::ZArray{<:Any,N}, i::Integer) where {N} len = length(z.metadata.shape[]) if 0 < i <= len @@ -55,17 +55,17 @@ Base.length(z::ZArray) = prod(z.metadata.shape[])::Int Base.lastindex(z::ZArray{<:Any,N}, n::Integer) where {N} = size(z, n)::Int Base.lastindex(z::ZArray{<:Any,1}) = size(z, 1)::Int -function Base.show(io::IO,z::ZArray) - print(io, "ZArray{", eltype(z) ,"} of size ",join(string.(size(z)), " x ")) +function Base.show(io::IO, z::ZArray) + print(io, "ZArray{", eltype(z), "} of size ", join(string.(size(z)), " x ")) end -function Base.show(io::IO,::MIME"text/plain",z::ZArray) - print(io, "ZArray{", eltype(z) ,"} of size ",join(string.(size(z)), " x ")) +function Base.show(io::IO, ::MIME"text/plain", z::ZArray) + print(io, "ZArray{", eltype(z), "} of size ", join(string.(size(z)), " x ")) end zname(z::ZArray) = zname(z.path) function zname(s::String) - spl = split(rstrip(s,'/'),'/') + spl = split(rstrip(s, '/'), '/') isempty(last(spl)) ? "root" : last(spl) end @@ -75,7 +75,7 @@ end Returns the size of the compressed data stored in the ZArray `z` in bytes """ -storagesize(z::ZArray) = storagesize(z.storage,z.path) +storagesize(z::ZArray) = storagesize(z.storage, z.path) """ storageratio(z::ZArray) @@ -90,28 +90,32 @@ nobytes(z::ZArray) = length(z)*sizeof(eltype(z)) nobytes(z::ZArray{<:Vector}) = "unknown" nobytes(z::ZArray{<:String}) = "unknown" -zinfo(z::ZArray) = zinfo(stdout,z) -function zinfo(io::IO,z::ZArray) +zinfo(z::ZArray) = zinfo(stdout, z) +function zinfo(io::IO, z::ZArray) ninit = sum(chunkindices(z)) do i store_isinitialized(z.storage, z.path, i, z.metadata.chunk_key_encoding) end allinfos = [ - "Type" => "ZArray", - "Data type" => eltype(z), - "Shape" => size(z), - "Chunk Shape" => z.metadata.chunks, - "Order" => try get_order(z.metadata) catch e "unknown ($(e.msg))" end, - "Read-Only" => !z.writeable, - "Compressor" => z.metadata isa MetadataV2 ? z.metadata.compressor : get_pipeline(z.metadata), - "Filters" => z.metadata isa MetadataV2 ? z.metadata.filters : nothing, - "Store type" => z.storage, - "No. bytes" => nobytes(z), - "No. bytes stored" => storagesize(z), - "Storage ratio" => storageratio(z), - "Chunks initialized" => "$(ninit)/$(length(chunkindices(z)))" + "Type" => "ZArray", + "Data type" => eltype(z), + "Shape" => size(z), + "Chunk Shape" => eachchunk(z), + "Order" => try + get_order(z.metadata) + catch e + "unknown ($(e.msg))" + end, + "Read-Only" => !z.writeable, + "Compressor" => z.metadata isa MetadataV2 ? z.metadata.compressor : get_pipeline(z.metadata), + "Filters" => z.metadata isa MetadataV2 ? z.metadata.filters : nothing, + "Store type" => z.storage, + "No. bytes" => nobytes(z), + "No. bytes stored" => storagesize(z), + "Storage ratio" => storageratio(z), + "Chunks initialized" => "$(ninit)/$(length(chunkindices(z)))" ] foreach(allinfos) do ii - println(io,rpad(ii[1],20),": ",ii[2]) + println(io, rpad(ii[1], 20), ": ", ii[2]) end end @@ -124,7 +128,7 @@ function ZArray(s::T, mode="r", path="", zarr_format=:auto; fill_as_missing=fals metadata = getmetadata(zv, s, path, fill_as_missing) attrs = getattrs(zv, s, path) writeable = mode == "w" - startswith(path,"/") && error("Paths should never start with a leading '/'") + startswith(path, "/") && error("Paths should never start with a leading '/'") ZArray(metadata, s, string(path), attrs, writeable) end @@ -132,33 +136,28 @@ zarr_format(z::ZArray) = zarr_format(z.metadata) dimension_separator(z::ZArray) = dimension_separator(z.metadata) -""" - trans_ind(r, bs) - -For a given index and blocksize determines which chunks of the Zarray will have to -be accessed. -""" -trans_ind(r::AbstractUnitRange, bs) = fld1(first(r),bs):fld1(last(r),bs) -trans_ind(r::Integer, bs) = fld1(r,bs) - function boundint(r1, s2, o2) - r2 = range(o2+1,length=s2) - f1, f2 = first(r1), first(r2) - l1, l2 = last(r1),last(r2) + r2 = range(o2+1, length=s2) + f1, f2 = first(r1), first(r2) + l1, l2 = last(r1), last(r2) UnitRange(f1 > f2 ? f1 : f2, l1 < l2 ? l1 : l2) end +chunk_unbounded(gc::GridChunks, i) = map(chunk_unbounded, gc.chunks, i.I) +chunk_unbounded(r::RegularChunks, i) = ((i-1)*r.chunksize+1-r.offset):(i*r.chunksize-r.offset) +chunk_unbounded(chunks::IrregularChunks, i) = (chunks.offsets[i]+1):chunks.offsets[i+1] + function getchunkarray(z::ZArray{>:Missing}) # temporary workaround to use strings as data values - inner = fill(z.metadata.fill_value, z.metadata.chunks) - a = SenMissArray(inner,z.metadata.fill_value) + inner = fill(z.metadata.fill_value, DiskArrays.max_chunksize.(eachchunk(z).chunks)) + a = SenMissArray(inner, z.metadata.fill_value) end _zero(T) = zero(T) _zero(T::Type{<:MaxLengthString}) = zero(T) _zero(T::Type{ASCIIChar}) = ASCIIChar(0) _zero(::Type{<:Vector{T}}) where T = T[] _zero(::Type{Char}) = Char(0) -getchunkarray(z::ZArray) = fill(_zero(eltype(z)), z.metadata.chunks) +getchunkarray(z::ZArray) = fill(_zero(eltype(z)), DiskArrays.max_chunksize.(eachchunk(z).chunks)) # Same as `getchunkarray` but skips the zero/fill_value-fill. Use only when # the caller guarantees the buffer will be fully overwritten before any read @@ -169,14 +168,14 @@ getchunkarray(z::ZArray) = fill(_zero(eltype(z)), z.metadata.chunks) # of a `SenMissArray`, not an `Array{Union{Missing,T}}` directly (Blosc and # friends reject non-isbits eltypes). function getchunkarray_undef(z::ZArray{T}) where {T} - Missing <: T && return getchunkarray(z) - return Array{T}(undef, z.metadata.chunks) + Missing <: T && return getchunkarray(z) + return Array{T}(undef, DiskArrays.max_chunksize.(eachchunk(z).chunks)) end maybeinner(a::Array) = a maybeinner(a::SenMissArray) = a.x -resetbuffer!(fv,a::Array) = fv === nothing || fill!(a,fv) -resetbuffer!(_,a::SenMissArray) = fill!(a,missing) +resetbuffer!(fv, a::Array) = fv === nothing || fill!(a, fv) +resetbuffer!(_, a::SenMissArray) = fill!(a, missing) # Returns the chunk index when the call qualifies for the single-chunk fast # path: a plain `Array{T,N}` matching the chunk shape, `Missing <: T` false, @@ -185,7 +184,7 @@ resetbuffer!(_,a::SenMissArray) = fill!(a,missing) # coverage check is needed. function singlechunk_fastpath(arr, z::ZArray{T,N}, blockr::CartesianIndices{N}) where {T,N} arr isa Array{T,N} && !(Missing <: T) && - size(arr) == z.metadata.chunks && length(blockr) == 1 || return nothing + size(arr) == DiskArrays.max_chunksize.(eachchunk(z).chunks) && length(blockr) == 1 || return nothing return first(blockr) end @@ -215,7 +214,7 @@ end # fastpath guarantees `ain` is the caller's input array, not the shared # scratch buffer the multi-chunk path reuses across iterations. function write_singlechunk_fastpath!( - z::ZArray{T,N,<:AbstractStore,<:MetadataV2{T,N,NoCompressor,Nothing}}, + z::ZArray{T,N,<:AbstractStore,<:MetadataV2{T,N,NoCompressor,Nothing,<:DiskArrays.GridChunks{N}}}, ain::Array{T,N}, bI::CartesianIndex, ) where {T,N} fv = z.metadata.fill_value @@ -226,17 +225,17 @@ function write_singlechunk_fastpath!( return nothing end store_writechunk(z.storage, _reinterpret(UInt8, ain), - z.path, bI, z.metadata.chunk_key_encoding) + z.path, bI, z.metadata.chunk_key_encoding) return nothing end # Function to read or write from a zarr array. Could be refactored # using type system to get rid of the `if readmode` statements. -function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::CartesianIndices{N}) where {N} +function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::CartesianIndices{N}) where {N} - output_base_offsets = map(i->first(i)-1,r.indices) + output_base_offsets = map(i->first(i)-1, r.indices) # Determines which chunks are affected - blockr = CartesianIndices(map(trans_ind, r.indices, z.metadata.chunks)) + blockr = CartesianIndices(map(DiskArrays.findchunk, eachchunk(z).chunks, r.indices)) # Fast path: single-chunk full-read decodes directly into `aout`, skipping the readtask channel and scratch buffer. bI = singlechunk_fastpath(aout, z, blockr) if bI !== nothing @@ -251,37 +250,48 @@ function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes a = getchunkarray_undef(z) # Now loop through the chunks c = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) - + task = @async begin read_items!($(z.storage), c, $(z.metadata.chunk_key_encoding), $(z.path), $(blockr)) end - bind(c,task) + bind(c, task) - try + chunks = eachchunk(z) + + try for i in 1:length(blockr) - - bI,chunk_compressed = take!(c) - - current_chunk_offsets = map((s,i)->s*(i-1),size(a),Tuple(bI)) - - indranges = map(boundint,r.indices,size(a),current_chunk_offsets) - - uncompress_to_output!(aout,output_base_offsets,z,chunk_compressed,current_chunk_offsets,a,indranges) + + bI, chunk_compressed = take!(c) + + current_chunk_inds = chunk_unbounded(chunks, bI) + + current_chunks_size = length.(current_chunk_inds) + + current_chunk_offsets = first.(current_chunk_inds) .- 1 + + indranges = map(boundint, r.indices, current_chunks_size, current_chunk_offsets) + #If the chunk size is smaller than the buffer size we need to write a smaller buffer + if current_chunks_size != size(a) + inds_reduced = Base.OneTo.(current_chunks_size) + uncompress_to_output!(aout, output_base_offsets, z, chunk_compressed, current_chunk_offsets, view(a, inds_reduced...), indranges) + else + uncompress_to_output!(aout, output_base_offsets, z, chunk_compressed, current_chunk_offsets, a, indranges) + end nothing end finally close(c) end - + aout end -function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::CartesianIndices{N}) where {N} +function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any,N}, r::CartesianIndices{N}) where {N} z.writeable || error("Can not write to read-only ZArray") - input_base_offsets = map(i->first(i)-1,r.indices) + input_base_offsets = map(i->first(i)-1, r.indices) # Determines which chunks are affected - blockr = CartesianIndices(map(trans_ind, r.indices, z.metadata.chunks)) + blockr = CartesianIndices(map(DiskArrays.findchunk, eachchunk(z).chunks, r.indices)) # Fast path: single-chunk full-overwrite skips the readtask/writetask channels and the scratch buffer. bI = singlechunk_fastpath(ain, z, blockr) if bI !== nothing @@ -295,45 +305,67 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes a = z.metadata.fill_value === nothing ? getchunkarray(z) : getchunkarray_undef(z) # Now loop through the chunks readchannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) - - readtask = @async begin + + readtask = @async begin read_items!(z.storage, readchannel, z.metadata.chunk_key_encoding, z.path, blockr) end - bind(readchannel,readtask) + bind(readchannel, readtask) writechannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage)) writetask = @async begin write_items!(z.storage, writechannel, z.metadata.chunk_key_encoding, z.path, blockr) end - bind(writechannel,writetask) - - try + bind(writechannel, writetask) + + chunks = eachchunk(z) + + try for i in 1:length(blockr) - - bI,chunk_compressed = take!(readchannel) - - current_chunk_offsets = map((s,i)->s*(i-1),size(a),Tuple(bI)) - indranges = map(boundint,r.indices,size(a),current_chunk_offsets) + bI, chunk_compressed = take!(readchannel) + + current_chunk_inds = chunk_unbounded(chunks, bI) + + current_chunk_offsets = first.(current_chunk_inds) .- 1 + + current_chunks_size = length.(current_chunk_inds) + + indranges = map(boundint, r.indices, size(a), current_chunk_offsets) if isnothing(chunk_compressed) || (length.(indranges) != size(a)) - resetbuffer!(z.metadata.fill_value,a) + resetbuffer!(z.metadata.fill_value, a) end curchunk = if length.(indranges) != size(a) - view(a,dotminus.(indranges,current_chunk_offsets)...) + view(a, dotminus.(indranges, current_chunk_offsets)...) else a end - - if chunk_compressed !== nothing - uncompress_raw!(a,z,chunk_compressed) - end - curchunk .= view(ain,dotminus.(indranges,input_base_offsets)...) - put!(writechannel,bI=>compress_raw(maybeinner(a),z)) + if current_chunks_size != size(a) + inds_reduced = Base.OneTo.(length.(current_chunk_inds)) + + if chunk_compressed !== nothing + uncompress_raw!(view(a, inds_reduced...), z, chunk_compressed) + end + + curchunk .= view(ain, dotminus.(indranges, input_base_offsets)...) + + #If the chunk size is smaller than the buffer size we need to write a smaller buffer + + inds_reduced = Base.OneTo.(length.(current_chunk_inds)) + put!(writechannel, bI=>compress_raw(maybeinner(a)[inds_reduced...], z)) + + else + + if chunk_compressed !== nothing + uncompress_raw!(a, z, chunk_compressed) + end + curchunk .= view(ain, dotminus.(indranges, input_base_offsets)...) + put!(writechannel, bI=>compress_raw(maybeinner(a), z)) + end nothing end finally @@ -344,19 +376,19 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes ain end -DiskArrays.readblock!(a::ZArray,aout,i::AbstractUnitRange...) = readblock!(aout,a,CartesianIndices(i)) -DiskArrays.writeblock!(a::ZArray,v,i::AbstractUnitRange...) = writeblock!(v,a,CartesianIndices(i)) +DiskArrays.readblock!(a::ZArray, aout, i::AbstractUnitRange...) = readblock!(aout, a, CartesianIndices(i)) +DiskArrays.writeblock!(a::ZArray, v, i::AbstractUnitRange...) = writeblock!(v, a, CartesianIndices(i)) DiskArrays.haschunks(::ZArray) = DiskArrays.Chunked() -DiskArrays.eachchunk(a::ZArray) = DiskArrays.GridChunks(a,a.metadata.chunks) +DiskArrays.eachchunk(a::ZArray) = a.metadata.chunks[] """ uncompress_raw!(a::DenseArray{T},z::ZArray{T,N},i::CartesianIndex{N}) Read the chunk specified by `i` from the Zarray `z` and write its content to `a` """ -function uncompress_raw!(a,z::ZArray{<:Any,N},curchunk) where N +function uncompress_raw!(a, z::ZArray{<:Any,N}, curchunk) where N if curchunk === nothing - if isnothing(z.metadata.fill_value) + if isnothing(z.metadata.fill_value) throw(ArgumentError("The array $z got missing chunks and no fill_value")) end fill!(a, z.metadata.fill_value) @@ -366,22 +398,20 @@ function uncompress_raw!(a,z::ZArray{<:Any,N},curchunk) where N a end -dotminus(x,y) = x.-y +dotminus(x, y) = x .- y + +function uncompress_to_output!(aout, output_base_offsets, z, chunk_compressed, current_chunk_offsets, a, indranges) -function uncompress_to_output!(aout,output_base_offsets,z,chunk_compressed,current_chunk_offsets,a,indranges) - - uncompress_raw!(a,z,chunk_compressed) - + uncompress_raw!(a, z, chunk_compressed) if length.(indranges) == size(a) aout[dotminus.(indranges, output_base_offsets)...] = ndims(a) == 0 ? a[1] : a else - curchunk = a[dotminus.(indranges,current_chunk_offsets)...] + curchunk = a[dotminus.(indranges, current_chunk_offsets)...] aout[dotminus.(indranges, output_base_offsets)...] = curchunk end end -function compress_raw(a,z) - length(a) == prod(z.metadata.chunks) || throw(DimensionMismatch("Array size does not equal chunk size")) +function compress_raw(a, z) pipeline_encode(get_pipeline(z.metadata), a, z.metadata.fill_value) end @@ -396,7 +426,7 @@ Creates a new empty zarr array with element type `T` and array dimensions `dims` * `name=""` name of the zarr array, defaults to the directory name * `zarr_format`=$(DV) Zarr format version (2 or 3) * `storagetype` determines the storage to use, current options are `DirectoryStore` or `DictStore` -* `chunks=dims` size of the individual array chunks, must be a tuple of length `length(dims)` +* `chunks=dims` size of the individual array chunks. Either a tuple of length `length(dims)` specifying the chunk size along each axis (regular chunking), or a `DiskArrays.GridChunks` object for full control over the chunk grid, which also enables irregular (rectilinear) chunking where chunk sizes vary along an axis. See [Chunking](../UserGuide/chunking) for details. * `fill_value=nothing` value to represent missing values * `fill_as_missing=false` set to `true` shall fillvalue s be converted to `missing`s * `filters`=filters to be applied @@ -412,7 +442,7 @@ function zcreate(::Type{T}, dims::Integer...; zarr_format=DV, dimension_separator=default_sep(zarr_format), kwargs... - ) where T +) where T if path===nothing store = DictStore() @@ -423,26 +453,26 @@ function zcreate(::Type{T}, dims::Integer...; end struct ShapeOnlyArray{T,N} <: AbstractArray{T,N} - sz::Dims{N} + sz::Dims{N} end Base.size(a::ShapeOnlyArray) = a.sz Base.getindex(::ShapeOnlyArray, ::Vararg{Any}) = - error("ShapeOnlyArray carries no data") + error("ShapeOnlyArray carries no data") -function zcreate(::Type{T},storage::AbstractStore, +function zcreate(::Type{T}, storage::AbstractStore, dims...; - path = "", - zarr_format = DV, + path="", + zarr_format=DV, chunks=dims, fill_value=nothing, fill_as_missing=false, compressor=BloscCompressor(), - filters = filterfromtype(T), + filters=filterfromtype(T), attrs=Dict(), writeable=true, indent_json=false, dimension_separator=nothing - ) where {T} +) where {T} v = ZarrFormat(zarr_format) if isnothing(dimension_separator) @@ -453,31 +483,37 @@ function zcreate(::Type{T},storage::AbstractStore, dimension_separator = only(dimension_separator) end chunk_key_encoding = ChunkKeyEncoding(dimension_separator, default_prefix(v)) - - length(dims) == length(chunks) || throw(DimensionMismatch("Dims must have the same length as chunks")) N = length(dims) + + if chunks isa Tuple + length(dims) == length(chunks) || throw(DimensionMismatch("Dims must have the same length as chunks")) + elseif chunks isa GridChunks + length(dims) == ndims(chunks) || throw(DimensionMismatch("Dims must have the same length as chunks")) + else + throw(ArgumentError("chunks must be provided either as a Tuple of Ints or as a DiskArrays.GridChunks object")) + end C = typeof(compressor) - + # Create a dummy array to use with Metadata constructor # This allows us to leverage the multiple dispatch in Metadata constructors dummy_array = ShapeOnlyArray{T,N}(dims) metadata = Metadata(dummy_array, chunks, v; - compressor=compressor, - fill_value=fill_value, - filters=filters, - fill_as_missing=fill_as_missing, + compressor=compressor, + fill_value=fill_value, + filters=filters, + fill_as_missing=fill_as_missing, chunk_key_encoding=chunk_key_encoding ) - + # Extract the element type from the metadata (handles T2 calculation) T2 = eltype(metadata) - - isemptysub(storage,path) || error("$storage $path is not empty") - + + isemptysub(storage, path) || error("$storage $path is not empty") + writemetadata(v, storage, path, metadata, indent_json=indent_json) - + writeattrs(v, storage, path, attrs, indent_json=indent_json) - + ZArray(metadata, storage, path, attrs, writeable) end @@ -488,8 +524,8 @@ function filterfromtype(::Type{<:AbstractArray{T}}) where T (VLenArrayFilter{T}(),) end -filterfromtype(::Type{<:Union{<:AbstractString, Union{<:AbstractString, Missing}}}) = (VLenUTF8Filter(),) -filterfromtype(::Type{<:Union{MaxLengthString, Union{MaxLengthString, Missing}}}) = nothing +filterfromtype(::Type{<:Union{<:AbstractString,Union{<:AbstractString,Missing}}}) = (VLenUTF8Filter(),) +filterfromtype(::Type{<:Union{MaxLengthString,Union{MaxLengthString,Missing}}}) = nothing #Not all Array types can be mapped directly to a valid ZArray encoding. #Here we try to determine the correct element type @@ -508,22 +544,26 @@ end Returns the Cartesian Indices of the chunks of a given ZArray """ -chunkindices(z::ZArray) = CartesianIndices(map((s, c) -> 1:ceil(Int, s/c), z.metadata.shape[], z.metadata.chunks)) +chunkindices(z::ZArray) = CartesianIndices(map(length, eachchunk(z).chunks)) """ zzeros(T, dims...; kwargs... ) Creates a zarr array and initializes all values with zero. Accepts the same keyword arguments as `zcreate` """ -function zzeros(T,dims...;kwargs...) - z = zcreate(T,dims...;kwargs...) - as = zeros(T, z.metadata.chunks...) - data_encoded = compress_raw(as,z) - p = z.path - if data_encoded !== nothing - for i in chunkindices(z) - store_writechunk(z.storage, data_encoded, p, i, z.metadata.chunk_key_encoding) +function zzeros(T, dims...; kwargs...) + z = zcreate(T, dims...; kwargs...) + if all(i->isa(i, RegularChunks), eachchunk(z).chunks) + as = zeros(T, DiskArrays.max_chunksize.(eachchunk(z).chunks)) + data_encoded = compress_raw(as, z) + p = z.path + if data_encoded !== nothing + for i in chunkindices(z) + store_writechunk(z.storage, data_encoded, p, i, z.metadata.chunk_key_encoding) + end end + else + z .= zero(T) end z end @@ -539,14 +579,19 @@ function Base.resize!(z::ZArray{T,N}, newsize::NTuple{N}) where {T,N} any(<(0), newsize) && throw(ArgumentError("Size must be positive")) oldsize = z.metadata.shape[] z.metadata.shape[] = newsize + gc = z.metadata.chunks[].chunks + new_chunks = map(gc, newsize) do c, s + c isa DiskArrays.RegularChunks ? DiskArrays.RegularChunks(c.chunksize, c.offset, s) : c + end + z.metadata.chunks[] = DiskArrays.GridChunks(new_chunks) #Check if array was shrunk - if any(map(<,newsize, oldsize)) - prune_oob_chunks(z.storage, z.path, oldsize, newsize, z.metadata.chunks, z.metadata.chunk_key_encoding) + if any(map(<, newsize, oldsize)) + prune_oob_chunks(z.storage, z.path, oldsize, newsize, eachchunk(z), z.metadata.chunk_key_encoding) end writemetadata(zarr_format(z), z.storage, z.path, z.metadata) nothing end -Base.resize!(z::ZArray, newsize::Integer...) = resize!(z,newsize) +Base.resize!(z::ZArray, newsize::Integer...) = resize!(z, newsize) """ append!(z::ZArray{<:Any, N},a;dims = N) @@ -563,12 +608,12 @@ append!(z,ones(Int,6,2)) #Add two new columns z[:,:] ```` """ -function Base.append!(z::ZArray{<:Any, N},a;dims = N) where N +function Base.append!(z::ZArray{<:Any,N}, a; dims=N) where N #Determine how many entries to add to axis - otherdims = sort!(setdiff(1:N,dims)) + otherdims = sort!(setdiff(1:N, dims)) othersize = size(z)[otherdims] if ndims(a)==N - nadd = size(a,dims) + nadd = size(a, dims) size(a)[otherdims]==othersize || throw(DimensionMismatch("Array to append does not have the correct size, expected: $(othersize)")) elseif ndims(a)==N-1 size(a)==othersize || throw(DimensionMismatch("Array to append does not have the correct size, expected: $(othersize)")) @@ -578,18 +623,23 @@ function Base.append!(z::ZArray{<:Any, N},a;dims = N) where N end oldsize = size(z) newsize = ntuple(i->i==dims ? oldsize[i]+nadd : oldsize[i], N) - resize!(z,newsize) - appendinds = ntuple(i->i==dims ? (oldsize[i]+1:newsize[i]) : Colon(),N) + resize!(z, newsize) + appendinds = ntuple(i->i==dims ? ((oldsize[i]+1):newsize[i]) : Colon(), N) z[appendinds...] = a nothing end -function prune_oob_chunks(s::AbstractStore, path, oldsize, newsize, chunks, chunk_key_encoding) - dimstoshorten = findall(map(<,newsize, oldsize)) +function prune_oob_chunks(s::AbstractStore, path, oldsize, newsize, chunks::DiskArrays.GridChunks, chunk_key_encoding) + # Resizing/pruning is only well-defined for regular chunk grids. + any(c -> c isa DiskArrays.IrregularChunks, chunks.chunks) && + throw(ArgumentError("Resizing arrays with irregular chunk grids is not supported")) + dimstoshorten = findall(map(<, newsize, oldsize)) for idim in dimstoshorten - delrange = (fld1(newsize[idim],chunks[idim])+1):(fld1(oldsize[idim],chunks[idim])) - allchunkranges = map(i->1:fld1(oldsize[i],chunks[i]),1:length(oldsize)) - r = (allchunkranges[1:idim-1]..., delrange, allchunkranges[idim+1:end]...) + nchunks_new = DiskArrays.findchunk(chunks.chunks[idim], max(newsize[idim], 1)) + nchunks_old = DiskArrays.findchunk(chunks.chunks[idim], oldsize[idim]) + delrange = (nchunks_new+1):nchunks_old + allchunkranges = map(i -> 1:DiskArrays.findchunk(chunks.chunks[i], oldsize[i]), 1:length(oldsize)) + r = (allchunkranges[1:(idim-1)]..., delrange, allchunkranges[(idim+1):end]...) for cI in CartesianIndices(r) store_deletechunk(s, path, cI, chunk_key_encoding) end diff --git a/src/ZGroup.jl b/lib/ZarrCore/src/ZGroup.jl similarity index 100% rename from src/ZGroup.jl rename to lib/ZarrCore/src/ZGroup.jl diff --git a/lib/ZarrCore/src/ZarrCore.jl b/lib/ZarrCore/src/ZarrCore.jl new file mode 100644 index 00000000..ec2b5f4a --- /dev/null +++ b/lib/ZarrCore/src/ZarrCore.jl @@ -0,0 +1,54 @@ +module ZarrCore + +import JSON +import Blosc +import Unicode +using OrderedCollections: OrderedDict + +struct ZarrFormat{V} + version::Val{V} +end +Base.Int(v::ZarrFormat{V}) where V = V +@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) +ZarrFormat(v::ZarrFormat) = v +#Default Zarr Version +const DV = ZarrFormat(Val(2)) + +include("types.jl") +include("chunkkeyencoding.jl") +include("metadata.jl") +include("metadata3.jl") +include("Compressors/Compressors.jl") +include("Codecs/Codecs.jl") +include("Storage/Storage.jl") +include("Filters/Filters.jl") +include("ZArray.jl") +include("pipeline.jl") +include("ZGroup.jl") +include("caching.jl") + +import .Codecs: Codec +import .Codecs.V3Codecs: V3Codec, BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, + ShardingCodec, TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, + CRC32cV3Codec, VLenUTF8V3Codec + +# ## Public API +# +# The rule of thumb is: anything a downstream consumer could need, and every +# documented extension point, is part of the public API. `export` is reserved +# for the handful of names that are convenient to have in scope after +# `using Zarr`; everything else is marked `public` and must be qualified. +# +# Things that are deliberately *not* public (and may change without notice): +# `Metadata`/`MetadataV2`/`MetadataV3`, `ZarrFormat`, `is_zarray`, `is_zgroup`, +# `normalize_path`, `MaxLengthString`, `ShapeOnlyArray` and the various +# `pipeline_*`/`store_*` internals. + +export ZArray, ZGroup, zopen, zzeros, zcreate, zgroup, zarrcache, + storagesize, storageratio, zinfo, + DirectoryStore, S3Store, GCStore + +@static if VERSION >= v"1.11" + include("public_names_core.jl") + end +end # module diff --git a/src/caching.jl b/lib/ZarrCore/src/caching.jl similarity index 99% rename from src/caching.jl rename to lib/ZarrCore/src/caching.jl index b404e623..2fda6cde 100644 --- a/src/caching.jl +++ b/lib/ZarrCore/src/caching.jl @@ -1,5 +1,4 @@ import DiskArrays: approx_chunksize, eachchunk, CachedDiskArray, ChunkIndex -export zarrcache struct PermanentZarrCache{T,N,A<:ZArray{T,N}} a::A diff --git a/src/chunkkeyencoding.jl b/lib/ZarrCore/src/chunkkeyencoding.jl similarity index 100% rename from src/chunkkeyencoding.jl rename to lib/ZarrCore/src/chunkkeyencoding.jl diff --git a/src/metadata.jl b/lib/ZarrCore/src/metadata.jl similarity index 73% rename from src/metadata.jl rename to lib/ZarrCore/src/metadata.jl index 55c1765b..90ba4c93 100644 --- a/src/metadata.jl +++ b/lib/ZarrCore/src/metadata.jl @@ -1,5 +1,6 @@ import Dates: Date, DateTime -using DateTimes64: DateTime64, pydatetime_string, datetime_from_pystring +using DateTimes64: DateTime64, pydatetime_string, datetime_from_pystring +using DiskArrays: GridChunks """NumPy array protocol type string (typestr) format @@ -22,7 +23,7 @@ Base.codepoint(x::ASCIIChar) = UInt8(x) Base.show(io::IO, x::ASCIIChar) = print(io, Char(x)) Base.zero(::Union{ASCIIChar,Type{ASCIIChar}}) = ASCIIChar(Base.zero(UInt8)) -Base.zero(t::Union{String, Type{String}}) = "" +Base.zero(t::Union{String,Type{String}}) = "" typestr(t::Type) = string('<', 'V', sizeof(t)) typestr(t::Type{>:Missing}) = typestr(Base.nonmissingtype(t)) @@ -40,7 +41,7 @@ typestr(t::Type{<:DateTime64}) = pydatetime_string(t) typestr(::Type{<:AbstractString}) = "|O" const typestr_regex = r"^([<|>])([tbiufcmMOSUV])(\d*)(\[\w+\])?$" -const typemap = Dict{Tuple{Char, Int}, DataType}( +const typemap = Dict{Tuple{Char,Int},DataType}( ('b', 1) => Bool, ('S', 1) => ASCIIChar, ) @@ -49,10 +50,10 @@ typecharf(::Type{<:Signed}) = 'i' typecharf(::Type{<:Unsigned}) = 'u' typecharf(::Type{<:AbstractFloat}) = 'f' typecharf(::Type{<:Complex}) = 'c' -foreach([Float16,Float32,Float64,Int8,Int16,Int32,Int64,Int128, - UInt8,UInt16,UInt32,UInt64,UInt128, - Complex{Float16},Complex{Float32},Complex{Float64}]) do t - typemap[(typecharf(t),sizemapf(t))] = t +foreach([Float16, Float32, Float64, Int8, Int16, Int32, Int64, Int128, + UInt8, UInt16, UInt32, UInt64, UInt128, + Complex{Float16}, Complex{Float32}, Complex{Float64}]) do t + typemap[(typecharf(t), sizemapf(t))] = t end function typestr(s::AbstractString, filterlist=nothing) @@ -81,7 +82,7 @@ function typestr(s::AbstractString, filterlist=nothing) return datetime_from_pystring(s) end # convert typecode to Char and typesize to Int - typemap[(tc,ts)] + typemap[(tc, ts)] end end @@ -102,29 +103,29 @@ value of the ".zarray" key within an array store. https://zarr.readthedocs.io/en/stable/spec/v2.html#metadata """ -abstract type AbstractMetadata{T,N,E <: AbstractChunkKeyEncoding} end +abstract type AbstractMetadata{T,N,E<:AbstractChunkKeyEncoding} end Base.ndims(::AbstractMetadata{<:Any,N}) where N = N """Metadata for Zarr version 2 arrays""" -struct MetadataV2{T,N,C,F} <: AbstractMetadata{T,N,ChunkKeyEncoding} +struct MetadataV2{T,N,C,F,CT<:GridChunks{N}} <: AbstractMetadata{T,N,ChunkKeyEncoding} zarr_format::Int node_type::String - shape::Base.RefValue{NTuple{N, Int}} - chunks::NTuple{N, Int} + shape::Base.RefValue{NTuple{N,Int}} + chunks::Base.RefValue{CT} dtype::String # structured data types not yet supported compressor::C - fill_value::Union{T, Nothing} + fill_value::Union{T,Nothing} order::Char filters::F # not yet supported chunk_key_encoding::ChunkKeyEncoding - function MetadataV2{T2,N,C,F}(zarr_format, node_type, shape, chunks, dtype, compressor, fill_value, order, filters, chunk_key_encoding) where {T2,N,C,F} + function MetadataV2{T2,N,C,F,CT}(zarr_format, node_type, shape, chunks::CT, dtype, compressor, fill_value, order, filters, chunk_key_encoding) where {T2,N,C,F,CT<:GridChunks{N}} zarr_format == 2 || throw(ArgumentError("MetadataV2 only functions if zarr_format == 2")) #Do some sanity checks to make sure we have a sane array any(<(0), shape) && throw(ArgumentError("Size must be positive")) - any(<(1), chunks) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) + any(<(1), DiskArrays.max_chunksize.(chunks.chunks)) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) order === 'C' || throw(ArgumentError("Currently only 'C' storage order is supported")) - new{T2,N,C,F}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, compressor, fill_value, order, filters, chunk_key_encoding) + new{T2,N,C,F,CT}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), Ref(chunks), dtype, compressor, fill_value, order, filters, chunk_key_encoding) end end zarr_format(::MetadataV2) = ZarrFormat(Val(2)) @@ -134,29 +135,29 @@ const Metadata = AbstractMetadata #To make unit tests pass with ref shape function Base.:(==)(m1::MetadataV2, m2::MetadataV2) - m1.zarr_format == m2.zarr_format && - m1.node_type == m2.node_type && - m1.shape[] == m2.shape[] && - m1.chunks == m2.chunks && - m1.dtype == m2.dtype && - m1.compressor == m2.compressor && - m1.fill_value == m2.fill_value && - m1.order == m2.order && - m1.filters == m2.filters && - m1.chunk_key_encoding == m2.chunk_key_encoding + m1.zarr_format == m2.zarr_format && + m1.node_type == m2.node_type && + m1.shape[] == m2.shape[] && + m1.chunks[] == m2.chunks[] && + m1.dtype == m2.dtype && + m1.compressor == m2.compressor && + m1.fill_value == m2.fill_value && + m1.order == m2.order && + m1.filters == m2.filters && + m1.chunk_key_encoding == m2.chunk_key_encoding end "Construct Metadata based on your data" -function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, zarr_format=DV; - node_type::String="array", - compressor::C=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - filters=nothing, - fill_as_missing = false, - dimension_separator::Char = '.' - ) where {T, N, C} +function Metadata(A::AbstractArray{T,N}, chunks, zarr_format=DV; + node_type::String="array", + compressor::C=BloscCompressor(), + fill_value::Union{T,Nothing}=nothing, + order::Char=('C'), + filters=nothing, + fill_as_missing=false, + dimension_separator::Char=('.') +) where {T,N,C} return Metadata(A, chunks, ZarrFormat(zarr_format); node_type=node_type, compressor=compressor, @@ -169,17 +170,20 @@ function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, zarr_format=DV; end # V2 constructor -function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{2}; - node_type::String="array", - compressor::C=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - filters::F=nothing, - fill_as_missing = false, +function Metadata(A::AbstractArray{T,N}, chunks::Union{NTuple{N,Int},GridChunks{N}}, ::ZarrFormat{2}; + node_type::String="array", + compressor::C=BloscCompressor(), + fill_value::Union{T,Nothing}=nothing, + order::Char=('C'), + filters::F=nothing, + fill_as_missing=false, chunk_key_encoding=ChunkKeyEncoding('.', false) - ) where {T, N, C, F} +) where {T,N,C,F} T2 = (fill_value === nothing || !fill_as_missing) ? T : Union{T,Missing} - MetadataV2{T2,N,C,typeof(filters)}( + if chunks isa NTuple + chunks = GridChunks(size(A), chunks) + end + MetadataV2{T2,N,C,typeof(filters),typeof(chunks)}( 2, node_type, size(A), @@ -193,12 +197,12 @@ function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{2}; ) end -Metadata(s::Union{AbstractString, IO}, fill_as_missing) = Metadata(JSON.parse(s; dicttype=Dict{String,Any}), fill_as_missing) +Metadata(s::Union{AbstractString,IO}, fill_as_missing) = Metadata(JSON.parse(s; dicttype=Dict{String,Any}), fill_as_missing) "Construct Metadata from Dict" function Metadata(d::AbstractDict, fill_as_missing) zarr_format = d["zarr_format"]::Int - zarr_format ∉ (2, 3) && throw(ArgumentError("Zarr.jl currently only supports v2 or v3 of the specification")) + zarr_format ∉ (2, 3) && throw(ArgumentError("ZarrCore.jl currently only supports v2 or v3 of the specification")) return Metadata(d, fill_as_missing, ZarrFormat(zarr_format)) end @@ -229,11 +233,14 @@ function Metadata(d::AbstractDict, fill_as_missing, ::ZarrFormat{2}) dim_sep = only(get(d, "dimension_separator", '.')) - MetadataV2{TU,N,C,F}( + shape_jl = NTuple{N,Int}(d["shape"]) |> reverse + chunks_tuple = NTuple{N,Int}(d["chunks"]) |> reverse + chunks_jl = GridChunks(shape_jl, chunks_tuple) + MetadataV2{TU,N,C,F,typeof(chunks_jl)}( d["zarr_format"], node_type, - NTuple{N, Int}(d["shape"]) |> reverse, - NTuple{N, Int}(d["chunks"]) |> reverse, + shape_jl, + chunks_jl, d["dtype"], compressor, fv, @@ -246,11 +253,11 @@ end "Describes how to lower Metadata to JSON, used in json(::Metadata)" function JSON.lower(md::MetadataV2) - Dict{String, Any}( + Dict{String,Any}( "zarr_format" => Int(md.zarr_format), "node_type" => md.node_type, "shape" => md.shape[] |> reverse, - "chunks" => md.chunks |> reverse, + "chunks" => reverse(DiskArrays.max_chunksize.(md.chunks[].chunks)), "dtype" => md.dtype, "compressor" => md.compressor, "fill_value" => fill_value_encoding(md.fill_value), @@ -265,7 +272,7 @@ end # https://zarr.readthedocs.io/en/stable/spec/v2.html#fill-value-encoding fill_value_encoding(v) = v -fill_value_encoding(::Nothing)=nothing +fill_value_encoding(::Nothing) = nothing function fill_value_encoding(v::AbstractFloat) if isnan(v) "NaN" @@ -284,7 +291,7 @@ fill_value_decoding(v::Nothing, ::Any) = v fill_value_decoding(v, T) = T(v) fill_value_decoding(v::Number, T::Type{String}) = v == 0 ? "" : T(UInt8[v]) fill_value_decoding(v, ::Type{ASCIIChar}) = v == "" ? nothing : v -fill_value_decoding(v::Nothing, ::Type{Zarr.ASCIIChar}) = v +fill_value_decoding(v::Nothing, ::Type{ZarrCore.ASCIIChar}) = v fill_value_decoding(v::Vector, T::Type{<:Complex}) = T(v[1], v[2]) # Sometimes when translating between CF (climate and forecast) convention data # and Zarr groups, fill values are left as "negative integers" to encode unsigned diff --git a/src/metadata3.jl b/lib/ZarrCore/src/metadata3.jl similarity index 64% rename from src/metadata3.jl rename to lib/ZarrCore/src/metadata3.jl index 9101e977..aa05c90d 100644 --- a/src/metadata3.jl +++ b/lib/ZarrCore/src/metadata3.jl @@ -1,8 +1,10 @@ +using DiskArrays: GridChunks, RegularChunks, IrregularChunks + """ Prototype Zarr version 3 support """ -const typemap3 = Dict{String, DataType}() +const typemap3 = Dict{String,DataType}() foreach([Bool, Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64]) do t typemap3[lowercase(string(t))] = t end @@ -14,11 +16,11 @@ function typestr3(t::Type) return lowercase(string(t)) end -function typestr3(::Type{MaxLengthString{N, UInt32}}) where {N} - return Dict{String, Any}( +function typestr3(::Type{MaxLengthString{N,UInt32}}) where {N} + return Dict{String,Any}( "name" => "fixed_length_utf32", - "configuration" => Dict{String, Any}("length_bytes" => N * 4) - ) + "configuration" => Dict{String,Any}("length_bytes" => N * 4) + ) end # TODO: Check raw types @@ -49,7 +51,7 @@ function parse_datatype3(d) name = get(d, "name", nothing) if name == "fixed_length_utf32" - return MaxLengthString{d["configuration"]["length_bytes"] ÷ 4, UInt32} + return MaxLengthString{d["configuration"]["length_bytes"] ÷ 4,UInt32} end throw(ArgumentError("Unsupported Zarr v3 data_type: $d")) end @@ -63,24 +65,23 @@ function check_keys(d::AbstractDict, keys) end """Metadata for Zarr version 3 arrays""" -struct MetadataV3{T,N,P<:AbstractCodecPipeline,E<:AbstractChunkKeyEncoding} <: AbstractMetadata{T,N,E} +struct MetadataV3{T,N,P<:AbstractCodecPipeline,E<:AbstractChunkKeyEncoding,CT<:GridChunks{N}} <: AbstractMetadata{T,N,E} zarr_format::Int node_type::String - shape::Base.RefValue{NTuple{N, Int}} - chunks::NTuple{N, Int} - dtype::Union{String, Dict{String, Any}} # data_type in v3 + shape::Base.RefValue{NTuple{N,Int}} + chunks::Base.RefValue{CT} + dtype::Union{String,Dict{String,Any}} # data_type in v3 pipeline::P - fill_value::Union{T, Nothing} + fill_value::Union{T,Nothing} chunk_key_encoding::E - function MetadataV3{T2,N,P,E}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) where {T2,N,P,E} + function MetadataV3{T2,N,P,E,CT}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) where {T2,N,P,E,CT} zarr_format == 3 || throw(ArgumentError("MetadataV3 only functions if zarr_format == 3")) #Do some sanity checks to make sure we have a sane array any(<(0), shape) && throw(ArgumentError("Size must be positive")) - any(<(1), chunks) && throw(ArgumentError("Chunk size must be >= 1 along each dimension")) - new{T2,N,P,E}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), chunks, dtype, pipeline, fill_value, chunk_key_encoding) + all(((c, s),)->DiskArrays.arraysize_from_chunksize(c) == s, zip(chunks.chunks, shape)) || error("Chunk array does not have the same size as the array shape") + new{T2,N,P,E,CT}(zarr_format, node_type, Base.RefValue{NTuple{N,Int}}(shape), Ref(chunks), dtype, pipeline, fill_value, chunk_key_encoding) end end -MetadataV3{T2,N,P}(args...) where {T2,N,P} = MetadataV3{T2,N,P,ChunkKeyEncoding}(args...) zarr_format(::MetadataV3) = ZarrFormat(Val(3)) """ @@ -88,13 +89,13 @@ Convenience constructor for MetadataV3 that builds the codec pipeline from `order` (translated to a TransposeCodec), `endian` (translated to a BytesCodec), and `compressor` (translated to bytes->bytes codecs). """ -function MetadataV3{T2,N}(zarr_format, node_type, shape::NTuple{N,Int}, chunks::NTuple{N,Int}, - dtype, fill_value; - order::Char='C', - endian::Symbol=:little, - compressor=BloscCompressor(), - chunk_key_encoding::E=ChunkKeyEncoding('/', true) - ) where {T2, N, E} +function MetadataV3{T2,N}(zarr_format, node_type, shape::NTuple{N,Int}, chunks, + dtype, fill_value; + order::Char=('C'), + endian::Symbol=:little, + compressor=BloscCompressor(), + chunk_key_encoding::E=ChunkKeyEncoding('/', true) +) where {T2,N,E} T_base = Base.nonmissingtype(T2) array_array_codecs = if order == 'F' (Codecs.V3Codecs.TransposeCodec(ntuple(i -> N - i + 1, N)),) @@ -108,6 +109,9 @@ function MetadataV3{T2,N}(zarr_format, node_type, shape::NTuple{N,Int}, chunks:: array_bytes_codec = Codecs.V3Codecs.BytesCodec(endian) typesize = sizeof(T_base) end + if chunks isa Tuple + chunks = GridChunks(shape, chunks) + end bytes_bytes_codecs = if compressor isa NoCompressor () elseif compressor isa BloscCompressor @@ -122,18 +126,18 @@ function MetadataV3{T2,N}(zarr_format, node_type, shape::NTuple{N,Int}, chunks:: throw(ArgumentError("Unsupported compressor type for v3: $(typeof(compressor))")) end pipeline = V3Pipeline(array_array_codecs, array_bytes_codec, bytes_bytes_codecs) - return MetadataV3{T2,N,typeof(pipeline),E}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) + return MetadataV3{T2,N,typeof(pipeline),E,typeof(chunks)}(zarr_format, node_type, shape, chunks, dtype, pipeline, fill_value, chunk_key_encoding) end function Base.:(==)(m1::MetadataV3, m2::MetadataV3) - m1.zarr_format == m2.zarr_format && - m1.node_type == m2.node_type && - m1.shape[] == m2.shape[] && - m1.chunks == m2.chunks && - m1.dtype == m2.dtype && - m1.fill_value == m2.fill_value && - m1.pipeline == m2.pipeline && - m1.chunk_key_encoding == m2.chunk_key_encoding + m1.zarr_format == m2.zarr_format && + m1.node_type == m2.node_type && + m1.shape[] == m2.shape[] && + m1.chunks[] == m2.chunks[] && + m1.dtype == m2.dtype && + m1.fill_value == m2.fill_value && + m1.pipeline == m2.pipeline && + m1.chunk_key_encoding == m2.chunk_key_encoding end """ @@ -164,8 +168,8 @@ function get_order(md::MetadataV3) )) end N = ndims(md) - c_perm = ntuple(identity, N) - f_perm = ntuple(i -> N - i + 1, N) + c_perm = ntuple(identity, N) + f_perm = ntuple(i -> N - i + 1, N) if codec.order == c_perm return 'C' elseif codec.order == f_perm @@ -206,7 +210,7 @@ function Metadata3(d::AbstractDict, fill_as_missing) end group_pipeline = V3Pipeline((), Codecs.V3Codecs.BytesCodec(), ()) - return MetadataV3{Int,0,typeof(group_pipeline),ChunkKeyEncoding}(zarr_format, node_type, (), (), "", group_pipeline, 0, ChunkKeyEncoding('/', true)) + return MetadataV3{Int,0,typeof(group_pipeline),ChunkKeyEncoding,GridChunks{0,Tuple{}}}(zarr_format, node_type, (), GridChunks{0,Tuple{}}(()), "", group_pipeline, 0, ChunkKeyEncoding('/', true)) end # Array keys @@ -242,14 +246,40 @@ function Metadata3(d::AbstractDict, fill_as_missing) # Chunk Grid chunk_grid = d["chunk_grid"] if chunk_grid["name"] == "regular" - chunks = Int.(chunk_grid["configuration"]["chunk_shape"]) - if length(shape) != length(chunks) - throw(ArgumentError("Shape has rank $(length(shape)) which does not match the chunk_shape rank of $(length(chunks))")) + cshape = Int.(chunk_grid["configuration"]["chunk_shape"]) + if length(shape) != length(cshape) + throw(ArgumentError("Shape has rank $(length(shape)) which does not match the chunk_shape rank of $(length(cshape))")) end + shape_jl = NTuple{length(shape),Int}(shape) |> reverse + chunks = GridChunks(shape_jl, NTuple{length(cshape),Int}(cshape) |> reverse) + elseif chunk_grid["name"] == "rectilinear" + chunk_grid["configuration"]["kind"] == "inline" || throw(ArgumentError("Only chunk grid descriptors of kind \"inline\" are allowed")) + cshapes = chunk_grid["configuration"]["chunk_shapes"] + length(cshapes) == length(shape) || throw(ArgumentError("chunk_shapes rank does not match the array shape rank")) + chunkspecs = map(cshapes, shape) do csh, s + if csh isa Integer + # a single integer declares a regular grid along this axis + RegularChunks(csh, 0, s) + else + # a list of edge lengths, possibly with [V, n] run-length encoding + chunksizes = Int[] + for spec in csh + if spec isa Int + push!(chunksizes, spec) + else + value, n = spec + for _ in 1:n + push!(chunksizes, value) + end + end + end + IrregularChunks(; chunksizes) + end + end + chunks = GridChunks(reverse(chunkspecs)...) else throw(ArgumentError("Unknown chunk_grid of name, $(chunk_grid["name"])")) end - # Chunk Key Encoding chunk_key_encoding = d["chunk_key_encoding"] @@ -257,7 +287,7 @@ function Metadata3(d::AbstractDict, fill_as_missing) T = typestr3(data_type) N = length(shape) - codec_ctx = (shape = shape, elsize = _sizeof(Base.nonmissingtype(T))) + codec_ctx = (shape=shape, elsize=_sizeof(Base.nonmissingtype(T))) pipeline = Codecs.V3Codecs.getCodec(d["codecs"], codec_ctx) fv = fill_value_decoding(d["fill_value"], T)::T @@ -267,11 +297,11 @@ function Metadata3(d::AbstractDict, fill_as_missing) chunk_key_encoding = parse_chunk_key_encoding(chunk_key_encoding) E = typeof(chunk_key_encoding) - MetadataV3{TU, N, typeof(pipeline), E}( + MetadataV3{TU,N,typeof(pipeline),E,typeof(chunks)}( zarr_format, node_type, - NTuple{N, Int}(shape) |> reverse, - NTuple{N, Int}(chunks) |> reverse, + NTuple{N,Int}(shape) |> reverse, + chunks, typestr3(T), pipeline, fv, @@ -280,21 +310,24 @@ function Metadata3(d::AbstractDict, fill_as_missing) end "Construct MetadataV3 based on your data" -function Metadata3(A::AbstractArray{T, N}, chunks::NTuple{N, Int}; - node_type::String="array", - compressor=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - endian::Symbol=:little, - filters=nothing, - fill_as_missing = false, - dimension_separator::Char = '/' - ) where {T, N} +function Metadata3(A::AbstractArray{T,N}, chunks; + node_type::String="array", + compressor=BloscCompressor(), + fill_value::Union{T,Nothing}=nothing, + order::Char=('C'), + endian::Symbol=:little, + filters=nothing, + fill_as_missing=false, + dimension_separator::Char=('/') +) where {T,N} T2 = (fill_value === nothing || !fill_as_missing) ? T : Union{T,Missing} if fill_value === nothing fill_value = zero(T) end - return MetadataV3{T2, N}( + if chunks isa Tuple + chunks = GridChunks(size(A), chunks) + end + return MetadataV3{T2,N}( 3, node_type, size(A), @@ -309,19 +342,38 @@ function Metadata3(A::AbstractArray{T, N}, chunks::NTuple{N, Int}; end function lower3(md::MetadataV3{T}) where T - chunk_grid = Dict{String,Any}( - "name" => "regular", - "configuration" => Dict{String,Any}( - "chunk_shape" => md.chunks |> reverse + + + chunk_grid = if any(i->isa(i, IrregularChunks), md.chunks[].chunks) + Dict{String,Any}( + "name" => "rectilinear", + "configuration" => Dict{String,Any}( + "kind" => "inline", + "chunk_shapes" => map(reverse(md.chunks[].chunks)) do c + if c isa RegularChunks + c.chunksize + else + length.(c) + end + end + ) ) - ) + else + Dict{String,Any}( + "name" => "regular", + "configuration" => Dict{String,Any}( + "chunk_shape" => reverse(DiskArrays.max_chunksize.(md.chunks[].chunks)) + ) + ) + end + # chunk_key_encoding chunk_key_encoding = lower_chunk_key_encoding(md.chunk_key_encoding) codecs = Codecs.V3Codecs._pipeline_to_codec_list(md.pipeline) - Dict{String, Any}( + Dict{String,Any}( "zarr_format" => Int(md.zarr_format), "node_type" => md.node_type, "shape" => md.shape[] |> reverse, @@ -333,16 +385,16 @@ function lower3(md::MetadataV3{T}) where T ) end -function Metadata(A::AbstractArray{T,N}, chunks::NTuple{N,Int}, ::ZarrFormat{3}; - node_type::String="array", - compressor::C=BloscCompressor(), - fill_value::Union{T, Nothing}=nothing, - order::Char='C', - endian::Symbol=:little, - filters::F=nothing, - fill_as_missing = false, - chunk_key_encoding::E=ChunkKeyEncoding('/', true) - ) where {T, N, C, F, E} +function Metadata(A::AbstractArray{T,N}, chunks, ::ZarrFormat{3}; + node_type::String="array", + compressor::C=BloscCompressor(), + fill_value::Union{T,Nothing}=nothing, + order::Char=('C'), + endian::Symbol=:little, + filters::F=nothing, + fill_as_missing=false, + chunk_key_encoding::E=ChunkKeyEncoding('/', true) +) where {T,N,C,F,E} return Metadata3(A, chunks; node_type=node_type, compressor=compressor, diff --git a/src/pipeline.jl b/lib/ZarrCore/src/pipeline.jl similarity index 100% rename from src/pipeline.jl rename to lib/ZarrCore/src/pipeline.jl diff --git a/lib/ZarrCore/src/public_names_core.jl b/lib/ZarrCore/src/public_names_core.jl new file mode 100644 index 00000000..538ee1e6 --- /dev/null +++ b/lib/ZarrCore/src/public_names_core.jl @@ -0,0 +1,39 @@ +public zname, zopen_noerr + +# Stores. Every store type is public; `DirectoryStore`, `S3Store` and `GCStore` +# are additionally exported for backwards compatibility. +public AbstractStore, DictStore, HTTPStore, ZipStore, CachingStore, + ConsolidatedStore + +public consolidate_metadata, writezip + +# The interface a new store backend has to implement, see `?AbstractStore`. +# (`storagesize` is part of it too, but is exported above.) +public subdirs, subkeys, isinitialized, storefromstring, + store_read_strategy, SequentialRead, ConcurrentRead, storageregexlist, + cloud_list_objects, concurrent_io_tasks + +# Chunk key encodings and the registry used to add new ones. +public AbstractChunkKeyEncoding, ChunkKeyEncoding, SuffixChunkKeyEncoding, + citostring, register_chunk_key_encoding, parse_chunk_key_encoding, + lower_chunk_key_encoding + +# Filters and the interface a new filter has to implement, see `?Filter`. +public Filter, VLenArrayFilter, VLenUTF8Filter, Fletcher32Filter, + FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter +public zencode, zdecode, getfilter, sourcetype, desttype, filterdict + +# Compressors and the interface a new compressor has to implement. +public Compressor, NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor +public zcompress, zcompress!, zuncompress, zuncompress!, getCompressor, + compressortypes + +# v3 codecs and the interface a new codec has to implement, see `?Codec`. +public Codecs, Codec, V3Codec, BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, + ShardingCodec, TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, + CRC32cV3Codec, VLenUTF8V3Codec + +# Data type and fill value encoding, needed to map Zarr dtypes to Julia types. +# `DateTime64` is re-exported from DateTimes64.jl because it shows up as the +# `eltype` of datetime arrays. +public typestr, fill_value_encoding, fill_value_decoding \ No newline at end of file diff --git a/src/types.jl b/lib/ZarrCore/src/types.jl similarity index 100% rename from src/types.jl rename to lib/ZarrCore/src/types.jl diff --git a/src/Zarr.jl b/src/Zarr.jl index 8292da9e..1dd4b8a9 100644 --- a/src/Zarr.jl +++ b/src/Zarr.jl @@ -1,33 +1,28 @@ module Zarr -import JSON -import Blosc -import Unicode -using OrderedCollections: OrderedDict +import ZarrCore -struct ZarrFormat{V} - version::Val{V} -end -Base.Int(v::ZarrFormat{V}) where V = V -@inline ZarrFormat(v::Int) = ZarrFormat(Val(v)) -ZarrFormat(v::ZarrFormat) = v -#Default Zarr Version -const DV = ZarrFormat(Val(2)) +# Mirror ZarrCore's export/public split. Internals stay at Zarr.ZarrCore. -include("types.jl") -include("chunkkeyencoding.jl") -include("metadata.jl") -include("metadata3.jl") -include("Compressors/Compressors.jl") -include("Codecs/Codecs.jl") -include("Storage/Storage.jl") -include("Filters/Filters.jl") -include("ZArray.jl") -include("pipeline.jl") -include("ZGroup.jl") -include("caching.jl") +for name in names(ZarrCore) + if name !== :ZarrCore + @eval import ZarrCore: $name + + if Base.isexported(ZarrCore, name) + @eval export $name + end + end +end -export ZArray, ZGroup, zopen, zzeros, zcreate, storagesize, storageratio, - zinfo, DirectoryStore, S3Store, GCStore, zgroup +@static if VERSION >= v"1.11" + include("public_names_zarr.jl") +else + # For Julia 1.10, we have to parse the public names from the source file, since + # `public` is not a keyword and `names(ZarrCore)` only returns exported names. + let public_names = read(joinpath(@__DIR__, "..", "lib", "ZarrCore", "src", "public_names_core.jl"), String) + public_names = replace(public_names, "public" => "using ZarrCore: ") + eval(Meta.parseall(public_names)) + end +end -end # module +end diff --git a/src/public_names_zarr.jl b/src/public_names_zarr.jl new file mode 100644 index 00000000..18c0075f --- /dev/null +++ b/src/public_names_zarr.jl @@ -0,0 +1,26 @@ +public zname + +# Stores. Every store type is public; `DirectoryStore`, `S3Store` and `GCStore` +# are additionally exported for backwards compatibility. +public DictStore, HTTPStore, ZipStore, CachingStore, + ConsolidatedStore + +public consolidate_metadata, writezip + +# Chunk key encodings and the registry used to add new ones. +public ChunkKeyEncoding, SuffixChunkKeyEncoding + +# Filters and the interface a new filter has to implement, see `?Filter`. +public Filter, VLenArrayFilter, VLenUTF8Filter, Fletcher32Filter, + FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter + +# Compressors and the interface a new compressor has to implement. +public Compressor, NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor + +# v3 codecs and the interface a new codec has to implement, see `?Codec`. +public Codecs, Codec, V3Codec, BloscCodec, BytesCodec, CRC32cCodec, GzipCodec, + ShardingCodec, TransposeCodec, GzipV3Codec, BloscV3Codec, ZstdV3Codec, + CRC32cV3Codec, VLenUTF8V3Codec + +# Data type and fill value encoding, needed to map Zarr dtypes to Julia types. +public typestr, fill_value_encoding, fill_value_decoding \ No newline at end of file diff --git a/test/Filters.jl b/test/Filters.jl index 5c09a8ca..0325850c 100644 --- a/test/Filters.jl +++ b/test/Filters.jl @@ -1,8 +1,9 @@ using Test -using Zarr: DateTime64 # for datetime reinterpret +using DateTimes64: DateTime64 # for datetime reinterpret using Zarr: zencode, zdecode using Zarr: Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter +import Zarr: ZarrCore @testset "Fletcher32Filter" begin # These tests are copied exactly from the [`numcodecs`](https://github.com/zarr-developers/numcodecs/) Python package, @@ -59,6 +60,7 @@ end end @testset "ShuffleFilter" begin + using DateTimes64: DateTime64 codecs = [ ShuffleFilter(), @@ -72,7 +74,7 @@ end LinRange(1000, 1001, 1000), # equivalent to np.linspace(1000, 1001, 1000, dtype='f8') reshape(randn(1000) .* 1 .+ 1000, (100, 10)), # equivalent to np.random.normal(loc=1000, scale=1, size=(100, 10)) reshape(rand(Bool, 1000), (10, 100)), # equivalent to np.random.randint(0, 2, size=1000, dtype=bool).reshape(100, 10, order='F') - reshape(rand(Zarr.MaxLengthString{3, UInt8}["a", "bb", "ccc"], 1000), (10, 10, 10)), # equivalent to np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10) + reshape(rand(ZarrCore.MaxLengthString{3, UInt8}["a", "bb", "ccc"], 1000), (10, 10, 10)), # equivalent to np.random.choice([b'a', b'bb', b'ccc'], size=1000).reshape(10, 10, 10) reinterpret(DateTime64{Dates.Nanosecond}, rand(UInt64(0):UInt64(2^60)-1, 1000)), # equivalent to np.random.randint(0, 2**60, size=1000, dtype='u8').view('M8[ns]') Nanosecond.(rand(UInt64(0):UInt64(2^60-1), 1000)), # equivalent to np.random.randint(0, 2**60, size=1000, dtype='u8').view('m8[ns]') reinterpret(DateTime64{Dates.Minute}, rand(UInt64(0):UInt64(2^25-1), 1000)), # equivalent to np.random.randint(0, 2**25, size=1000, dtype='u8').view('M8[m]') diff --git a/test/Project.toml b/test/Project.toml index 3c9c2469..5ca39ecd 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,7 +1,9 @@ [deps] AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" +DateTimes64 = "b342263e-b350-472a-b1a9-8dfd21b51589" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Minio = "4281f0d9-7ae0-406e-9172-b7277c1efa20" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" diff --git a/test/arraycache.jl b/test/arraycache.jl index dedb14d8..7ad1adf6 100644 --- a/test/arraycache.jl +++ b/test/arraycache.jl @@ -5,7 +5,7 @@ a .= reshape(1:200, 10, 20) # Start HTTP server - using Zarr.HTTP: HTTP + using Zarr.ZarrCore.HTTP: HTTP server = HTTP.serve!(g, "127.0.0.1", 0) port = server.bound_port @@ -15,7 +15,7 @@ g2 = zarrcache(base_array, cache_dir) - @test g2["a1"].cache.a.storage.folder == Zarr.normalize_path(joinpath(cache_dir, "a1")) + @test g2["a1"].cache.a.storage.folder == ZarrCore.normalize_path(joinpath(cache_dir, "a1")) # We also open the cache array on disk directly g_disk = zopen(cache_dir) @test g_disk.attrs == Dict("groupatt"=>5) @@ -30,7 +30,7 @@ # Now test if we can open the cache store from an existing path g3 = zarrcache(base_array, cache_dir) - @test g3["a1"].cache.a.storage.folder == Zarr.normalize_path(joinpath(cache_dir, "a1")) + @test g3["a1"].cache.a.storage.folder == ZarrCore.normalize_path(joinpath(cache_dir, "a1")) @test g3["a1"][1:5,6:10] == a[1:5,6:10] @test g_disk["a1"][1:5,6:10] == a[1:5,6:10] # Stop server diff --git a/test/consolidated.jl b/test/consolidated.jl index 553e3b19..03d95b38 100644 --- a/test/consolidated.jl +++ b/test/consolidated.jl @@ -1,9 +1,11 @@ using CondaPkg: CondaPkg, PkgSpec using JSON using PythonCall +using Zarr +import Zarr: ZarrCore CondaPkg.add([ - PkgSpec("numpy"), + PkgSpec("numpy"; version=">=2.3.3,<3"), PkgSpec("zarr"; version="3.*"), PkgSpec("numcodecs") ]) @@ -68,40 +70,40 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") @testset "ConsolidatedStore v3 constructor error paths" begin s = Zarr.DictStore() # missing zarr.json - @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", ZarrCore.ZarrFormat(3)) # zarr.json present but no consolidated_metadata s["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"node_type":"group"}""") - @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", ZarrCore.ZarrFormat(3)) # consolidated_metadata present but no metadata subkey s["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"consolidated_metadata":{"kind":"inline"}}""") - @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", ZarrCore.ZarrFormat(3)) end @testset "getmetadata v3 on ConsolidatedStore" begin path_jl = joinpath(path_v3_julia, "consolidated") cs = zopen(path_jl, consolidated=true) # getmetadata v3 reads from cons["metadata"][key] - meta = Zarr.getmetadata(Zarr.ZarrFormat(3), cs.storage, "1d.chunked.i2", false) + meta = ZarrCore.getmetadata(ZarrCore.ZarrFormat(3), cs.storage, "1d.chunked.i2", false) @test eltype(meta) == Int16 - @test meta.chunks == (2,) + @test meta.chunks[] == ZarrCore.DiskArrays.GridChunks((4,), (2,)) end @testset "is_zarray / is_zgroup v3 on ConsolidatedStore" begin path_jl = joinpath(path_v3_julia, "consolidated") cs = zopen(path_jl, consolidated=true) s = cs.storage # the ConsolidatedStore - V3 = Zarr.ZarrFormat(3) - @test Zarr.is_zarray(V3, s, "1d.chunked.i2") - @test !Zarr.is_zarray(V3, s, "nested") - @test !Zarr.is_zarray(V3, s, "nonexistent") - @test Zarr.is_zgroup(V3, s, "nested") - @test !Zarr.is_zgroup(V3, s, "1d.chunked.i2") - @test !Zarr.is_zgroup(V3, s, "nonexistent") + V3 = ZarrCore.ZarrFormat(3) + @test ZarrCore.is_zarray(V3, s, "1d.chunked.i2") + @test !ZarrCore.is_zarray(V3, s, "nested") + @test !ZarrCore.is_zarray(V3, s, "nonexistent") + @test ZarrCore.is_zgroup(V3, s, "nested") + @test !ZarrCore.is_zgroup(V3, s, "1d.chunked.i2") + @test !ZarrCore.is_zgroup(V3, s, "nonexistent") end @testset "subdirs v3 on ConsolidatedStore" begin ds = Zarr.DirectoryStore(joinpath(path_v3_julia, "consolidated")) - cs = Zarr.ConsolidatedStore(ds, "", Zarr.ZarrFormat(3)) + cs = Zarr.ConsolidatedStore(ds, "", ZarrCore.ZarrFormat(3)) # v3 subdirs returns only immediate children (length(sp) == lp + 1) dirs = Zarr.subdirs(cs, "") @test sort(dirs) == ["1d.chunked.i2", "2d.contiguous.i2", "nested"] @@ -149,28 +151,28 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") # Missing .zmetadata s = Zarr.DictStore() zgroup(s) - @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", Zarr.ZarrFormat(2)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", ZarrCore.ZarrFormat(2)) # .zmetadata present but missing "metadata" field s2 = Zarr.DictStore() s2[".zmetadata"] = Vector{UInt8}("""{"zarr_consolidated_format":1}""") - @test_throws ArgumentError Zarr.ConsolidatedStore(s2, "", Zarr.ZarrFormat(2)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s2, "", ZarrCore.ZarrFormat(2)) end @testset "v3 ConsolidatedStore constructor errors" begin # Missing zarr.json s = Zarr.DictStore() - @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s, "", ZarrCore.ZarrFormat(3)) # zarr.json present but no consolidated_metadata s2 = Zarr.DictStore() s2["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"node_type":"group"}""") - @test_throws ArgumentError Zarr.ConsolidatedStore(s2, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s2, "", ZarrCore.ZarrFormat(3)) # consolidated_metadata present but no metadata subkey s3 = Zarr.DictStore() s3["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"consolidated_metadata":{"kind":"inline"}}""") - @test_throws ArgumentError Zarr.ConsolidatedStore(s3, "", Zarr.ZarrFormat(3)) + @test_throws ArgumentError Zarr.ConsolidatedStore(s3, "", ZarrCore.ZarrFormat(3)) end @testset "auto-detect constructor errors when no format found" begin @@ -184,12 +186,12 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") zcreate(Int, g, "arr", 4, chunks=(2,), attrs=Dict("b" => 2)) zgroup(g, "sub") # group with no attrs cs = Zarr.consolidate_metadata(s, "") - V2 = Zarr.ZarrFormat(2) - @test Zarr.getattrs(V2, cs, "") == Dict("a" => 1) - @test Zarr.getattrs(V2, cs, "arr") == Dict("b" => 2) + V2 = ZarrCore.ZarrFormat(2) + @test ZarrCore.getattrs(V2, cs, "") == Dict("a" => 1) + @test ZarrCore.getattrs(V2, cs, "arr") == Dict("b" => 2) # No .zattrs written for sub → returns empty dict - @test Zarr.getattrs(V2, cs, "sub") == Dict{String,Any}() - @test Zarr.getattrs(V2, cs, "nonexistent") == Dict{String,Any}() + @test ZarrCore.getattrs(V2, cs, "sub") == Dict{String,Any}() + @test ZarrCore.getattrs(V2, cs, "nonexistent") == Dict{String,Any}() end @testset "v2 is_zarray / is_zgroup" begin @@ -198,15 +200,15 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") zgroup(g, "sub") zcreate(Float64, g, "data", 8, chunks=(4,)) cs = Zarr.consolidate_metadata(s, "") - V2 = Zarr.ZarrFormat(2) - @test Zarr.is_zgroup(V2, cs, "") - @test !Zarr.is_zarray(V2, cs, "") - @test Zarr.is_zgroup(V2, cs, "sub") - @test !Zarr.is_zarray(V2, cs, "sub") - @test Zarr.is_zarray(V2, cs, "data") - @test !Zarr.is_zgroup(V2, cs, "data") - @test !Zarr.is_zarray(V2, cs, "nonexistent") - @test !Zarr.is_zgroup(V2, cs, "nonexistent") + V2 = ZarrCore.ZarrFormat(2) + @test ZarrCore.is_zgroup(V2, cs, "") + @test !ZarrCore.is_zarray(V2, cs, "") + @test ZarrCore.is_zgroup(V2, cs, "sub") + @test !ZarrCore.is_zarray(V2, cs, "sub") + @test ZarrCore.is_zarray(V2, cs, "data") + @test !ZarrCore.is_zgroup(V2, cs, "data") + @test !ZarrCore.is_zarray(V2, cs, "nonexistent") + @test !ZarrCore.is_zgroup(V2, cs, "nonexistent") end @testset "v2 subdirs" begin @@ -227,10 +229,10 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") a = zcreate(Int32, g, "arr", 10, 20, chunks=(5,5), fill_value=Int32(-1)) # I think this `fill_value` way of passing things is related to issue: https://github.com/JuliaIO/Zarr.jl/issues/292 cs = Zarr.consolidate_metadata(s, "") - V2 = Zarr.ZarrFormat(2) - meta = Zarr.getmetadata(V2, cs, "arr", false) + V2 = ZarrCore.ZarrFormat(2) + meta = ZarrCore.getmetadata(V2, cs, "arr", false) @test meta.dtype == Int32 || eltype(meta) == Int32 - @test meta.chunks == (5, 5) + @test meta.chunks[] == ZarrCore.DiskArrays.GridChunks((10, 20), (5, 5)) end @testset "v3 getattrs" begin @@ -242,16 +244,16 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") ) ) cs = Zarr.ConsolidatedStore(Zarr.DictStore(), "", cons) - @test Zarr.getattrs(Zarr.ZarrFormat(3), cs, "") == Dict("foo" => "bar") + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), cs, "") == Dict("foo" => "bar") # zarr.json present but no "attributes" cs2 = Zarr.ConsolidatedStore(Zarr.DictStore(), "", Dict{String,Any}("zarr.json" => Dict{String,Any}("node_type" => "group"))) - @test Zarr.getattrs(Zarr.ZarrFormat(3), cs2, "") == Dict{String,Any}() + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), cs2, "") == Dict{String,Any}() # zarr.json key absent entirely cs3 = Zarr.ConsolidatedStore(Zarr.DictStore(), "", Dict{String,Any}()) - @test Zarr.getattrs(Zarr.ZarrFormat(3), cs3, "") == Dict{String,Any}() + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), cs3, "") == Dict{String,Any}() end @testset "v3 is_zarray / is_zgroup on ConsolidatedStore (unit)" begin @@ -270,15 +272,15 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") s = Zarr.DictStore() s["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"node_type":"group"}""") cs = Zarr.ConsolidatedStore(s, "", cons) - V3 = Zarr.ZarrFormat(3) + V3 = ZarrCore.ZarrFormat(3) # is_zarray: key = "group1/arr" (no suffix) - @test Zarr.is_zarray(V3, cs, "group1/arr") - @test !Zarr.is_zarray(V3, cs, "group1") # no bare "group1" key - @test !Zarr.is_zarray(V3, cs, "nonexistent") + @test ZarrCore.is_zarray(V3, cs, "group1/arr") + @test !ZarrCore.is_zarray(V3, cs, "group1") # no bare "group1" key + @test !ZarrCore.is_zarray(V3, cs, "nonexistent") # is_zgroup: key = "group1/zarr.json" (_unconcpath + "zarr.json") - @test Zarr.is_zgroup(V3, cs, "group1") - @test !Zarr.is_zgroup(V3, cs, "group1/arr") # "group1/arr/zarr.json" not in metadata - @test !Zarr.is_zgroup(V3, cs, "nonexistent") + @test ZarrCore.is_zgroup(V3, cs, "group1") + @test !ZarrCore.is_zgroup(V3, cs, "group1/arr") # "group1/arr/zarr.json" not in metadata + @test !ZarrCore.is_zgroup(V3, cs, "nonexistent") end @testset "is_zgroup v3 fallback to parent" begin @@ -290,18 +292,18 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") # write a real group zarr.json to the parent so the fallback succeeds s["zarr.json"] = Vector{UInt8}("""{"zarr_format":3,"node_type":"group"}""") cs = Zarr.ConsolidatedStore(s, "", cons) - V3 = Zarr.ZarrFormat(3) + V3 = ZarrCore.ZarrFormat(3) # not in cons["metadata"], falls back to parent — parent has zarr.json with node_type=group - @test Zarr.is_zgroup(V3, cs, "") + @test ZarrCore.is_zgroup(V3, cs, "") # not in cons["metadata"], fallback finds nothing → false - @test !Zarr.is_zgroup(V3, cs, "nonexistent") + @test !ZarrCore.is_zgroup(V3, cs, "nonexistent") end @testset "consolidate_metadata v3 adds consolidated_metadata when missing" begin tmp = mktempdir() try ds = Zarr.DirectoryStore(tmp) - g = zgroup(ds, "", Zarr.ZarrFormat(3)) + g = zgroup(ds, "", ZarrCore.ZarrFormat(3)) zcreate(Int16, g, "arr", 4, chunks=(2,), compressor=Zarr.NoCompressor()) zj_path = joinpath(tmp, "zarr.json") @@ -313,7 +315,7 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") JSON.print(io, root, 4) end # act - cs = Zarr.consolidate_metadata(ds, "", Zarr.ZarrFormat(3)) + cs = Zarr.consolidate_metadata(ds, "", ZarrCore.ZarrFormat(3)) # type check @test cs isa Zarr.ConsolidatedStore # reload file @@ -374,17 +376,17 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") s = Zarr.DictStore() zgroup(s) cs = Zarr.consolidate_metadata(s, "") - @test Zarr.ZarrFormat(cs, "") == Zarr.ZarrFormat(s, "") + @test ZarrCore.ZarrFormat(cs, "") == ZarrCore.ZarrFormat(s, "") end @testset "_unconcpath" begin s = Zarr.DictStore() cs = Zarr.ConsolidatedStore(s, "a/b", Dict{String,Any}()) - @test Zarr._unconcpath(cs, "a/b/c/d") == "c/d" - @test Zarr._unconcpath(cs, "a/b") == "" - @test_throws ErrorException Zarr._unconcpath(cs, "x/y") + @test ZarrCore._unconcpath(cs, "a/b/c/d") == "c/d" + @test ZarrCore._unconcpath(cs, "a/b") == "" + @test_throws ErrorException ZarrCore._unconcpath(cs, "x/y") # with suffix - @test Zarr._unconcpath(cs, "a/b/c", ".zarray") == "c/.zarray" + @test ZarrCore._unconcpath(cs, "a/b/c", ".zarray") == "c/.zarray" end @testset "store_read_strategy and has_configurable_missing_chunks delegate" begin @@ -392,7 +394,7 @@ path_v3_julia = joinpath(@__DIR__, "v3_julia", "data.zarr") zgroup(s) cs = Zarr.consolidate_metadata(s, "") @test Zarr.store_read_strategy(cs) == Zarr.store_read_strategy(s) - @test Zarr.has_configurable_missing_chunks(cs) == Zarr.has_configurable_missing_chunks(s) + @test ZarrCore.has_configurable_missing_chunks(cs) == ZarrCore.has_configurable_missing_chunks(s) end @testset "v2 full data round-trip through ConsolidatedStore" begin diff --git a/test/http_sharded.jl b/test/http_sharded.jl index 2f567633..fa6ff83a 100644 --- a/test/http_sharded.jl +++ b/test/http_sharded.jl @@ -45,7 +45,7 @@ const TESSERA_BASE = "https://dl2.geotessera.org/zarr/v1/2024.zarr" # Julia reverses to column-major: (128, 66560, 1355776) @test size(z) == (128, 66560, 1355776) # Outer shard shape [256, 256, 128] → Julia (128, 256, 256) - @test z.metadata.chunks == (128, 256, 256) + @test DiskArrays.max_chunksize.(z.metadata.chunks[].chunks) == (128, 256, 256) # Verify the codec is sharding_indexed with the expected inner chunk shape pipeline = z.metadata.pipeline @@ -144,7 +144,7 @@ const FLAMINGO_BASE = "https://radosgw.public.os.wwu.de/n4bi-goe" @test eltype(z) == UInt16 @test ndims(z) == 3 @test size(z) == (1024, 1024, 192) - @test z.metadata.chunks == (512, 512, 512) + @test DiskArrays.max_chunksize.(z.metadata.chunks[].chunks) == (512, 512, 512) sharding = z.metadata.pipeline.array_bytes @test sharding isa Zarr.Codecs.V3Codecs.ShardingCodec @@ -173,7 +173,7 @@ const MUENSTER_BASE = "https://radosgw.public.os2.wwu.de/ngff" @test eltype(z) == UInt8 @test ndims(z) == 3 @test size(z) == (6000, 6000, 6000) - @test z.metadata.chunks == (8192, 8192, 1) + @test DiskArrays.max_chunksize.(z.metadata.chunks[].chunks) == (8192, 8192, 1) sharding = z.metadata.pipeline.array_bytes @test sharding isa Zarr.Codecs.V3Codecs.ShardingCodec @@ -234,7 +234,7 @@ end # @testset "Remote HTTP sharded arrays (SSBD — RIKEN)" @test eltype(z) == UInt16 @test ndims(z) == 5 @test size(z) == (522693, 244215, 1, 3, 1) - @test z.metadata.chunks == (2048, 2048, 1, 1, 1) + @test DiskArrays.max_chunksize.(z.metadata.chunks[].chunks) == (2048, 2048, 1, 1, 1) sharding = z.metadata.pipeline.array_bytes @test sharding isa Zarr.Codecs.V3Codecs.ShardingCodec diff --git a/test/python.jl b/test/python.jl index 749f3b95..b604c15f 100644 --- a/test/python.jl +++ b/test/python.jl @@ -12,306 +12,369 @@ CondaPkg.add([ @testset "Python zarr implementation" begin -import Mmap -using PythonCall -#If we are on conda, import zarr -zarr = pyimport("zarr") -zarr_storage = pyimport("zarr.storage") - -#Create some directories -proot = tempname() -mkpath(proot) -pjulia = joinpath(proot,"julia") -ppython = joinpath(proot,"python") - -#First create an array in Julia and read with python zarr -groupattrs = Dict("String attribute"=>"One", "Int attribute"=>5, "Float attribute"=>10.5) -g = zgroup(pjulia,attrs=groupattrs) - -# Test all supported data types and compressors -import Zarr: NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor, MaxLengthString, - Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter -using Random: randstring -numeric_dtypes = (UInt8, UInt16, UInt32, UInt64, - Int8, Int16, Int32, Int64, - Float16, Float32, Float64, - Complex{Float32}, Complex{Float64}, - Bool,) -dtypes = (numeric_dtypes..., - MaxLengthString{10,UInt8},MaxLengthString{10,UInt32}, - String) -dtypesp = ("uint8","uint16","uint32","uint64", - "int8","int16","int32","int64", - "float16","float32","float64", - "complex64", "complex128","bool","S10","U10", "O") -compressors = ( - "no"=>NoCompressor(), - "blosc"=>BloscCompressor(cname="zstd"), - "blosc_autoshuffle"=>BloscCompressor(cname="zstd",shuffle=-1), - "blosc_noshuffle"=>BloscCompressor(cname="zstd",shuffle=0), - "blosc_bitshuffle"=>BloscCompressor(cname="zstd",shuffle=2), - "zlib"=>ZlibCompressor(), - "zlib_2"=>ZlibCompressor(;clevel=2), - "zstd"=>ZstdCompressor(), -) -filters = ( - "fletcher32"=>Fletcher32Filter(), - "scale_offset"=>FixedScaleOffsetFilter(offset=1000, scale=10^6, T=Float64, Tenc=Int32), - "shuffle"=>ShuffleFilter(elementsize=4), - "quantize"=>QuantizeFilter{Float64,Float32}(digits=5), - "delta"=>DeltaFilter{Int32}() -) -testarrays = Dict(t=>(t<:AbstractString) ? [randstring(maximum(i.I)) for i in CartesianIndices((1:10,1:6,1:2))] : rand(t,10,6,2) for t in dtypes) -testzerodimarrays = Dict(t=>(t<:AbstractString) ? randstring(10) : rand(t) for t in dtypes) - -# Test arrays with compressors -for t in dtypes, co in compressors - compstr, comp = co - att = Dict("This is a nested attribute"=>Dict("a"=>5)) - a = zcreate(t, g,string("a",t,compstr),10,6,2,attrs=att, chunks = (5,2,2),compressor=comp) - a[:,:,:] = testarrays[t] - - a = zcreate(t, g,string("azerodim",t,compstr), compressor=comp) - a[] = testzerodimarrays[t] -end - -# Test arrays with filters -for (filterstr, filter) in filters - t = eltype(filter) == Any ? Float64 : eltype(filter) - att = Dict("Filter test attribute"=>Dict("b"=>6)) - a = zcreate(t, g,string("filter_",filterstr),10,6,2,attrs=att, chunks = (5,2,2),filters=[filter]) - testdata = rand(t,10,6,2) - a[:,:,:] = testdata - - # Test zero-dimensional array - a = zcreate(t, g,string("filter_zerodim_",filterstr), filters=[filter]) - testzerodim = rand(t) - a[] = testzerodim -end - -#Also save as zip file. -open(pjulia*".zip";write=true) do io - Zarr.writezip(io, g) -end + import Mmap + using PythonCall + #If we are on conda, import zarr + zarr = pyimport("zarr") + zarr_storage = pyimport("zarr.storage") + + #Create some directories + proot = tempname() + mkpath(proot) + pjulia = joinpath(proot, "julia") + ppython = joinpath(proot, "python") + + #First create an array in Julia and read with python zarr + groupattrs = Dict("String attribute"=>"One", "Int attribute"=>5, "Float attribute"=>10.5) + g = zgroup(pjulia, attrs=groupattrs) + + # Test all supported data types and compressors + import Zarr: NoCompressor, BloscCompressor, ZlibCompressor, ZstdCompressor, + Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter + import Zarr: ZarrCore + using Random: randstring + numeric_dtypes = (UInt8, UInt16, UInt32, UInt64, + Int8, Int16, Int32, Int64, + Float16, Float32, Float64, + Complex{Float32}, Complex{Float64}, + Bool,) + dtypes = (numeric_dtypes..., + ZarrCore.MaxLengthString{10,UInt8}, ZarrCore.MaxLengthString{10,UInt32}, + String) + dtypesp = ("uint8", "uint16", "uint32", "uint64", + "int8", "int16", "int32", "int64", + "float16", "float32", "float64", + "complex64", "complex128", "bool", "S10", "U10", "O") + compressors = ( + "no"=>NoCompressor(), + "blosc"=>BloscCompressor(cname="zstd"), + "blosc_autoshuffle"=>BloscCompressor(cname="zstd", shuffle=-1), + "blosc_noshuffle"=>BloscCompressor(cname="zstd", shuffle=0), + "blosc_bitshuffle"=>BloscCompressor(cname="zstd", shuffle=2), + "zlib"=>ZlibCompressor(), + "zlib_2"=>ZlibCompressor(; clevel=2), + "zstd"=>ZstdCompressor(), + ) + filters = ( + "fletcher32"=>Fletcher32Filter(), + "scale_offset"=>FixedScaleOffsetFilter(offset=1000, scale=10^6, T=Float64, Tenc=Int32), + "shuffle"=>ShuffleFilter(elementsize=4), + "quantize"=>QuantizeFilter{Float64,Float32}(digits=5), + "delta"=>DeltaFilter{Int32}() + ) + testarrays = Dict(t=>(t<:AbstractString) ? [randstring(maximum(i.I)) for i in CartesianIndices((1:10, 1:6, 1:2))] : rand(t, 10, 6, 2) for t in dtypes) + testzerodimarrays = Dict(t=>(t<:AbstractString) ? randstring(10) : rand(t) for t in dtypes) + + # Test arrays with compressors + for t in dtypes, co in compressors + compstr, comp = co + att = Dict("This is a nested attribute"=>Dict("a"=>5)) + a = zcreate(t, g, string("a", t, compstr), 10, 6, 2, attrs=att, chunks=(5, 2, 2), compressor=comp) + a[:, :, :] = testarrays[t] + + a = zcreate(t, g, string("azerodim", t, compstr), compressor=comp) + a[] = testzerodimarrays[t] + end -@testset "reading in julia" begin - g = zopen(pjulia) - #Test group attributes - @test g.attrs == groupattrs - for (t, co) in Iterators.product(dtypes, compressors) - compstr,comp = co - arname = string("a",t,compstr) - ar = g[arname] - @test ar.attrs == Dict("This is a nested attribute"=>Dict("a"=>5)) - @test ar == testarrays[t] + # Test arrays with filters + for (filterstr, filter) in filters + t = eltype(filter) == Any ? Float64 : eltype(filter) + att = Dict("Filter test attribute"=>Dict("b"=>6)) + a = zcreate(t, g, string("filter_", filterstr), 10, 6, 2, attrs=att, chunks=(5, 2, 2), filters=[filter]) + testdata = rand(t, 10, 6, 2) + a[:, :, :] = testdata + + # Test zero-dimensional array + a = zcreate(t, g, string("filter_zerodim_", filterstr), filters=[filter]) + testzerodim = rand(t) + a[] = testzerodim end -end -# Test reading in python -for julia_path in (pjulia, pjulia*".zip") - # zarr-python 3.x requires explicitly opening zip files as ZipStore - if endswith(julia_path, ".zip") - store = zarr_storage.ZipStore(julia_path) - g = zarr.open_group(store=store, mode="r") - else - g = zarr.open_group(julia_path, mode="r") + #Also save as zip file. + open(pjulia*".zip"; write=true) do io + Zarr.writezip(io, g) end - gatts = pyconvert(Any, g.attrs) - - #Test group attributes - @test gatts["String attribute"] == "One" - @test gatts["Int attribute"] == 5 - @test gatts["Float attribute"] == 10.5 - - #Test accessing arrays from python and reading data - for i=1:length(dtypes), co in compressors - compstr,comp = co - t = dtypes[i] - tp = dtypesp[i] - # zarr-python 3.x does not support fixed-length (5) - @test pyeq(Bool, ar.dtype, tp) - @test pyconvert(Tuple, ar.shape) == (2,6,10) - @test PyArray(ar[pybuiltins.Ellipsis]) == permutedims(testarrays[t],(3,2,1)) + + @testset "reading in julia" begin + g = zopen(pjulia) + #Test group attributes + @test g.attrs == groupattrs + for (t, co) in Iterators.product(dtypes, compressors) + compstr, comp = co + arname = string("a", t, compstr) + ar = g[arname] + @test ar.attrs == Dict("This is a nested attribute"=>Dict("a"=>5)) + @test ar == testarrays[t] + end end - # Test reading filtered arrays from python - for (filterstr, filter) in filters - t = eltype(filter) == Any ? Float64 : eltype(filter) - arname = string("filter_",filterstr) - local ar - try + # Test reading in python + for julia_path in (pjulia, pjulia*".zip") + # zarr-python 3.x requires explicitly opening zip files as ZipStore + if endswith(julia_path, ".zip") + store = zarr_storage.ZipStore(julia_path) + g = zarr.open_group(store=store, mode="r") + else + g = zarr.open_group(julia_path, mode="r") + end + gatts = pyconvert(Any, g.attrs) + + #Test group attributes + @test gatts["String attribute"] == "One" + @test gatts["Int attribute"] == 5 + @test gatts["Float attribute"] == 10.5 + + #Test accessing arrays from python and reading data + for i=1:length(dtypes), co in compressors + compstr, comp = co + t = dtypes[i] + tp = dtypesp[i] + # zarr-python 3.x does not support fixed-length (5) + @test pyeq(Bool, ar.dtype, tp) + @test pyconvert(Tuple, ar.shape) == (2, 6, 10) + @test PyArray(ar[pybuiltins.Ellipsis]) == permutedims(testarrays[t], (3, 2, 1)) end - - @test pyconvert(Any, ar.attrs["Filter test attribute"]) == Dict("b"=>6) - @test pyconvert(Tuple, ar.shape) == (2,6,10) - - # Test zero-dimensional filtered array - arname = string("filter_zerodim_",filterstr) - ar_zero=g[arname] - @test pyconvert(Tuple, ar_zero.shape) == () - end - for i=1:length(dtypes), co in compressors - compstr,comp = co - t = dtypes[i] - tp = dtypesp[i] - if t == UInt64 - continue - # need to exclude UInt64: - # need explicit conversion because of https://github.com/JuliaPy/PyCall.jl/issues/744 - # but explicit conversion uses PyLong_AsLongLongAndOverflow, which converts everything - # to a signed 64-bit integer, which can error out if the UInt64 is too large. - # Adding an overload to PyCall for unsigned ints doesn't work with NumPy scalars because - # they are not subtypes of integer: https://stackoverflow.com/a/58816671 + # Test reading filtered arrays from python + for (filterstr, filter) in filters + t = eltype(filter) == Any ? Float64 : eltype(filter) + arname = string("filter_", filterstr) + local ar + try + ar=g[arname] + catch e + @error "Error loading group with filter $filterstr" exception=(e, catch_backtrace()) + @test false # test failed. + end + + @test pyconvert(Any, ar.attrs["Filter test attribute"]) == Dict("b"=>6) + @test pyconvert(Tuple, ar.shape) == (2, 6, 10) + + # Test zero-dimensional filtered array + arname = string("filter_zerodim_", filterstr) + ar_zero=g[arname] + @test pyconvert(Tuple, ar_zero.shape) == () end - # zarr-python 3.x does not support fixed-length ("Hi"))) -z1 = g.require_array("a1", shape=(2,6,10), chunks=(1,2,3), dtype="i4") -z1[pybuiltins.Ellipsis] = numpy.array(data) -z1.update_attributes(pydict(Dict("test" => pydict(Dict("b" => 6))))) -z2 = g.require_array("a2", shape=(5,), chunks=(5,), dtype="S1", compressor=numcodecs.Zlib()) -z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"]) -z3 = g.require_array("a3", shape=(2,), dtype=pybuiltins.str) -z3[pybuiltins.Ellipsis]=numpy.asarray(["test1", "test234"], dtype="O") -z4 = g.require_array("a4", shape=(2,3), chunks=(2,3), dtype="c16", fill_value=cfill) -z4[pybuiltins.Ellipsis] = numpy.array(cdata) -zarr.consolidate_metadata(ppython) - -#Open in Julia -g = zopen(ppython) -@test g isa Zarr.ZGroup -@test g.attrs["groupatt"] == "Hi" -a1 = g["a1"] -@test a1 isa ZArray -@test a1[:,:,:]==permutedims(data,(3,2,1)) -@test a1.attrs["test"]==Dict("b"=>6) -# Test reading the string array -@test String(g["a2"][:])=="hallo" -@test g["a3"] == ["test1", "test234"] -a4 = g["a4"] -@test eltype(a4) === ComplexF64 -@test a4.metadata.fill_value === cfill -@test a4[:,:] == permutedims(cdata,(2,1)) - -# And test for consolidated metadata -# Delete files so we make sure they are not accessed -rm(joinpath(ppython,".zattrs")) -rm(joinpath(ppython,"a1",".zattrs")) -rm(joinpath(ppython,"a1",".zarray")) -rm(joinpath(ppython,"a2",".zarray")) -g = zopen(ppython, "w", consolidated=true) -@test g isa Zarr.ZGroup -@test g.attrs["groupatt"] == "Hi" -a1 = g["a1"] -@test a1 isa ZArray -@test a1[:,:,:]==permutedims(data,(3,2,1)) -@test a1.attrs["test"]==Dict("b"=>6) -@test storagesize(a1) == 960 -@test sort(Zarr.subkeys(a1.storage,"a1"))[1:5] == ["0.0.0","0.0.1","0.0.2","0.0.3","0.1.0"] -a1[:,1,1] = 1:10 -@test a1[:,1,1] == 1:10 -# Test reading the string array -@test String(g["a2"][:])=="hallo" - - -# Test zip file can be read -ppythonzip = ppython*".zip" -store = zarr_storage.ZipStore(ppythonzip, mode="w") -g = zarr.open_group(store=store, mode="w", zarr_format=2) -g.update_attributes(pydict(Dict("groupatt"=>"Hi"))) -z1 = g.require_array("a1", shape=(2,6,10), chunks=(1,2,3), dtype="i4") -z1[pybuiltins.Ellipsis] = numpy.array(data) -z1.update_attributes(pydict(Dict("test" => pydict(Dict("b" => 6))))) -z2 = g.require_array("a2", shape=(5,), chunks=(5,), dtype="S1", compressor=numcodecs.Zlib()) -z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"]) -z3 = g.require_array("a3", shape=(2,), dtype=pybuiltins.str) -z3[pybuiltins.Ellipsis] = numpy.asarray(["test1", "test234"], dtype="O") -store.close() - -g = zopen(Zarr.ZipStore(Mmap.mmap(ppythonzip))) -@test g isa Zarr.ZGroup -@test g.attrs["groupatt"] == "Hi" -a1 = g["a1"] -@test a1 isa ZArray -@test a1[:,:,:]==permutedims(data,(3,2,1)) -@test a1.attrs["test"]==Dict("b"=>6) -# Test reading the string array -@test String(g["a2"][:])=="hallo" -@test g["a3"] == ["test1", "test234"] + ## Now the other way around, we create a zarr array using the python lib and read back into julia + data = rand(Int32, 2, 6, 10) + cdata = rand(ComplexF64, 2, 3) + cfill = ComplexF64(1.5, -2.5) + + numpy = pyimport("numpy") + numcodecs = pyimport("numcodecs") + g = zarr.open_group(ppython, mode="w", zarr_format=2) + g.update_attributes(pydict(Dict("groupatt"=>"Hi"))) + z1 = g.require_array("a1", shape=(2, 6, 10), chunks=(1, 2, 3), dtype="i4") + z1[pybuiltins.Ellipsis] = numpy.array(data) + z1.update_attributes(pydict(Dict("test" => pydict(Dict("b" => 6))))) + z2 = g.require_array("a2", shape=(5,), chunks=(5,), dtype="S1", compressor=numcodecs.Zlib()) + z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"]) + z3 = g.require_array("a3", shape=(2,), dtype=pybuiltins.str) + z3[pybuiltins.Ellipsis]=numpy.asarray(["test1", "test234"], dtype="O") + z4 = g.require_array("a4", shape=(2, 3), chunks=(2, 3), dtype="c16", fill_value=cfill) + z4[pybuiltins.Ellipsis] = numpy.array(cdata) + zarr.consolidate_metadata(ppython) + + #Open in Julia + g = zopen(ppython) + @test g isa Zarr.ZGroup + @test g.attrs["groupatt"] == "Hi" + a1 = g["a1"] + @test a1 isa ZArray + @test a1[:, :, :]==permutedims(data, (3, 2, 1)) + @test a1.attrs["test"]==Dict("b"=>6) + # Test reading the string array + @test String(g["a2"][:])=="hallo" + @test g["a3"] == ["test1", "test234"] + a4 = g["a4"] + @test eltype(a4) === ComplexF64 + @test a4.metadata.fill_value === cfill + @test a4[:, :] == permutedims(cdata, (2, 1)) + + # And test for consolidated metadata + # Delete files so we make sure they are not accessed + rm(joinpath(ppython, ".zattrs")) + rm(joinpath(ppython, "a1", ".zattrs")) + rm(joinpath(ppython, "a1", ".zarray")) + rm(joinpath(ppython, "a2", ".zarray")) + g = zopen(ppython, "w", consolidated=true) + @test g isa Zarr.ZGroup + @test g.attrs["groupatt"] == "Hi" + a1 = g["a1"] + @test a1 isa ZArray + @test a1[:, :, :]==permutedims(data, (3, 2, 1)) + @test a1.attrs["test"]==Dict("b"=>6) + @test storagesize(a1) == 960 + @test sort(Zarr.subkeys(a1.storage, "a1"))[1:5] == ["0.0.0", "0.0.1", "0.0.2", "0.0.3", "0.1.0"] + a1[:, 1, 1] = 1:10 + @test a1[:, 1, 1] == 1:10 + # Test reading the string array + @test String(g["a2"][:])=="hallo" + + + # Test zip file can be read + ppythonzip = ppython*".zip" + store = zarr_storage.ZipStore(ppythonzip, mode="w") + g = zarr.open_group(store=store, mode="w", zarr_format=2) + g.update_attributes(pydict(Dict("groupatt"=>"Hi"))) + z1 = g.require_array("a1", shape=(2, 6, 10), chunks=(1, 2, 3), dtype="i4") + z1[pybuiltins.Ellipsis] = numpy.array(data) + z1.update_attributes(pydict(Dict("test" => pydict(Dict("b" => 6))))) + z2 = g.require_array("a2", shape=(5,), chunks=(5,), dtype="S1", compressor=numcodecs.Zlib()) + z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"]) + z3 = g.require_array("a3", shape=(2,), dtype=pybuiltins.str) + z3[pybuiltins.Ellipsis] = numpy.asarray(["test1", "test234"], dtype="O") + store.close() + + g = zopen(Zarr.ZipStore(Mmap.mmap(ppythonzip))) + @test g isa Zarr.ZGroup + @test g.attrs["groupatt"] == "Hi" + a1 = g["a1"] + @test a1 isa ZArray + @test a1[:, :, :]==permutedims(data, (3, 2, 1)) + @test a1.attrs["test"]==Dict("b"=>6) + # Test reading the string array + @test String(g["a2"][:])=="hallo" + @test g["a3"] == ["test1", "test234"] end @testset "Python datetime types" begin -using Dates, Test, Zarr, PythonCall -vd = Date(1970,1,1):Day(1):Date(1970,6,30) |> collect -vt = DateTime(1970,1,1):Second(1):DateTime(1970,1,1,2,0,0)|> collect -ad = ZArray(vd) -at = ZArray(vt) -@test eltype(ad)==Zarr.DateTime64{Day} -@test eltype(at)==Zarr.DateTime64{Millisecond} -@test DateTime.(at[:]) == vt[:] -@test Date.(ad[:]) == vd[:] - -p = tempname() -g = zgroup(p) -for pt in [Week, Day, Hour, Minute, Second, + using Dates, Test, Zarr, PythonCall + using DateTimes64: DateTime64 + vd = Date(1970, 1, 1):Day(1):Date(1970, 6, 30) |> collect + vt = DateTime(1970, 1, 1):Second(1):DateTime(1970, 1, 1, 2, 0, 0) |> collect + ad = ZArray(vd) + at = ZArray(vt) + @test eltype(ad)==DateTime64{Day} + @test eltype(at)==DateTime64{Millisecond} + @test DateTime.(at[:]) == vt[:] + @test Date.(ad[:]) == vd[:] + + p = tempname() + g = zgroup(p) + for pt in [Week, Day, Hour, Minute, Second, Millisecond] - - if pt <: DatePeriod - vd = range(Date(1970,1,1),step = pt(1), length=100) - a = zcreate(Zarr.DateTime64{pt},g,string(pt),100) - a[:] = vd - else - vd = range(DateTime(1970,1,1),step = pt(1), length=100) - a = zcreate(Zarr.DateTime64{pt},g,string(pt),100) - a[:] = vd + + if pt <: DatePeriod + vd = range(Date(1970, 1, 1), step=pt(1), length=100) + a = zcreate(DateTime64{pt}, g, string(pt), 100) + a[:] = vd + else + vd = range(DateTime(1970, 1, 1), step=pt(1), length=100) + a = zcreate(DateTime64{pt}, g, string(pt), 100) + a[:] = vd + end end -end -zarr = pyimport("zarr") -numpy = pyimport("numpy") -g_julia = zopen(p) -g_python = zarr.open(p) + zarr = pyimport("zarr") + numpy = pyimport("numpy") + g_julia = zopen(p) + g_python = zarr.open(p) -for unit in ["Week", "Day", "Hour", "Minute", "Second", + for unit in ["Week", "Day", "Hour", "Minute", "Second", "Millisecond"] - for i in [0, 9, 99] - @test pyeq(Bool, numpy.datetime64(g_julia[unit][i+1] |> DateTime |> string), g_python[unit][i]) + for i in [0, 9, 99] + @test pyeq(Bool, numpy.datetime64(g_julia[unit][i+1] |> DateTime |> string), g_python[unit][i]) + end end + end +@testset "Python zarr v3 irregular (rectilinear) chunks" begin + # Irregular (rectilinear) chunks are an experimental python-zarr feature, + # gated behind a config flag that must be enabled before opening/creating + # any rectilinear store. Irregular chunk grids only exist in zarr v3. + zarr = pyimport("zarr") + numpy = pyimport("numpy") + pybuiltins = pyimport("builtins") + zarr.config.set(pydict(Dict("array.rectilinear_chunks" => true))) + + import Zarr: ZarrCore + using DiskArrays: GridChunks, RegularChunks, IrregularChunks + + # Julia-side storage for irregular chunks pads every chunk to the global + # max chunk size, whereas python-zarr stores each rectilinear chunk at its + # exact data extent. The two on-disk layouts are therefore incompatible: + # - python cannot reshape Julia's padded chunks back to their extents, + # - Julia cannot decode python's exact-size chunks into its padded buffer. + # The metadata round-trips correctly in both directions; only the *data* + # read is broken. Those data assertions are marked `@test_broken` until the + # Julia chunk I/O writes exact-size chunks per grid position. + + pjulia = tempname() + ppython = tempname() + + # ---- Direction A: Julia writes an irregular grid, python reads it ---- + za = zcreate(Int32, 20, 20; zarr_format=3, path=pjulia, + chunks=GridChunks(RegularChunks(4, 0, 20), + IrregularChunks(chunksizes=[3, 4, 5, 6, 2]))) + data = reshape(Int32.(1:400), 20, 20) + za[:, :] = data + + # Metadata round-trips: python sees a rectilinear grid. + b = zarr.open_array(pjulia, mode="r") + grid = pyconvert(Dict, b.metadata.chunk_grid.to_dict()) + @test grid["name"] == "rectilinear" + @test grid["configuration"]["kind"] == "inline" + # The irregular dimension serializes as a bare int (regular) + a list. + cshapes = grid["configuration"]["chunk_shapes"] + @test cshapes[1] isa AbstractVector + @test collect(cshapes[1]) == [3, 4, 5, 6, 2] + @test cshapes[2] == 4 + + # Data read: currently broken (python cannot reshape Julia's padded chunks). + @test_broken pyconvert(Bool, (b[pybuiltins.Ellipsis] == numpy.array(data)).all()) + + # ---- Direction B: python writes an irregular grid, Julia reads it ---- + pd = reshape(Int32.(0:399), 20, 20) + a = zarr.create_array(ppython; zarr_format=3, shape=(20, 20), + chunks=pylist([pylist([3, 4, 5, 6, 2]), pylist([4, 4, 4, 4, 4])]), + dtype="int32") + a.__setitem__(pybuiltins.Ellipsis, numpy.array(pd)) + + # Metadata round-trips: Julia parses the rectilinear grid. + z = zopen(ppython) + @test any(c -> c isa IrregularChunks, ZarrCore.eachchunk(z).chunks) + + # Data read: currently broken (Julia cannot decode exact-size chunks into + # its padded buffer). + @test permutedims(z[:, :], (2, 1)) == pd end diff --git a/test/runtests.jl b/test/runtests.jl index a94f8a31..fbe16406 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,18 +1,63 @@ using Test using Zarr +import Zarr: ZarrCore using JSON using JSON: json using Pkg using Dates +using DiskArrays: GridChunks, DiskArrays, RegularChunks @testset "Zarr" begin - +@testset "public API surface" begin + # `names(M)` returns exported *and* public names, so `Zarr`'s re-export loop + # has to split them with `Base.isexported`; using `names` alone would export + # the whole extension API, and using `Base.ispublic` would export nothing. + # These tests pin that split down so it cannot regress silently. + # + # On Julia 1.10 there is no `public`, so `names` yields only exports and the + # public-only sets are empty; the assertions still hold. The final pair of + # tests covers what that blind spot hides. + # A module always lists its own name, and `Zarr` additionally carries the + # `ZarrCore` binding (public, so `Zarr.ZarrCore` is a documented escape hatch + # rather than something `using Zarr` drags in). Both are structural, not API. + modnames = Set((:Zarr, :ZarrCore)) + exported(m) = setdiff(Set(filter(n -> Base.isexported(m, n), names(m))), modnames) + publiconly(m) = setdiff(Set(filter(n -> !Base.isexported(m, n), names(m))), modnames) + + # `Zarr.ZarrCore` is reachable, but `using Zarr` must not bring it into scope. + @test isdefined(Zarr, :ZarrCore) + @test !Base.isexported(Zarr, :ZarrCore) + + # `Zarr` mirrors `ZarrCore`'s API surface exactly, split intact. + @test exported(Zarr) == exported(ZarrCore) + + # The specific failure mode: a name that is only `public` in `ZarrCore` must + # not become an export of `Zarr`, and vice versa. + @test isempty(intersect(publiconly(ZarrCore), exported(Zarr))) + @test isempty(intersect(exported(ZarrCore), publiconly(Zarr))) + + # Version-independent: the two assertions above compare `names` against + # `names`, so on 1.10 -- where `@public` expands to nothing and both + # public-only sets are empty -- they pass no matter what `Zarr` re-exports. + @test all(isdefined.(Ref(Zarr), [:zname]) ) + @test all(isdefined.(Ref(Zarr), [:DictStore, :HTTPStore, :ZipStore, :CachingStore, :ConsolidatedStore])) + @test all(isdefined.(Ref(Zarr), [:consolidate_metadata, :writezip])) + @test all(isdefined.(Ref(Zarr), [:ChunkKeyEncoding, :SuffixChunkKeyEncoding])) + @test all(isdefined.(Ref(Zarr), [:Filter, :VLenArrayFilter, :VLenUTF8Filter, :Fletcher32Filter, + :FixedScaleOffsetFilter, :ShuffleFilter, :QuantizeFilter, :DeltaFilter])) + @test all(isdefined.(Ref(Zarr), [:Compressor, :NoCompressor, :BloscCompressor, :ZlibCompressor, :ZstdCompressor])) + @test all(isdefined.(Ref(Zarr), [:Codecs, :Codec, :V3Codec, :BloscCodec, :BytesCodec, :CRC32cCodec, :GzipCodec, + :ShardingCodec, :TransposeCodec, :GzipV3Codec, :BloscV3Codec, :ZstdV3Codec, + :CRC32cV3Codec, :VLenUTF8V3Codec])) + +end + @testset "ZArray" begin @testset "fields" begin z = zzeros(Int64, 2, 3) - @test z isa ZArray{Int64,2,Zarr.DictStore,Zarr.MetadataV2{Int64,2,Zarr.BloscCompressor,Nothing}} + @test z isa ZArray{Int64,2,ZarrCore.DictStore,ZarrCore.MetadataV2{Int64,2,ZarrCore.BloscCompressor,Nothing,GridChunks{2,Tuple{RegularChunks,RegularChunks}}}} @test :a ∈ propertynames(z.storage) @test length(z.storage.a) === 3 @test length(z.storage.a["0.0"]) === 64 @@ -21,7 +66,7 @@ using Dates @test z.metadata.node_type === "array" @test z.metadata.shape[] === (2, 3) @test z.metadata.order === 'C' - @test z.metadata.chunks === (2, 3) + @test z.metadata.chunks[] === GridChunks((2, 3), (2, 3)) @test z.metadata.fill_value === nothing @test z.metadata.compressor isa Zarr.BloscCompressor @test z.metadata.compressor.blocksize === 0 @@ -30,15 +75,15 @@ using Dates @test z.metadata.compressor.shuffle === 1 @test z.attrs == Dict{Any, Any}() @test z.writeable === true - @test z.metadata.chunk_key_encoding === Zarr.ChunkKeyEncoding(Zarr.default_sep(Zarr.DV), Zarr.default_prefix(Zarr.DV)) + @test z.metadata.chunk_key_encoding === Zarr.ChunkKeyEncoding(ZarrCore.default_sep(ZarrCore.DV), ZarrCore.default_prefix(ZarrCore.DV)) @test_throws ArgumentError zzeros(Int64,2,3, chunks = (0,1)) @test_throws ArgumentError zzeros(Int64,0,-1) - @test_throws ArgumentError Zarr.Metadata(zeros(2,2), (2,2), order = 'F') + @test_throws ArgumentError ZarrCore.Metadata(zeros(2,2), (2,2), order = 'F') end @testset "methods" begin z = zzeros(Int64, 2, 3) - @test z isa ZArray{Int64,2,Zarr.DictStore,Zarr.MetadataV2{Int64,2,Zarr.BloscCompressor,Nothing}} + @test z isa ZArray{Int64,2,ZarrCore.DictStore,ZarrCore.MetadataV2{Int64,2,ZarrCore.BloscCompressor,Nothing,GridChunks{2,Tuple{RegularChunks,RegularChunks}}}} @test eltype(z) === Int64 @test ndims(z) === 2 @test size(z) === (2, 3) @@ -96,7 +141,7 @@ using Dates z3 = zcreate(Float32, 4, 4, 2; path=joinpath(dir, "disp"), chunks=(4, 4, 2), compressor=Zarr.NoCompressor()) ain = rand(Float32, 4, 4, 2) - m = which(Zarr.write_singlechunk_fastpath!, (typeof(z3), typeof(ain), CartesianIndex{3})) + m = which(ZarrCore.write_singlechunk_fastpath!, (typeof(z3), typeof(ain), CartesianIndex{3})) @test occursin("MetadataV2", string(m.sig)) @test occursin("NoCompressor", string(m.sig)) GC.gc() @@ -176,8 +221,8 @@ end store = DirectoryStore(tempname()) g = zgroup(store,"mygroup") g2 = zgroup(g,"asubgroup",attrs = Dict("a1"=>5)) - @test Zarr.is_zgroup(Zarr.DV, store, "mygroup") - @test Zarr.is_zgroup(Zarr.DV, store, "mygroup/asubgroup") + @test ZarrCore.is_zgroup(ZarrCore.DV, store, "mygroup") + @test ZarrCore.is_zgroup(ZarrCore.DV, store, "mygroup/asubgroup") @test g2.attrs["a1"]==5 @test isdir(joinpath(store.folder,"mygroup")) @test isdir(joinpath(store.folder,"mygroup","asubgroup")) @@ -185,68 +230,69 @@ end @testset "Groups format inheritance v2" begin store = DirectoryStore(tempname()) - zv = Zarr.ZarrFormat(2) + zv = ZarrCore.ZarrFormat(2) g = zgroup(store, "rootgroup", zv) sg = zgroup(g, "subgroup", attrs=Dict("a1" => 5)) - @test Zarr.is_zgroup(zv, store, "rootgroup") - @test Zarr.is_zgroup(zv, store, "rootgroup/subgroup") + @test ZarrCore.is_zgroup(zv, store, "rootgroup") + @test ZarrCore.is_zgroup(zv, store, "rootgroup/subgroup") @test sg.attrs["a1"] == 5 @test ispath(joinpath(store.folder, "rootgroup", ".zgroup")) @test ispath(joinpath(store.folder, "rootgroup", "subgroup", ".zgroup")) a_root = zcreate(Float64, g, "temperature", 2, 3) a_sub = zcreate(Float64, sg, "pressure", 2, 3) - @test Zarr.zarr_format(a_root) == zv - @test Zarr.zarr_format(a_sub) == zv + @test ZarrCore.zarr_format(a_root) == zv + @test ZarrCore.zarr_format(a_sub) == zv end @testset "Groups format inheritance v3" begin store = DirectoryStore(tempname()) - zv = Zarr.ZarrFormat(3) + zv = ZarrCore.ZarrFormat(3) g = zgroup(store, "rootgroup", zv) sg = zgroup(g, "subgroup", attrs=Dict("a1" => 5)) - @test Zarr.is_zgroup(zv, store, "rootgroup") - @test Zarr.is_zgroup(zv, store, "rootgroup/subgroup") + @test ZarrCore.is_zgroup(zv, store, "rootgroup") + @test ZarrCore.is_zgroup(zv, store, "rootgroup/subgroup") @test sg.attrs["a1"] == 5 @test ispath(joinpath(store.folder, "rootgroup", "zarr.json")) @test ispath(joinpath(store.folder, "rootgroup", "subgroup", "zarr.json")) a_root = zcreate(Float64, g, "temperature", 2, 3) a_sub = zcreate(Float64, sg, "pressure", 2, 3) - @test Zarr.zarr_format(a_root) == zv - @test Zarr.zarr_format(a_sub) == zv + @test ZarrCore.zarr_format(a_root) == zv + @test ZarrCore.zarr_format(a_sub) == zv end @testset "Metadata" begin @testset "Data type encoding" begin - @test Zarr.typestr(Bool) === "|b1" - @test Zarr.typestr(Int8) === "|i1" - @test Zarr.typestr(Int64) === " "b")) - @test Zarr.getattrs(V, ds, "bar") == Dict("a" => "b") + ZarrCore.writeattrs(V, ds, "bar", Dict("a" => "b")) + @test ZarrCore.getattrs(V, ds, "bar") == Dict("a" => "b") delete!(ds,"bar/" * first_ci_str) - @test !Zarr.store_isinitialized(ds, "bar", CartesianIndex((1, 1, 1)), enc) + @test !ZarrCore.store_isinitialized(ds, "bar", CartesianIndex((1, 1, 1)), enc) @test !Zarr.isinitialized(ds,"bar/" * first_ci_str) ds["bar/" * first_ci_str] = data - @test !Zarr.store_isinitialized(ds, "bar", CartesianIndex(0, 0, 0), enc) - @test Zarr.store_isinitialized(ds, "bar", CartesianIndex(1, 1, 1), enc) + @test !ZarrCore.store_isinitialized(ds, "bar", CartesianIndex(0, 0, 0), enc) + @test ZarrCore.store_isinitialized(ds, "bar", CartesianIndex(1, 1, 1), enc) #Add tests for empty storage - @test Zarr.isemptysub(ds,"ba") - @test Zarr.isemptysub(ds,"ba/") - @test !Zarr.isemptysub(ds,"bar") - @test !Zarr.isemptysub(ds,"bar/") + @test ZarrCore.isemptysub(ds,"ba") + @test ZarrCore.isemptysub(ds,"ba/") + @test !ZarrCore.isemptysub(ds,"bar") + @test !ZarrCore.isemptysub(ds,"bar/") end """ @@ -96,11 +96,11 @@ Function to test the interface of a read only AbstractStore. Every complete impl `closer` is a function that gets called to close the read only store. """ function test_read_only_store_common(converter, closer=Returns(nothing)) - V = Zarr.DV - enc = Zarr.ChunkKeyEncoding(Zarr.default_sep(V), Zarr.default_prefix(V)) + V = ZarrCore.DV + enc = Zarr.ChunkKeyEncoding(ZarrCore.default_sep(V), ZarrCore.default_prefix(V)) ds = Zarr.DictStore() rs = converter(ds) - @test !Zarr.is_zgroup(V, rs, "") + @test !ZarrCore.is_zgroup(V, rs, "") closer(rs) ds[".zgroup"]=rand(UInt8,50) @@ -108,20 +108,20 @@ function test_read_only_store_common(converter, closer=Returns(nothing)) @test haskey(rs,".zgroup") - @test Zarr.is_zgroup(V, rs, "") - @test !Zarr.is_zarray(V, rs, "") + @test ZarrCore.is_zgroup(V, rs, "") + @test !ZarrCore.is_zarray(V, rs, "") @test isempty(Zarr.subdirs(rs,"")) @test sort(collect(Zarr.subkeys(rs,"")))==[".zgroup"] #Create a subgroup - @test !Zarr.is_zarray(V, rs, "bar") + @test !ZarrCore.is_zarray(V, rs, "bar") closer(rs) ds["bar/.zarray"] = rand(UInt8,50) rs = converter(ds) - @test Zarr.is_zarray(V, rs, "bar") + @test ZarrCore.is_zarray(V, rs, "bar") @test Zarr.subdirs(rs,"") == ["bar"] @test Zarr.subdirs(rs,"bar") == String[] #Test getindex and setindex @@ -137,16 +137,16 @@ function test_read_only_store_common(converter, closer=Returns(nothing)) @test !Zarr.isinitialized(rs,"bar/0.0.1") closer(rs) - Zarr.writeattrs(V, ds, "bar", Dict("a" => "b")) + ZarrCore.writeattrs(V, ds, "bar", Dict("a" => "b")) rs = converter(ds) - @test Zarr.getattrs(V, rs, "bar") == Dict("a" => "b") + @test ZarrCore.getattrs(V, rs, "bar") == Dict("a" => "b") closer(rs) delete!(ds,"bar/0.0.0") rs = converter(ds) - @test !Zarr.store_isinitialized(rs, "bar", CartesianIndex((0, 0, 0)), enc) + @test !ZarrCore.store_isinitialized(rs, "bar", CartesianIndex((0, 0, 0)), enc) @test !Zarr.isinitialized(rs,"bar/0.0.0") closer(rs) @@ -154,17 +154,17 @@ function test_read_only_store_common(converter, closer=Returns(nothing)) rs = converter(ds) #Add tests for empty storage - @test Zarr.isemptysub(rs,"ba") - @test Zarr.isemptysub(rs,"ba/") - @test !Zarr.isemptysub(rs,"bar") - @test !Zarr.isemptysub(rs,"bar/") + @test ZarrCore.isemptysub(rs,"ba") + @test ZarrCore.isemptysub(rs,"ba/") + @test !ZarrCore.isemptysub(rs,"bar") + @test !ZarrCore.isemptysub(rs,"bar/") closer(rs) end @testset "DirectoryStore" begin A = fill(1.0, 30, 20) chunks = (5,10) - metadata = Zarr.Metadata(A, chunks; fill_value=-1.5) + metadata = ZarrCore.Metadata(A, chunks; fill_value=-1.5) p = tempname() mkpath(joinpath(p,"foo")) ds = Zarr.DirectoryStore(joinpath(p,"foo")) @@ -182,7 +182,7 @@ end @testset "DictStore" begin A = fill(1.0, 30, 20) chunks = (5,10) - metadata = Zarr.Metadata(A, chunks; fill_value=-1.5) + metadata = ZarrCore.Metadata(A, chunks; fill_value=-1.5) ds = Zarr.DictStore() test_store_common(ds) @test haskey(ds.a,".zgroup") @@ -195,7 +195,7 @@ end @info "Testing Minio S3 storage" A = fill(1.0, 30, 20) chunks = (5,10) - metadata = Zarr.Metadata(A, chunks; fill_value=-1.5) + metadata = ZarrCore.Metadata(A, chunks; fill_value=-1.5) using Minio if !isnothing(Minio.minio()) s = Minio.Server(joinpath("./",tempname()), address="localhost:9001") @@ -230,12 +230,12 @@ end end @testset "AWS S3 Storage" begin - V = Zarr.DV + V = ZarrCore.DV @info "Testing AWS S3 storage" S3, p = AWSS3.AWS.with_aws_config(AWSS3.AWS.AWSConfig(creds=nothing, region="us-west-2")) do Zarr.storefromstring("s3://mur-sst/zarr-v1") end - @test Zarr.is_zgroup(V, S3, p) + @test ZarrCore.is_zgroup(V, S3, p) @test storagesize(S3, p) == 10551 S3group = zopen(S3,path=p) S3Array = S3group["time"] @@ -280,7 +280,7 @@ end g = zgroup(s, attrs = Dict("groupatt"=>5)) a = zcreate(Int,g,"a1",10,20,chunks=(5,5),attrs=Dict("arratt"=>2.5)) a .= reshape(1:200,10,20) - using Zarr.HTTP: HTTP + using Zarr.ZarrCore.HTTP: HTTP server = HTTP.serve!(g, "127.0.0.1", 0) port = server.bound_port g2 = zopen("http://127.0.0.1:$port") @@ -340,10 +340,10 @@ end @testset "missing_chunk_return_code! on HTTPStore" begin hs = Zarr.HTTPStore("http://example.com") @test 403 ∉ hs.allowed_codes - Zarr.missing_chunk_return_code!(hs, 403) + ZarrCore.missing_chunk_return_code!(hs, 403) @test 403 ∈ hs.allowed_codes # Vector form - Zarr.missing_chunk_return_code!(hs, [410, 451]) + ZarrCore.missing_chunk_return_code!(hs, [410, 451]) @test 410 ∈ hs.allowed_codes @test 451 ∈ hs.allowed_codes end @@ -352,18 +352,18 @@ end hs = Zarr.HTTPStore("http://example.com") # Build a ConsolidatedStore wrapping the HTTPStore directly cs = Zarr.ConsolidatedStore(hs, "", Dict{String,Any}()) - Zarr.missing_chunk_return_code!(cs, 403) + ZarrCore.missing_chunk_return_code!(cs, 403) @test 403 ∈ hs.allowed_codes end @testset "store_read_strategy and has_configurable_missing_chunks" begin hs = Zarr.HTTPStore("http://example.com") @test Zarr.store_read_strategy(hs) isa Zarr.ConcurrentRead - @test Zarr.has_configurable_missing_chunks(hs) == true + @test ZarrCore.has_configurable_missing_chunks(hs) == true # ConsolidatedStore delegates both to parent cs = Zarr.ConsolidatedStore(hs, "", Dict{String,Any}()) @test Zarr.store_read_strategy(cs) isa Zarr.ConcurrentRead - @test Zarr.has_configurable_missing_chunks(cs) == true + @test ZarrCore.has_configurable_missing_chunks(cs) == true end @testset "storefromstring HTTP/HTTPS regex" begin @@ -397,7 +397,7 @@ end a3 = zcreate(Int, g3, "b", 4, 4, chunks=(2,2)) a3 .= reshape(1:16, 4, 4) # zarr_req_handler with default notfound=404 - server4 = HTTP.serve!(Zarr.zarr_req_handler(s3, g3.path), "127.0.0.1", 0) + server4 = HTTP.serve!(ZarrCore.zarr_req_handler(s3, g3.path), "127.0.0.1", 0) port4 = server4.bound_port g4 = zopen("http://127.0.0.1:$port4") @test g4.attrs == Dict("x" => 1) @@ -422,11 +422,11 @@ end s6 = Zarr.DictStore() g6 = zgroup(s6, attrs = Dict("groupatt"=>5)) a6 = zcreate(Int, g6, "a", 10, 20, chunks=(5,5), attrs=Dict("arratt"=>2.5), fill_value=-1) - server6 = HTTP.serve!(Zarr.zarr_req_handler(s6, g6.path, 403), "127.0.0.1", 0) + server6 = HTTP.serve!(ZarrCore.zarr_req_handler(s6, g6.path, 403), "127.0.0.1", 0) port6 = server6.bound_port httpstore6 = Zarr.HTTPStore("http://127.0.0.1:$port6") @test_throws "Received error code 403" Zarr.ConsolidatedStore(httpstore6, "") - Zarr.missing_chunk_return_code!(httpstore6, 403) + ZarrCore.missing_chunk_return_code!(httpstore6, 403) g7 = zopen(Zarr.ConsolidatedStore(httpstore6, "")) @test all(==(-1), g7["a"][:,:]) close(server6) @@ -467,18 +467,18 @@ end "attributes" => Dict{String,Any}("foo" => "bar") ) )) - @test Zarr.getattrs(Zarr.ZarrFormat(3), store, "") == Dict("foo" => "bar") + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), store, "") == Dict("foo" => "bar") # zarr.json present but no "attributes" key: fallback return Dict{String,Any}() node_meta = Dict{String,Any}("node_type" => "group", "zarr_format" => 3) store_noattrs = Zarr.ConsolidatedStore(Zarr.DictStore(), "", Dict{String,Any}( "zarr.json" => node_meta )) - @test Zarr.getattrs(Zarr.ZarrFormat(3), store_noattrs, "") == Dict{String,Any}() + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), store_noattrs, "") == Dict{String,Any}() # missing zarr.json key entirely: empty dict store_empty = Zarr.ConsolidatedStore(Zarr.DictStore(), "", Dict{String,Any}()) - @test Zarr.getattrs(Zarr.ZarrFormat(3), store_empty, "") == Dict{String,Any}() + @test ZarrCore.getattrs(ZarrCore.ZarrFormat(3), store_empty, "") == Dict{String,Any}() end @testset "Caching Storage" begin # Create source data @@ -488,6 +488,7 @@ end a .= reshape(1:200, 10, 20) # Start HTTP server + using Zarr.ZarrCore.HTTP: HTTP server = HTTP.serve!(g, "127.0.0.1", 0) port = server.bound_port diff --git a/test/v3_codecs.jl b/test/v3_codecs.jl index 4e79e561..58c1de3d 100644 --- a/test/v3_codecs.jl +++ b/test/v3_codecs.jl @@ -1,5 +1,6 @@ using Test using Zarr +import Zarr: ZarrCore using JSON @testset "V3 Codecs" begin @@ -67,40 +68,42 @@ end @testset "get_order" begin bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() + ch = DiskArrays.GridChunks((3, 3, 3), (3, 3, 3)) + cke = Zarr.ChunkKeyEncoding('/', true) # No array->array codecs → 'C' - p = Zarr.V3Pipeline((), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test Zarr.get_order(md) == 'C' + p = ZarrCore.V3Pipeline((), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) + @test ZarrCore.get_order(md) == 'C' # Single TransposeCodec with identity permutation → 'C' tc_c = Zarr.Codecs.V3Codecs.TransposeCodec((1,2,3)) - p = Zarr.V3Pipeline((tc_c,), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test Zarr.get_order(md) == 'C' + p = ZarrCore.V3Pipeline((tc_c,), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) + @test ZarrCore.get_order(md) == 'C' # Single TransposeCodec with reverse permutation → 'F' tc_f = Zarr.Codecs.V3Codecs.TransposeCodec((3,2,1)) - p = Zarr.V3Pipeline((tc_f,), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test Zarr.get_order(md) == 'F' + p = ZarrCore.V3Pipeline((tc_f,), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) + @test ZarrCore.get_order(md) == 'F' # Single TransposeCodec with arbitrary (non-C, non-F) permutation → ArgumentError tc_other = Zarr.Codecs.V3Codecs.TransposeCodec((2,1,3)) - p = Zarr.V3Pipeline((tc_other,), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test_throws ArgumentError Zarr.get_order(md) + p = ZarrCore.V3Pipeline((tc_other,), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) + @test_throws ArgumentError ZarrCore.get_order(md) # Multiple array->array codecs → ArgumentError - p = Zarr.V3Pipeline((tc_f, tc_f), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test_throws ArgumentError Zarr.get_order(md) + p = ZarrCore.V3Pipeline((tc_f, tc_f), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) + @test_throws ArgumentError ZarrCore.get_order(md) # Unrecognized array->array codec type → ArgumentError struct _FakeCodec <: Zarr.Codecs.V3Codecs.V3Codec{:array,:array} end - p = Zarr.V3Pipeline((_FakeCodec(),), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(p)}(3, "array", (3,3,3), (3,3,3), "int32", p, Int32(0), Zarr.ChunkKeyEncoding('/',true)) - @test_throws ArgumentError Zarr.get_order(md) + p = ZarrCore.V3Pipeline((_FakeCodec(),), bytes_codec, ()) + md = ZarrCore.MetadataV3{Int32,3,typeof(p),typeof(cke),typeof(ch)}(3, "array", (3,3,3), ch, "int32", p, Int32(0), cke) + @test_throws ArgumentError ZarrCore.get_order(md) end @testset "VLenUTF8V3Codec" begin @@ -151,8 +154,8 @@ end {"name":"bytes","configuration":{"endian":"little"}}, {"name":"blosc","configuration":{"cname":"lz4","clevel":5,"shuffle":"$shuffle_str","blocksize":0,"typesize":4}} ]}""" - md = Zarr.Metadata(json_str, false) - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata(json_str, false) + pipeline = ZarrCore.get_pipeline(md) blosc = pipeline.bytes_bytes[1] @test blosc isa Zarr.Codecs.V3Codecs.BloscV3Codec @test blosc.shuffle == expected_int @@ -167,8 +170,8 @@ end {"name":"bytes","configuration":{"endian":"little"}}, {"name":"blosc","configuration":{"cname":"lz4","clevel":5,"shuffle":$shuffle_int,"blocksize":0,"typesize":4}} ]}""" - md = Zarr.Metadata(json_str, false) - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata(json_str, false) + pipeline = ZarrCore.get_pipeline(md) blosc = pipeline.bytes_bytes[1] @test blosc.shuffle == expected_int end @@ -181,7 +184,7 @@ end {"name":"bytes","configuration":{"endian":"little"}}, {"name":"blosc","configuration":{"cname":"lz4","clevel":5,"shuffle":"invalid","blocksize":0,"typesize":4}} ]}""" - @test_throws ArgumentError Zarr.Metadata(bad_json, false) + @test_throws ArgumentError ZarrCore.Metadata(bad_json, false) # --- serialization: integer -> shuffle string --- for (shuffle_int, expected_str) in ((0, "noshuffle"), (1, "shuffle"), (2, "bitshuffle")) @@ -192,7 +195,7 @@ end {"name":"bytes","configuration":{"endian":"little"}}, {"name":"blosc","configuration":{"cname":"lz4","clevel":5,"shuffle":$shuffle_int,"blocksize":0,"typesize":4}} ]}""" - md = Zarr.Metadata(json_str, false) + md = ZarrCore.Metadata(json_str, false) lowered = JSON.lower(md) blosc_config = lowered["codecs"][2]["configuration"] @test blosc_config["shuffle"] == expected_str @@ -201,10 +204,12 @@ end # --- serialization: unknown shuffle integer raises ArgumentError via lower3 --- let bad_blosc = Zarr.Codecs.V3Codecs.BloscV3Codec("lz4", 5, 99, 0, 4), bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec(), - bad_pipeline = Zarr.V3Pipeline((), bytes_codec, (bad_blosc,)) - bad_md = Zarr.MetadataV3{Int32,1,typeof(bad_pipeline)}( - 3, "array", (4,), (4,), "int32", bad_pipeline, Int32(0), - Zarr.ChunkKeyEncoding('/', true) + bad_pipeline = ZarrCore.V3Pipeline((), bytes_codec, (bad_blosc,)) + ch = ZarrCore.DiskArrays.GridChunks((4,), (4,)) + cke = Zarr.ChunkKeyEncoding('/', true) + bad_md = ZarrCore.MetadataV3{Int32,1,typeof(bad_pipeline),typeof(cke),typeof(ch)}( + 3, "array", (4,), ch, "int32", bad_pipeline, Int32(0), + cke ) @test_throws ArgumentError JSON.lower(bad_md) end @@ -268,8 +273,8 @@ end {"name":"numcodecs.blosc","configuration":{"cname":"lz4","clevel":5, "shuffle":"noshuffle","blocksize":0,"typesize":2}} ]}""" - md = @test_nowarn Zarr.Metadata(json_str, false) - pipeline = Zarr.get_pipeline(md) + md = @test_nowarn ZarrCore.Metadata(json_str, false) + pipeline = ZarrCore.get_pipeline(md) @test pipeline.bytes_bytes[1] isa Zarr.Codecs.V3Codecs.BloscV3Codec end @@ -283,7 +288,7 @@ end {"name":"numcodecs.blosc","configuration":{"cname":"lz4","clevel":5, "shuffle":"noshuffle","blocksize":0,"typesize":2}} ]}""" - md = Zarr.Metadata(json_str, false) + md = ZarrCore.Metadata(json_str, false) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) data = Int16[1, 2, 3, 4] @@ -303,7 +308,7 @@ end "shuffle":"noshuffle","blocksize":0,"typesize":2}} ]}""" store["zarr.json"] = Vector{UInt8}(json_str) - @test_nowarn @test Zarr.is_zarray(Zarr.ZarrFormat(Val(3)), store, "") == true + @test_nowarn @test ZarrCore.is_zarray(ZarrCore.ZarrFormat(Val(3)), store, "") == true end end @@ -322,10 +327,12 @@ end # but data reads and writes must still work correctly. tc = Zarr.Codecs.V3Codecs.TransposeCodec((2, 1, 3)) bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() - pipeline = Zarr.V3Pipeline((tc,), bytes_codec, ()) - md = Zarr.MetadataV3{Int32,3,typeof(pipeline)}( - 3, "array", (2,3,4), (2,3,4), "int32", pipeline, Int32(0), - Zarr.ChunkKeyEncoding('/', true) + pipeline = ZarrCore.V3Pipeline((tc,), bytes_codec, ()) + ch = DiskArrays.GridChunks((2, 3, 4), (2, 3, 4)) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{Int32,3,typeof(pipeline),typeof(cke),typeof(ch)}( + 3, "array", (2, 3, 4), ch, "int32", pipeline, Int32(0), + cke ) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) @@ -334,12 +341,12 @@ end @test z[:,:,:] == data # get_order throws for non-canonical permutation - @test_throws ArgumentError Zarr.get_order(z.metadata) + @test_throws ArgumentError ZarrCore.get_order(z.metadata) end @testset "V3 group attributes round-trip" begin store = Zarr.DictStore() - g = zgroup(store, "", Zarr.ZarrFormat(3)) + g = zgroup(store, "", ZarrCore.ZarrFormat(3)) zgroup(g, "sub"; attrs=Dict("key" => "val", "num" => 42)) # Re-open the store and verify attributes are preserved @@ -351,9 +358,10 @@ end @testset "CRC32c end-to-end ZArray" begin crc32c_codec = Zarr.Codecs.V3Codecs.CRC32cV3Codec() bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() - pipeline = Zarr.V3Pipeline((), bytes_codec, (crc32c_codec,)) - md = Zarr.MetadataV3{Int32,1,typeof(pipeline),Zarr.ChunkKeyEncoding}( - 3, "array", (4,), (4,), "int32", pipeline, Int32(0), + pipeline = ZarrCore.V3Pipeline((), bytes_codec, (crc32c_codec,)) + ch = DiskArrays.GridChunks((4,), (4,)) + md = ZarrCore.MetadataV3{Int32,1,typeof(pipeline),Zarr.ChunkKeyEncoding,typeof(ch)}( + 3, "array", (4,), ch, "int32", pipeline, Int32(0), Zarr.ChunkKeyEncoding('/', true) ) store = Zarr.DictStore() @@ -373,72 +381,72 @@ end @testset "V2Pipeline encode/decode round-trip" begin comp = Zarr.BloscCompressor() - pipeline = Zarr.V2Pipeline(comp, nothing) + pipeline = ZarrCore.V2Pipeline(comp, nothing) data = zeros(Int64, 4, 4) data[1, 1] = 42 - encoded = Zarr.pipeline_encode(pipeline, data, nothing) + encoded = ZarrCore.pipeline_encode(pipeline, data, nothing) @test encoded isa Vector{UInt8} @test !isempty(encoded) output = zeros(Int64, 4, 4) - Zarr.pipeline_decode!(pipeline, output, encoded) + ZarrCore.pipeline_decode!(pipeline, output, encoded) @test output == data end @testset "V2Pipeline with fill_value returns nothing" begin comp = Zarr.BloscCompressor() - pipeline = Zarr.V2Pipeline(comp, nothing) + pipeline = ZarrCore.V2Pipeline(comp, nothing) data = fill(Int64(-1), 4, 4) - encoded = Zarr.pipeline_encode(pipeline, data, Int64(-1)) + encoded = ZarrCore.pipeline_encode(pipeline, data, Int64(-1)) @test encoded === nothing end @testset "V3Pipeline encode/decode round-trip" begin bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() gzip_codec = Zarr.Codecs.V3Codecs.GzipV3Codec(6) - pipeline = Zarr.V3Pipeline((), bytes_codec, (gzip_codec,)) + pipeline = ZarrCore.V3Pipeline((), bytes_codec, (gzip_codec,)) data = Int32[1, 2, 3, 4] - encoded = Zarr.pipeline_encode(pipeline, data, nothing) + encoded = ZarrCore.pipeline_encode(pipeline, data, nothing) @test encoded isa Vector{UInt8} output = zeros(Int32, 4) - Zarr.pipeline_decode!(pipeline, output, encoded) + ZarrCore.pipeline_decode!(pipeline, output, encoded) @test output == data end @testset "V3Pipeline with no compression" begin bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() - pipeline = Zarr.V3Pipeline((), bytes_codec, ()) + pipeline = ZarrCore.V3Pipeline((), bytes_codec, ()) data = Float64[1.5, 2.5, 3.5] - encoded = Zarr.pipeline_encode(pipeline, data, nothing) + encoded = ZarrCore.pipeline_encode(pipeline, data, nothing) @test encoded isa Vector{UInt8} output = zeros(Float64, 3) - Zarr.pipeline_decode!(pipeline, output, encoded) + ZarrCore.pipeline_decode!(pipeline, output, encoded) @test output == data end @testset "V3Pipeline fill_value returns nothing" begin bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() - pipeline = Zarr.V3Pipeline((), bytes_codec, ()) + pipeline = ZarrCore.V3Pipeline((), bytes_codec, ()) data = fill(Int32(0), 4) - encoded = Zarr.pipeline_encode(pipeline, data, Int32(0)) + encoded = ZarrCore.pipeline_encode(pipeline, data, Int32(0)) @test encoded === nothing end @testset "V3 Metadata Parsing" begin json_str = """{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}},"chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}},"fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":[0]}},{"name":"bytes","configuration":{"endian":"little"}},{"name":"gzip","configuration":{"level":6}}]}""" - md = Zarr.Metadata(json_str, false) - @test md isa Zarr.MetadataV3 + md = ZarrCore.Metadata(json_str, false) + @test md isa ZarrCore.MetadataV3 @test md.shape[] == (4,) - @test md.chunks == (4,) + @test md.chunks[] == GridChunks((4,), (4,)) @test md.fill_value == Int32(0) - pipeline = Zarr.get_pipeline(md) - @test pipeline isa Zarr.V3Pipeline + pipeline = ZarrCore.get_pipeline(md) + @test pipeline isa ZarrCore.V3Pipeline @test length(pipeline.array_array) == 1 @test pipeline.array_bytes isa Zarr.Codecs.V3Codecs.BytesCodec @test length(pipeline.bytes_bytes) == 1 @@ -446,8 +454,8 @@ end @testset "V3 Metadata JSON round-trip" begin json_str = """{"zarr_format":3,"node_type":"array","shape":[4,4],"data_type":"float64","chunk_grid":{"name":"regular","configuration":{"chunk_shape":[2,2]}},"chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}},"fill_value":0.0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}},{"name":"blosc","configuration":{"cname":"lz4","clevel":5,"shuffle":"noshuffle","blocksize":0,"typesize":4}}]}""" - md = Zarr.Metadata(json_str, false) - @test md isa Zarr.MetadataV3 + md = ZarrCore.Metadata(json_str, false) + @test md isa ZarrCore.MetadataV3 # Serialize back to JSON lowered = JSON.lower(md) @@ -458,17 +466,17 @@ end @testset "V3 Group Metadata Parsing" begin json_str = """{"zarr_format":3,"node_type":"group"}""" - md = Zarr.Metadata(json_str, false) - @test md isa Zarr.MetadataV3 + md = ZarrCore.Metadata(json_str, false) + @test md isa ZarrCore.MetadataV3 @test md.node_type == "group" end @testset "typestr3 raw types" begin - @test Zarr.typestr3("r8") == NTuple{1,UInt8} - @test Zarr.typestr3("r16") == NTuple{2,UInt8} - @test Zarr.typestr3("r64") == NTuple{8,UInt8} - @test_throws ArgumentError Zarr.typestr3("rxyz") # non-numeric bits - @test_throws ArgumentError Zarr.typestr3("r7") # not a multiple of 8 + @test ZarrCore.typestr3("r8") == NTuple{1,UInt8} + @test ZarrCore.typestr3("r16") == NTuple{2,UInt8} + @test ZarrCore.typestr3("r64") == NTuple{8,UInt8} + @test_throws ArgumentError ZarrCore.typestr3("rxyz") # non-numeric bits + @test_throws ArgumentError ZarrCore.typestr3("r7") # not a multiple of 8 end @testset "V3 Metadata parsing error paths" begin @@ -478,57 +486,57 @@ end "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""" # Unknown node_type - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"unknown"}""", false) + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"unknown"}""", false) # Extra key in group metadata - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"group","bad_key":1}""", false) + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"group","bad_key":1}""", false) # Missing required key (shape) - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Unknown chunk_grid name - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"unknown","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Shape/chunk rank mismatch - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[2,2]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Unknown chunk_key_encoding name - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"unknown"}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Unknown codec - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}},{"name":"unknown_codec"}]}""", false) # Deprecated string transpose order "C" - @test_logs (:warn,) Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_logs (:warn,) ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":"C"}}, {"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Deprecated string transpose order "F" - @test_logs (:warn,) Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_logs (:warn,) ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":"F"}}, {"name":"bytes","configuration":{"endian":"little"}}]}""", false) # Unknown string transpose order - @test_throws ArgumentError Zarr.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", + @test_throws ArgumentError ZarrCore.Metadata("""{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":"X"}}, @@ -542,8 +550,8 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}, {"name":"zstd","configuration":{"level":3}}]}""" - md = Zarr.Metadata(json_zstd, false) - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata(json_zstd, false) + pipeline = ZarrCore.get_pipeline(md) @test pipeline.bytes_bytes[1] isa Zarr.Codecs.V3Codecs.ZstdV3Codec @test pipeline.bytes_bytes[1].level == 3 @@ -553,8 +561,8 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}, {"name":"crc32c"}]}""" - md = Zarr.Metadata(json_crc, false) - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata(json_crc, false) + pipeline = ZarrCore.get_pipeline(md) @test pipeline.bytes_bytes[1] isa Zarr.Codecs.V3Codecs.CRC32cV3Codec # F-order from numeric reverse permutation sets order='F' @@ -563,15 +571,15 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":[1,0]}}, {"name":"bytes","configuration":{"endian":"little"}}]}""" - md = Zarr.Metadata(json_f, false) - @test Zarr.get_order(md) == 'F' + md = ZarrCore.Metadata(json_f, false) + @test ZarrCore.get_order(md) == 'F' # v2 chunk_key_encoding (prefix=false, separator='.') json_v2enc = """{"zarr_format":3,"node_type":"array","shape":[4],"data_type":"int32", "chunk_grid":{"name":"regular","configuration":{"chunk_shape":[4]}}, "chunk_key_encoding":{"name":"v2","configuration":{"separator":"."}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""" - md = Zarr.Metadata(json_v2enc, false) + md = ZarrCore.Metadata(json_v2enc, false) @test md.chunk_key_encoding.prefix == false @test md.chunk_key_encoding.sep == '.' end @@ -585,7 +593,7 @@ end "base_encoding":{"name":"default"} }}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""" - md = Zarr.Metadata(json_str, false) + md = ZarrCore.Metadata(json_str, false) @test md.chunk_key_encoding isa Zarr.SuffixChunkKeyEncoding @test md.chunk_key_encoding.suffix == ".tiff" @test md.chunk_key_encoding.base_encoding isa Zarr.ChunkKeyEncoding @@ -606,7 +614,7 @@ end "base_encoding":{"name":"v2"} }}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}]}""" - md2 = Zarr.Metadata(json_v2base, false) + md2 = ZarrCore.Metadata(json_v2base, false) @test md2.chunk_key_encoding.suffix == ".shard.zip" @test md2.chunk_key_encoding.base_encoding.prefix == false # "v2" has no prefix @test Zarr.citostring(md2.chunk_key_encoding, CartesianIndex(1, 1)) == "0.0.shard.zip" @@ -622,10 +630,11 @@ end store = Zarr.DictStore() cke = Zarr.SuffixChunkKeyEncoding(".tiff", Zarr.ChunkKeyEncoding('/', true)) bytes_codec = Zarr.Codecs.V3Codecs.BytesCodec() - pipeline = Zarr.V3Pipeline((), bytes_codec, ()) + pipeline = ZarrCore.V3Pipeline((), bytes_codec, ()) P = typeof(pipeline) E = typeof(cke) - md = Zarr.MetadataV3{Int32,2,P,E}(3, "array", (4,4), (2,2), "int32", pipeline, Int32(0), cke) + ch = DiskArrays.GridChunks((4, 4), (2, 2)) + md = ZarrCore.MetadataV3{Int32,2,P,E,typeof(ch)}(3, "array", (4,4), ch, "int32", pipeline, Int32(0), cke) z = Zarr.ZArray(md, store, "", Dict(), true) z[:,:] = reshape(Int32.(1:16), 4, 4) @test z[:,:] == reshape(Int32.(1:16), 4, 4) @@ -639,7 +648,7 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}, {"name":"zstd","configuration":{"level":5}}]}""" - md = Zarr.Metadata(json_zstd, false) + md = ZarrCore.Metadata(json_zstd, false) lowered = JSON.lower(md) @test lowered["codecs"][2]["name"] == "zstd" @test lowered["codecs"][2]["configuration"]["level"] == 5 @@ -650,7 +659,7 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"bytes","configuration":{"endian":"little"}}, {"name":"crc32c"}]}""" - md = Zarr.Metadata(json_crc, false) + md = ZarrCore.Metadata(json_crc, false) lowered = JSON.lower(md) @test lowered["codecs"][2]["name"] == "crc32c" @@ -660,7 +669,7 @@ end "chunk_key_encoding":{"name":"default","configuration":{"separator":"/"}}, "fill_value":0,"codecs":[{"name":"transpose","configuration":{"order":[1,0]}}, {"name":"bytes","configuration":{"endian":"little"}}]}""" - md = Zarr.Metadata(json_trans, false) + md = ZarrCore.Metadata(json_trans, false) lowered = JSON.lower(md) @test lowered["codecs"][1]["name"] == "transpose" @test lowered["codecs"][1]["configuration"]["order"] == [1, 0] @@ -669,24 +678,24 @@ end @testset "MetadataV3 convenience constructor" begin # order='F' creates a TransposeCodec data = zeros(Int32, 4, 4) - md = Zarr.Metadata3(data, (4,4); order='F') - @test Zarr.get_order(md) == 'F' - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata3(data, (4,4); order='F') + @test ZarrCore.get_order(md) == 'F' + pipeline = ZarrCore.get_pipeline(md) @test length(pipeline.array_array) == 1 @test pipeline.array_array[1] isa Zarr.Codecs.V3Codecs.TransposeCodec # ZstdCompressor translates to ZstdV3Codec - md_zstd = Zarr.Metadata3(data, (4,4); compressor=Zarr.ZstdCompressor()) - pipeline_zstd = Zarr.get_pipeline(md_zstd) + md_zstd = ZarrCore.Metadata3(data, (4,4); compressor=Zarr.ZstdCompressor()) + pipeline_zstd = ZarrCore.get_pipeline(md_zstd) @test pipeline_zstd.bytes_bytes[1] isa Zarr.Codecs.V3Codecs.ZstdV3Codec # fill_value=nothing defaults to zero(T) - md_nofv = Zarr.Metadata3(data, (4,4)) + md_nofv = ZarrCore.Metadata3(data, (4,4)) @test md_nofv.fill_value == Int32(0) # Unsupported compressor throws ArgumentError struct _BadCompressor <: Zarr.Compressor end - @test_throws ArgumentError Zarr.Metadata3(data, (4,4); compressor=_BadCompressor()) + @test_throws ArgumentError ZarrCore.Metadata3(data, (4,4); compressor=_BadCompressor()) end @testset "Metadata3 fixed_length_utf32" begin @@ -705,9 +714,9 @@ end "codecs" => [Dict{String, Any}("name" => "bytes", "configuration" => Dict{String, Any}("endian" => "little"))] ) - md = Zarr.Metadata3(d, false) + md = ZarrCore.Metadata3(d, false) # 40 bytes / 4 bytes per code unit = 10 code units - @test eltype(md) == Zarr.MaxLengthStrings.MaxLengthString{10, UInt32} + @test eltype(md) == ZarrCore.MaxLengthString{10, UInt32} # Test lowering back to JSON preserves the dict structure lowered = JSON.lower(md) @@ -744,7 +753,7 @@ end @testset "V3 Group Creation" begin store = Zarr.DictStore() - g = zgroup(store, "", Zarr.ZarrFormat(3)) + g = zgroup(store, "", ZarrCore.ZarrFormat(3)) @test haskey(store, "zarr.json") md = JSON.parse(String(copy(store["zarr.json"]))) @test md["zarr_format"] == 3 @@ -885,7 +894,7 @@ end @testset "V3 group with arrays" begin store = Zarr.DictStore() - g = zgroup(store, "", Zarr.ZarrFormat(3)) + g = zgroup(store, "", ZarrCore.ZarrFormat(3)) a = zcreate(Float64, g, "myarray", 10; zarr_format=3, chunks=(5,), fill_value=0.0) a[:] = Float64.(1:10) @@ -1123,21 +1132,23 @@ end @testset "ShardingCodec ragged inner chunks" begin # Outer chunk (shard) size does not evenly divide by inner chunk size. # shard shape (3,), inner chunk shape (2,): 2 inner chunks — full (1:2) + partial (3:3) - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{Int16,1,typeof(pipeline)}( - 3, "array", (3,), (3,), "int16", pipeline, Int16(0), - Zarr.ChunkKeyEncoding('/', true) + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + ch = DiskArrays.GridChunks((3,), (3,)) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline),typeof(cke),typeof(ch)}( + 3, "array", (3,), ch, "int16", pipeline, Int16(0), + cke ) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) @@ -1147,21 +1158,23 @@ end @test z[:] == data # 2D: shard (3,3), inner (2,2) — partial chunks on both axes - inner_pipeline2 = Zarr.V3Pipeline( + inner_pipeline2 = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), () ) - index_pipeline2 = Zarr.V3Pipeline( + index_pipeline2 = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding2 = Zarr.Codecs.V3Codecs.ShardingCodec((2,2), inner_pipeline2, index_pipeline2, :end) - pipeline2 = Zarr.V3Pipeline((), sharding2, ()) - md2 = Zarr.MetadataV3{Int32,2,typeof(pipeline2)}( - 3, "array", (3,3), (3,3), "int32", pipeline2, Int32(0), - Zarr.ChunkKeyEncoding('/', true) + pipeline2 = ZarrCore.V3Pipeline((), sharding2, ()) + ch2 = DiskArrays.GridChunks((3, 3), (3, 3)) + cke2 = Zarr.ChunkKeyEncoding('/', true) + md2 = ZarrCore.MetadataV3{Int32,2,typeof(pipeline2),typeof(cke2),typeof(ch2)}( + 3, "array", (3,3), ch2, "int32", pipeline2, Int32(0), + cke2 ) store2 = Zarr.DictStore() z2 = Zarr.ZArray(md2, store2, "", Dict(), true) @@ -1174,21 +1187,23 @@ end @testset "ShardingCodec ZArray write and read" begin # Build a pipeline where ShardingCodec is the array->bytes codec. # Shard shape (outer chunk): (4,). Inner chunk shape: (2,). - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.GzipV3Codec(6),) ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{Int16,1,typeof(pipeline)}( - 3, "array", (4,), (4,), "int16", pipeline, Int16(0), - Zarr.ChunkKeyEncoding('/', true) + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + ch = DiskArrays.GridChunks((4,), (4,)) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline),typeof(cke),typeof(ch)}( + 3, "array", (4,), ch, "int16", pipeline, Int16(0), + cke ) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) @@ -1203,21 +1218,23 @@ end # :start index location. zencode! stored absolute offsets (shifted by # index_size), and zdecode! added chunk_data_offset=index_size again, # reading at 2×index_size + relative. - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), () ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :start) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{Int16,1,typeof(pipeline)}( - 3, "array", (4,), (4,), "int16", pipeline, Int16(0), - Zarr.ChunkKeyEncoding('/', true) + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + ch = DiskArrays.GridChunks((4,), (4,)) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline),typeof(cke),typeof(ch)}( + 3, "array", (4,), ch, "int16", pipeline, Int16(0), + cke ) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) @@ -1228,12 +1245,12 @@ end end @testset "ShardingCodec zdecode! fill_value for empty shard" begin - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), () ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) @@ -1271,8 +1288,8 @@ end "index_location":"end" }} ]}""" - md = Zarr.Metadata(json_str, false) - pipeline = Zarr.get_pipeline(md) + md = ZarrCore.Metadata(json_str, false) + pipeline = ZarrCore.get_pipeline(md) sharding = pipeline.array_bytes blosc = sharding.codecs.bytes_bytes[1] @test blosc isa Zarr.Codecs.V3Codecs.BloscV3Codec @@ -1282,21 +1299,23 @@ end @testset "ShardingCodec multi-shard array" begin # Array size (8,) with shard (outer chunk) size (4,) and inner chunk size (2,). # Two shards, each containing 2 inner chunks. - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), () ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{Int16,1,typeof(pipeline)}( - 3, "array", (8,), (4,), "int16", pipeline, Int16(0), - Zarr.ChunkKeyEncoding('/', true) + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + ch = DiskArrays.GridChunks((8,), (4,)) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline),typeof(cke),typeof(ch)}( + 3, "array", (8,), ch, "int16", pipeline, Int16(0), + cke ) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) @@ -1311,21 +1330,23 @@ end @testset "ShardingCodec non-zero fill_value" begin # Shard shape (4,), inner chunk (2,); only write to first inner chunk. # The second inner chunk should read back as fill_value (Int16(99)), not zero. - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), () ) - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec((2,), inner_pipeline, index_pipeline, :end) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{Int16,1,typeof(pipeline)}( - 3, "array", (4,), (4,), "int16", pipeline, Int16(99), - Zarr.ChunkKeyEncoding('/', true) + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + ch = DiskArrays.GridChunks((4,), (4,)) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{Int16,1,typeof(pipeline),typeof(cke),typeof(ch)}( + 3, "array", (4,), ch, "int16", pipeline, Int16(99), + cke ) store = Zarr.DictStore() z = Zarr.ZArray(md, store, "", Dict(), true) @@ -1353,7 +1374,7 @@ end "index_location":"end" }} ]}""" - @test_throws ArgumentError Zarr.Metadata(json_str, false) + @test_throws ArgumentError ZarrCore.Metadata(json_str, false) end end # V3 Codecs diff --git a/test/v3_julia.jl b/test/v3_julia.jl index 8600d041..a3a37299 100644 --- a/test/v3_julia.jl +++ b/test/v3_julia.jl @@ -2,6 +2,8 @@ # Mirrors the examples from v3_python.jl using Zarr +import Zarr: ZarrCore +using DiskArrays: GridChunks using JSON # Paths @@ -14,7 +16,7 @@ end # Create store and root group for v3 store = Zarr.DirectoryStore(path_v3) -g = zgroup(store, "", Zarr.ZarrFormat(3)) +g = zgroup(store, "", ZarrCore.ZarrFormat(3)) # Helper: create array and set data function create_and_fill(store, name, data; @@ -216,25 +218,27 @@ function create_sharded(store, name, data, outer_chunk_shape, inner_chunk_shape; index_location::Symbol=:end, index_crc32c::Bool=true) T = eltype(data) N = ndims(data) - inner_pipeline = Zarr.V3Pipeline( + inner_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), (Zarr.Codecs.V3Codecs.GzipV3Codec(1),), ) index_bytes_bytes = index_crc32c ? (Zarr.Codecs.V3Codecs.CRC32cV3Codec(),) : () - index_pipeline = Zarr.V3Pipeline( + index_pipeline = ZarrCore.V3Pipeline( (), Zarr.Codecs.V3Codecs.BytesCodec(:little), index_bytes_bytes, ) sharding = Zarr.Codecs.V3Codecs.ShardingCodec(inner_chunk_shape, inner_pipeline, index_pipeline, index_location) - pipeline = Zarr.V3Pipeline((), sharding, ()) - md = Zarr.MetadataV3{T, N, typeof(pipeline)}( - 3, "array", size(data), outer_chunk_shape, Zarr.typestr3(T), pipeline, zero(T), - Zarr.ChunkKeyEncoding('/', true), + pipeline = ZarrCore.V3Pipeline((), sharding, ()) + ch = GridChunks(size(data), outer_chunk_shape) + cke = Zarr.ChunkKeyEncoding('/', true) + md = ZarrCore.MetadataV3{T, N, typeof(pipeline), typeof(cke), typeof(ch)}( + 3, "array", size(data), ch, ZarrCore.typestr3(T), pipeline, zero(T), + cke, ) z = Zarr.ZArray(md, store, name, Dict(), true) - Zarr.writemetadata(Zarr.zarr_format(md), store, name, md) + ZarrCore.writemetadata(ZarrCore.zarr_format(md), store, name, md) z[:] = data return z end @@ -327,6 +331,6 @@ create_and_fill(store, "consolidated/nested/1d.i2", Int16[10, 20, 30, 40]; compressor=Zarr.NoCompressor(), ) # Consolidate metadata for the consolidated group only -Zarr.consolidate_metadata(store, "consolidated", Zarr.ZarrFormat(3)) +Zarr.consolidate_metadata(store, "consolidated", ZarrCore.ZarrFormat(3)) @info "Zarr v3 fixtures generated at: $path_v3" diff --git a/test/v3_python.jl b/test/v3_python.jl index ec319a4d..20e4f1c7 100644 --- a/test/v3_python.jl +++ b/test/v3_python.jl @@ -6,7 +6,7 @@ using JSON # Install Python deps into Conda env used by PythonCall (zarr v3 and numpy) CondaPkg.add([ - PkgSpec("numpy"), + PkgSpec("numpy"; version=">=2.3.3,<3"), PkgSpec("zarr"; version="3.*"), PkgSpec("numcodecs") ])