From c40381d0a8cdb50cf38830b712602441be7b9af7 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Thu, 8 Oct 2026 16:27:04 +0200 Subject: [PATCH 1/7] migrate Zarr backend to ZarrCore and add ZarrZip extension support --- Project.toml | 14 ++++---- README.md | 10 ++++++ ext/{ZarrExt.jl => ZarrCoreExt.jl} | 54 +++++++----------------------- ext/ZarrZipExt.jl | 35 +++++++++++++++++++ src/datasets/datasetinterface.jl | 8 +++++ test/Project.toml | 5 ++- test/datasets.jl | 39 ++++++++++++++++++++- 7 files changed, 116 insertions(+), 49 deletions(-) rename ext/{ZarrExt.jl => ZarrCoreExt.jl} (63%) create mode 100644 ext/ZarrZipExt.jl diff --git a/Project.toml b/Project.toml index a7501ad..80be18c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "YAXArrayBase" uuid = "90b8fcef-0c2d-428d-9c56-5f86629e9d14" authors = ["Fabian Gans "] -version = "0.7.11" +version = "0.8.0" [deps] DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" @@ -15,8 +15,10 @@ ArchGDAL = "0.10" AxisArrays = "0.4" AxisKeys = "0.2" DimensionalData = "0.27, 0.28, 0.29, 0.30" +DiskArrays = "0.4" NetCDF = "0.11, 0.12" -Zarr = "0.10" +ZarrCore = "0.11" +ZarrZip = "0.11" [extensions] @@ -26,16 +28,16 @@ AxisKeysExt = "AxisKeys" DimensionalDataExt = "DimensionalData" NamedDimsExt = "NamedDims" NetCDFExt = "NetCDF" -ZarrExt = ["Zarr", "ZipArchives", "DiskArrays"] +ZarrCoreExt = "ZarrCore" +ZarrZipExt = ["ZarrCore", "ZarrZip", "DiskArrays"] [weakdeps] ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" -Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" NamedDims = "356022a1-0364-5f58-8944-0da4b18d706f" NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9" -Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" -ZipArchives = "49080126-0e18-4c2a-b176-c102e4b3760c" +ZarrCore = "77f5b75c-4c08-499f-ba13-550b0a0af171" +ZarrZip = "54bcf1c8-e61d-4e4f-b375-463581846cca" diff --git a/README.md b/README.md index 8808df7..09e374b 100644 --- a/README.md +++ b/README.md @@ -83,3 +83,13 @@ dimvals(a,1) ```` Look at `src/axisinterface` to get a full description of the interface. + +## Zarr backend + +The Zarr dataset backend is enabled by loading `ZarrCore`. ZarrCore on its own +only handles local stores and writes uncompressed data. Load the subpackages +you need, or simply `using Zarr` to get all of them: + +- `ZarrHTTP`, `ZarrS3`, `ZarrGCS` for remote stores +- `ZarrBlosc`, `ZarrZlib`, `ZarrZstd` for compression (`ZarrBlosc` makes Blosc the default compressor) +- `ZarrZip` for reading `.zarr.zip` files diff --git a/ext/ZarrExt.jl b/ext/ZarrCoreExt.jl similarity index 63% rename from ext/ZarrExt.jl rename to ext/ZarrCoreExt.jl index b599233..13ada3c 100644 --- a/ext/ZarrExt.jl +++ b/ext/ZarrCoreExt.jl @@ -1,8 +1,6 @@ -module ZarrExt +module ZarrCoreExt using YAXArrayBase -using Zarr: ZArray, ZGroup, zgroup, zcreate, to_zarrtype, zopen, Compressor, ZipStore, NoCompressor, MetadataV2, MetadataV3 -import DiskArrays: AbstractDiskArray, DiskArrays, Unchunked, Chunked, GridChunks -using ZipArchives: ZipReader +using ZarrCore: ZarrCore, ZArray, ZGroup, zgroup, zcreate, zopen, NoCompressor, CRC32cV3Codec import YAXArrayBase: YAXArrayBase as YAB export ZarrDataset @@ -20,7 +18,7 @@ function ZarrDataset(g::Union{String,ZGroup}; mode="r", path="", kwargs...) return ZarrDataset(g) end store = if endswith(g, "zip") - ZipStore(ZipReader(SimpleFileDiskArray(g))) + YAB.zarr_zipstore(g) else g end @@ -49,6 +47,9 @@ Base.haskey(ds::ZarrDataset, k) = haskey(ds.g, k) function YAB.add_var(p::ZarrDataset, T::Type, varname, s, dimnames, attr; chunksize=s, fill_as_missing=false, kwargs...) + if !haskey(kwargs, :compressor) && ZarrCore.default_compressor() isa NoCompressor + @info "No Zarr compressor package is loaded, so data will be written uncompressed. Load e.g. ZarrBlosc or Zarr to enable compression." maxlog = 1 + end attr2 = merge(attr, Dict("_ARRAY_DIMENSIONS" => reverse(collect(String, dimnames)))) fv = get(attr, "_FillValue", get(attr, "missing_value", YAB.defaultfillval(T))) attr3 = filter(attr2) do (k, v) @@ -61,7 +62,8 @@ end #Special case for init with Arrays function YAB.add_var(p::ZarrDataset, a::AbstractArray, varname, dimnames, attr; kwargs...) - T = to_zarrtype(a) + # to_zarrtype is not public in ZarrCore + T = ZarrCore.to_zarrtype(a) b = add_var(p, T, varname, size(a), dimnames, attr; kwargs...) b .= a a @@ -74,40 +76,10 @@ YAB.create_empty(::Type{ZarrDataset}, path, gatts=Dict()) = ZarrDataset(zgroup(p YAB.allow_parallel_write(::ZarrDataset) = true YAB.allow_missings(::ZarrDataset) = false YAB.to_dataset(g::ZGroup; kwargs...) = ZarrDataset(g; kwargs...) -YAB.iscompressed(a::ZArray) = _iscompressed(a.metadata) -_iscompressed(m::MetadataV3) = !isempty(m.codec.byte_to_byte) -_iscompressed(m::MetadataV2) = !isa(m.compressor, NoCompressor) - - -#Add ability to read zipped zarrs - +# get_pipeline, V2Pipeline and V3Pipeline are not public in ZarrCore +YAB.iscompressed(a::ZArray) = _iscompressed(ZarrCore.get_pipeline(a.metadata)) +_iscompressed(p::ZarrCore.V2Pipeline) = !isa(p.compressor, NoCompressor) +# Checksum codecs are bytes-to-bytes codecs too, but do not compress +_iscompressed(p::ZarrCore.V3Pipeline) = any(c -> !isa(c, CRC32cV3Codec), p.bytes_bytes) -struct SimpleFileDiskArray{C<:Union{Int,Nothing}} <: AbstractDiskArray{UInt8,1} - file::String - s::Int - chunksize::C -end -Base.size(s::SimpleFileDiskArray) = (s.s,) -function SimpleFileDiskArray(filename; chunksize=nothing) - isfile(filename) || throw(ArgumentError("File $filename does not exist")) - s = filesize(filename) - SimpleFileDiskArray(filename, s, chunksize) -end -function DiskArrays.readblock!(a::SimpleFileDiskArray, aout, i::AbstractUnitRange) - open(a.file) do f - seek(f, first(i) - 1) - read!(f, aout) - end end -DiskArrays.haschunks(a::SimpleFileDiskArray) = a.chunksize === nothing ? Unchunked() : Chunked() -function DiskArrays.eachchunk(a::SimpleFileDiskArray) - if a.chunksize === nothing - DiskArrays.estimate_chunksize(a) - else - GridChunks((a.s,), (a.chunksize,)) - end -end - - - -end \ No newline at end of file diff --git a/ext/ZarrZipExt.jl b/ext/ZarrZipExt.jl new file mode 100644 index 0000000..e2cfe12 --- /dev/null +++ b/ext/ZarrZipExt.jl @@ -0,0 +1,35 @@ +module ZarrZipExt +using ZarrZip: ZipStore +import DiskArrays: AbstractDiskArray, DiskArrays, Unchunked, Chunked, GridChunks +import YAXArrayBase: YAXArrayBase as YAB + +#Add ability to read zipped zarrs +YAB.zarr_zipstore(path::AbstractString) = ZipStore(SimpleFileDiskArray(path)) + +struct SimpleFileDiskArray{C<:Union{Int,Nothing}} <: AbstractDiskArray{UInt8,1} + file::String + s::Int + chunksize::C +end +Base.size(s::SimpleFileDiskArray) = (s.s,) +function SimpleFileDiskArray(filename; chunksize=nothing) + isfile(filename) || throw(ArgumentError("File $filename does not exist")) + s = filesize(filename) + SimpleFileDiskArray(filename, s, chunksize) +end +function DiskArrays.readblock!(a::SimpleFileDiskArray, aout, i::AbstractUnitRange) + open(a.file) do f + seek(f, first(i) - 1) + read!(f, aout) + end +end +DiskArrays.haschunks(a::SimpleFileDiskArray) = a.chunksize === nothing ? Unchunked() : Chunked() +function DiskArrays.eachchunk(a::SimpleFileDiskArray) + if a.chunksize === nothing + DiskArrays.estimate_chunksize(a) + else + GridChunks((a.s,), (a.chunksize,)) + end +end + +end diff --git a/src/datasets/datasetinterface.jl b/src/datasets/datasetinterface.jl index d2a65c2..7b5c9c4 100644 --- a/src/datasets/datasetinterface.jl +++ b/src/datasets/datasetinterface.jl @@ -81,6 +81,14 @@ backendlist = OrderedDict{Symbol, Any}( backendregex = Pair[] +""" + zarr_zipstore(path) + +Open the zipped Zarr store at `path`. Used by the Zarr backend and implemented +when `ZarrZip` is loaded. +""" +zarr_zipstore(path) = throw(ArgumentError("Reading zipped Zarr stores requires ZarrZip. Run `using ZarrZip` (or `using Zarr`) first.")) + function backendfrompath(g::String; driver = :all) if driver == :all isempty(backendregex) && throw("No backend found. Load a backend by using the corresponding package.") diff --git a/test/Project.toml b/test/Project.toml index e845daa..eb6a89d 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -11,4 +11,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" TestItems = "1c621080-faea-4a02-84b6-bbd5e436b8fe" YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14" -Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" +ZarrBlosc = "5717d1f5-dd42-40cd-8e77-00dfad1079f5" +ZarrCore = "77f5b75c-4c08-499f-ba13-550b0a0af171" +ZarrHTTP = "904a97b2-8abf-4ac8-b211-6540db5c2c8c" +ZarrZip = "54bcf1c8-e61d-4e4f-b375-463581846cca" diff --git a/test/datasets.jl b/test/datasets.jl index f89ae04..ef8d6e3 100644 --- a/test/datasets.jl +++ b/test/datasets.jl @@ -3,7 +3,7 @@ using YAXArrayBase, Test @test_throws "No backend found." YAXArrayBase.backendfrompath("test.zarr") end -using NetCDF, Zarr +using NetCDF, ZarrCore using Pkg.Artifacts import Downloads @@ -63,6 +63,43 @@ YAXArrayBase.open_dataset_handle(ds_nc2) do ds_nc end end +@testset "Zarr with bare ZarrCore" begin + path = tempname() * ".zarr" + ds = create_empty(YAXArrayBase.backendlist[:zarr], path) + v = @test_logs (:info, r"uncompressed") add_var(ds, Float32, "tas", (3, 4), ("lon", "lat"), Dict{String,Any}()) + v[:, :] = reshape(1:12, 3, 4) + ds_loaded = to_dataset(path) + @test get_var_dims(ds_loaded, "tas") == ["lon", "lat"] + h = get_var_handle(ds_loaded, "tas") + @test h[:, :] == reshape(1:12, 3, 4) + @test !YAXArrayBase.iscompressed(h) + @test !YAXArrayBase.iscompressed(zcreate(Float32, 3, 4, zarr_format=3)) + @test_throws "ZarrZip" to_dataset(tempname() * ".zarr.zip") +end + +using ZarrHTTP, ZarrBlosc, ZarrZip + +@testset "Zarr iscompressed" begin + for zarr_format in (2, 3) + @test YAXArrayBase.iscompressed(zcreate(Float32, 3, 4; zarr_format)) + @test !YAXArrayBase.iscompressed(zcreate(Float32, 3, 4; zarr_format, compressor=ZarrCore.NoCompressor())) + end +end + +@testset "Reading zipped Zarr" begin + path = tempname() * ".zarr" + ds = create_empty(YAXArrayBase.backendlist[:zarr], path) + add_var(ds, reshape(1.0:12.0, 3, 4), "tas", ("lon", "lat"), Dict{String,Any}("units" => "K")) + zippath = path * ".zip" + open(io -> ZarrZip.writezip(io, ds.g), zippath, "w") + ds_zip = to_dataset(zippath) + @test ds_zip isa YAXArrayBase.backendlist[:zarr] + @test get_varnames(ds_zip) == ["tas"] + @test get_var_dims(ds_zip, "tas") == ["lon", "lat"] + @test get_var_attrs(ds_zip, "tas")["units"] == "K" + @test get_var_handle(ds_zip, "tas")[:, :] == reshape(1.0:12.0, 3, 4) +end + @testset "Reading Zarr" begin p = "https://s3.bgc-jena.mpg.de:9000/esdl-esdc-v3.0.2/esdc-16d-2.5deg-46x72x1440-3.0.2.zarr" for ds_zarr in [to_dataset(p,driver=:zarr), to_dataset(zopen(p))] From c529523af92f88644e7fbccfffbae338a9b75fee Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Thu, 8 Oct 2026 17:06:07 +0200 Subject: [PATCH 2/7] add support for Zarr v3 format and dimension names in ZarrCore extension --- Project.toml | 2 +- ext/NetCDFExt.jl | 2 +- ext/ZarrCoreExt.jl | 29 ++++++++++++++--- src/datasets/datasetinterface.jl | 9 +++-- test/Project.toml | 4 +++ test/datasets.jl | 56 +++++++++++++++++++++++++++++--- 6 files changed, 90 insertions(+), 12 deletions(-) diff --git a/Project.toml b/Project.toml index 80be18c..e805238 100644 --- a/Project.toml +++ b/Project.toml @@ -17,7 +17,7 @@ AxisKeys = "0.2" DimensionalData = "0.27, 0.28, 0.29, 0.30" DiskArrays = "0.4" NetCDF = "0.11, 0.12" -ZarrCore = "0.11" +ZarrCore = "0.11.1" ZarrZip = "0.11" diff --git a/ext/NetCDFExt.jl b/ext/NetCDFExt.jl index 2480a14..560a775 100644 --- a/ext/NetCDFExt.jl +++ b/ext/NetCDFExt.jl @@ -95,7 +95,7 @@ function YAB.add_var(p::NetCDFDataset, T::Type, varname, s, dimnames, attr; NetCDFVariable{T,length(s)}(p.filename,varname,(s...,)) end -function YAB.create_empty(::Type{NetCDFDataset}, path, gatts=Dict()) +function YAB.create_empty(::Type{NetCDFDataset}, path, gatts=Dict(); kwargs...) NetCDF.create(_->nothing, path, NcVar[], gatts = gatts) NetCDFDataset(path) end diff --git a/ext/ZarrCoreExt.jl b/ext/ZarrCoreExt.jl index 13ada3c..b79c5d3 100644 --- a/ext/ZarrCoreExt.jl +++ b/ext/ZarrCoreExt.jl @@ -25,7 +25,16 @@ function ZarrDataset(g::Union{String,ZGroup}; mode="r", path="", kwargs...) ZarrDataset(zopen(store, mode, fill_as_missing=false, path=path)) end -YAB.get_var_dims(ds::ZarrDataset, name) = reverse(ds[name].attrs["_ARRAY_DIMENSIONS"]) +function YAB.get_var_dims(ds::ZarrDataset, name) + a = ds[name] + # Zarr v3 stores dimension names in the array metadata, already in Julia order + dn = ZarrCore.dimension_names(a) + if dn !== nothing && all(!isnothing, dn) + return collect(String, dn) + end + haskey(a.attrs, "_ARRAY_DIMENSIONS") || throw(ArgumentError("Zarr array $name has no dimension names")) + reverse(a.attrs["_ARRAY_DIMENSIONS"]) +end YAB.get_varnames(ds::ZarrDataset) = collect(keys(ds.g.arrays)) function YAB.get_var_attrs(ds::ZarrDataset, name) #We add the fill value to the attributes to be consistent with NetCDF @@ -46,11 +55,20 @@ Base.haskey(ds::ZarrDataset, k) = haskey(ds.g, k) # end function YAB.add_var(p::ZarrDataset, T::Type, varname, s, dimnames, attr; - chunksize=s, fill_as_missing=false, kwargs...) + chunksize=s, fill_as_missing=false, zarr_format=nothing, kwargs...) + # The format is set by the group in create_empty + if zarr_format !== nothing && zarr_format != _zarr_format(p) + throw(ArgumentError("Can not create a Zarr v$zarr_format array in a Zarr v$(_zarr_format(p)) group")) + end if !haskey(kwargs, :compressor) && ZarrCore.default_compressor() isa NoCompressor @info "No Zarr compressor package is loaded, so data will be written uncompressed. Load e.g. ZarrBlosc or Zarr to enable compression." maxlog = 1 end - attr2 = merge(attr, Dict("_ARRAY_DIMENSIONS" => reverse(collect(String, dimnames)))) + if _zarr_format(p) == 2 + attr2 = merge(attr, Dict("_ARRAY_DIMENSIONS" => reverse(collect(String, dimnames)))) + else + attr2 = attr + kwargs = (; kwargs..., dimension_names=Tuple(collect(String, dimnames))) + end fv = get(attr, "_FillValue", get(attr, "missing_value", YAB.defaultfillval(T))) attr3 = filter(attr2) do (k, v) !isa(v, AbstractFloat) || !isnan(v) @@ -69,7 +87,10 @@ function YAB.add_var(p::ZarrDataset, a::AbstractArray, varname, dimnames, attr; a end -YAB.create_empty(::Type{ZarrDataset}, path, gatts=Dict()) = ZarrDataset(zgroup(path, attrs=gatts)) +YAB.create_empty(::Type{ZarrDataset}, path, gatts=Dict(); zarr_format=2, kwargs...) = + ZarrDataset(zgroup(ZarrCore.storefromstring(path, true)..., zarr_format; attrs=gatts)) +_zarr_format(ds::ZarrDataset) = _zarr_format(ds.g.zarr_format) +_zarr_format(::ZarrCore.ZarrFormat{N}) where N = N diff --git a/src/datasets/datasetinterface.jl b/src/datasets/datasetinterface.jl index 7b5c9c4..fa2f876 100644 --- a/src/datasets/datasetinterface.jl +++ b/src/datasets/datasetinterface.jl @@ -15,7 +15,12 @@ function get_var_attrs end function get_global_attrs end #Functions to be implemented for Dataset sinks -"Initialize and return a handle to a new empty dataset" +""" + create_empty(T, path, gatts; kwargs...) + +Initialize and return a handle to a new empty dataset. Backends ignore keyword +arguments they do not support. +""" function create_empty end "Apply a function `f` on a dataset `ds` while keeping possible file handles open during the operations" @@ -56,7 +61,7 @@ function add_var(ds,x::AbstractArray,name,dimlist,atts;kwargs...) end function create_dataset(T::Type, path, gatts, dimnames, dimvals, dimattrs, vartypes, varnames, vardims, varattrs, varchunks; kwargs...) - ds = create_empty(T, path, gatts) + ds = create_empty(T, path, gatts; kwargs...) axlengths = Dict{String, Int}() for (dname, dval, dattr) in zip(dimnames, dimvals, dimattrs) add_var(ds, dval, dname, (dname,), dattr) diff --git a/test/Project.toml b/test/Project.toml index eb6a89d..336d296 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -4,6 +4,7 @@ AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" NamedDims = "356022a1-0364-5f58-8944-0da4b18d706f" NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" @@ -15,3 +16,6 @@ ZarrBlosc = "5717d1f5-dd42-40cd-8e77-00dfad1079f5" ZarrCore = "77f5b75c-4c08-499f-ba13-550b0a0af171" ZarrHTTP = "904a97b2-8abf-4ac8-b211-6540db5c2c8c" ZarrZip = "54bcf1c8-e61d-4e4f-b375-463581846cca" + +[compat] +ZarrCore = "0.11.1" diff --git a/test/datasets.jl b/test/datasets.jl index ef8d6e3..7f89daf 100644 --- a/test/datasets.jl +++ b/test/datasets.jl @@ -140,9 +140,9 @@ end @test allow_parallel_write(ds_tif) == false @test allow_missings(ds_tif) == true end -function test_write(T) +function test_write(T; kwargs...) p = tempname() - ds = create_empty(T, p) + ds = create_empty(T, p; kwargs...) add_var(ds, 0.5:1:9.5, "lon", ("lon",), Dict("units"=>"degrees_east")) add_var(ds, 20:-1.0:1, "lat", ("lat",), Dict("units"=>"degrees_north")) v = add_var(ds, Float32, "tas", (10,20), ("lon", "lat"), Dict{String,Any}("units"=>"Celsius")) @@ -173,6 +173,54 @@ end test_write(YAXArrayBase.backendlist[:netcdf]) end -@testset "Writing Zarr" begin - test_write(YAXArrayBase.backendlist[:zarr]) +@testset "Writing Zarr v$zarr_format" for zarr_format in (2, 3) + test_write(YAXArrayBase.backendlist[:zarr]; zarr_format) +end + +import JSON +@testset "create_dataset Zarr v$zarr_format" for zarr_format in (2, 3) + ZD = YAXArrayBase.backendlist[:zarr] + path = tempname() * ".zarr" + ds = YAXArrayBase.create_dataset(ZD, path, Dict("title" => "test"), + ["lon", "lat"], [0.5:1:2.5, 10.0:-1:7], [Dict("units" => "degrees_east"), Dict("units" => "degrees_north")], + [Float32, Int], ["tas", "count"], [["lon", "lat"], ["lat"]], [Dict{String,Any}(), Dict{String,Any}()], + [(3, 4), (4,)]; zarr_format) + add_var(ds, fill(1), "scalar", (), Dict{String,Any}()) + if zarr_format == 3 + nodes = [joinpath(r, f) for (r, _, fs) in walkdir(path) for f in fs if f == "zarr.json"] + @test length(nodes) == 6 + for n in nodes + j = JSON.parsefile(n) + @test j["zarr_format"] == 3 + if j["node_type"] == "array" + @test haskey(j, "dimension_names") + @test !haskey(get(j, "attributes", Dict()), "_ARRAY_DIMENSIONS") + end + end + @test !any(f -> f in (".zgroup", ".zarray", ".zattrs"), (f for (_, _, fs) in walkdir(path) for f in fs)) + else + @test isfile(joinpath(path, ".zgroup")) + @test !isfile(joinpath(path, "zarr.json")) + @test ds["tas"].attrs["_ARRAY_DIMENSIONS"] == ["lat", "lon"] + @test ZarrCore.dimension_names(ds["tas"]) === nothing + end + ds2 = to_dataset(path) + @test get_var_dims(ds2, "tas") == ["lon", "lat"] + @test get_var_dims(ds2, "count") == ["lat"] + @test get_var_dims(ds2, "lon") == ["lon"] + @test get_var_dims(ds2, "scalar") == [] + @test YAXArrayBase.get_global_attrs(ds2)["title"] == "test" + @test_throws ArgumentError add_var(ds, Float32, "bad", (3,), ("lon",), Dict{String,Any}(); zarr_format=5 - zarr_format) +end + +@testset "Reading Zarr v3 with only dimension_names" begin + path = tempname() * ".zarr" + g = zgroup(ZarrCore.storefromstring(path, true)..., 3) + zcreate(Float32, g, "a", 3, 4; dimension_names=("x", "y")) + zcreate(Float32, g, "partial", 3, 4; dimension_names=("x", nothing), attrs=Dict("_ARRAY_DIMENSIONS" => ["y", "x"])) + zcreate(Float32, g, "unnamed", 3, 4) + ds = to_dataset(path) + @test get_var_dims(ds, "a") == ["x", "y"] + @test get_var_dims(ds, "partial") == ["x", "y"] + @test_throws ArgumentError get_var_dims(ds, "unnamed") end From 86d4858c1ddbf837cf4ab6d42111b94ef6251385 Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Thu, 8 Oct 2026 19:14:22 +0200 Subject: [PATCH 3/7] Apply compression settings to coordinate arrays and fix NetCDF iscompressed create_dataset now forwards `compressor` (Zarr) and `compress` (NetCDF) to the coordinate arrays, not only to the data variables. Other keyword arguments such as filters stay data-variable only, since they can be type specific. NetCDF iscompressed always returned false because NetCDF.jl does not read the deflate level back when opening a variable. Query it with nc_inq_var_deflate instead. --- ext/NetCDFExt.jl | 9 ++++++++- src/datasets/datasetinterface.jl | 5 ++++- test/datasets.jl | 21 +++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/ext/NetCDFExt.jl b/ext/NetCDFExt.jl index 560a775..a6523cd 100644 --- a/ext/NetCDFExt.jl +++ b/ext/NetCDFExt.jl @@ -70,7 +70,14 @@ function readblock!(v::NetCDFVariable, aout, r::AbstractUnitRange...) aout .= aouttemp end end -YAB.iscompressed(v::NetCDFVariable) = NetCDF.open(v->v.compress > 0, v.filename, v.varname) +# NcVar.compress is not read back from the file, so ask the library for the deflate setting +function YAB.iscompressed(v::NetCDFVariable) + NetCDF.open(v.filename, v.varname) do nv + shuffle, deflate, level = Ref{Cint}(0), Ref{Cint}(0), Ref{Cint}(0) + NetCDF.nc_inq_var_deflate(nv.ncid, nv.varid, shuffle, deflate, level) + deflate[] != 0 + end +end Base.size(v::NetCDFVariable) = v.size diff --git a/src/datasets/datasetinterface.jl b/src/datasets/datasetinterface.jl index fa2f876..ccb4353 100644 --- a/src/datasets/datasetinterface.jl +++ b/src/datasets/datasetinterface.jl @@ -63,8 +63,11 @@ end function create_dataset(T::Type, path, gatts, dimnames, dimvals, dimattrs, vartypes, varnames, vardims, varattrs, varchunks; kwargs...) ds = create_empty(T, path, gatts; kwargs...) axlengths = Dict{String, Int}() + # Coordinates share the compression settings (Zarr compressor, NetCDF compress), + # other options like filters may be type specific + coordkwargs = (; (k => v for (k, v) in pairs(kwargs) if k in (:compressor, :compress))...) for (dname, dval, dattr) in zip(dimnames, dimvals, dimattrs) - add_var(ds, dval, dname, (dname,), dattr) + add_var(ds, dval, dname, (dname,), dattr; coordkwargs...) axlengths[dname] = length(dval) end for (T, vn, vd, va, vc) in zip(vartypes, varnames, vardims, varattrs, varchunks) diff --git a/test/datasets.jl b/test/datasets.jl index 7f89daf..d9765db 100644 --- a/test/datasets.jl +++ b/test/datasets.jl @@ -173,6 +173,18 @@ end test_write(YAXArrayBase.backendlist[:netcdf]) end +@testset "create_dataset NetCDF compress" begin + ND = YAXArrayBase.backendlist[:netcdf] + for (compress, compressed) in ((-1, false), (5, true)) + path = tempname() * ".nc" + YAXArrayBase.create_dataset(ND, path, Dict(), ["lon"], [0.5:1:2.5], [Dict()], + [Float32], ["tas"], [["lon"]], [Dict{String,Any}()], [(3,)]; compress) + ds = to_dataset(path, driver=:netcdf) + @test YAXArrayBase.iscompressed(get_var_handle(ds, "tas")) == compressed + @test YAXArrayBase.iscompressed(get_var_handle(ds, "lon")) == compressed + end +end + @testset "Writing Zarr v$zarr_format" for zarr_format in (2, 3) test_write(YAXArrayBase.backendlist[:zarr]; zarr_format) end @@ -211,6 +223,15 @@ import JSON @test get_var_dims(ds2, "scalar") == [] @test YAXArrayBase.get_global_attrs(ds2)["title"] == "test" @test_throws ArgumentError add_var(ds, Float32, "bad", (3,), ("lon",), Dict{String,Any}(); zarr_format=5 - zarr_format) + @test all(n -> YAXArrayBase.iscompressed(ds2[n]), ["tas", "lon", "lat"]) + + # The compressor also applies to the coordinate arrays + path = tempname() * ".zarr" + ds = YAXArrayBase.create_dataset(ZD, path, Dict(), ["lon"], [0.5:1:2.5], [Dict()], + [Float32], ["tas"], [["lon"]], [Dict{String,Any}()], [(3,)]; + zarr_format, compressor=ZarrCore.NoCompressor()) + @test !YAXArrayBase.iscompressed(ds["tas"]) + @test !YAXArrayBase.iscompressed(ds["lon"]) end @testset "Reading Zarr v3 with only dimension_names" begin From 9040952b8aa34ec5f0dc384193032455028b153e Mon Sep 17 00:00:00 2001 From: Lazaro Alonso Date: Thu, 8 Oct 2026 19:33:06 +0200 Subject: [PATCH 4/7] Allow Zarr v3 scalar arrays without dimension_names Zarr v3 allows omitting dimension_names, which zarr-python does for 0-dimensional arrays. get_var_dims threw for such arrays because neither dimension_names nor _ARRAY_DIMENSIONS was present. Return an empty list for arrays without dimensions instead. --- ext/ZarrCoreExt.jl | 8 ++++++-- test/datasets.jl | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ext/ZarrCoreExt.jl b/ext/ZarrCoreExt.jl index b79c5d3..98f922e 100644 --- a/ext/ZarrCoreExt.jl +++ b/ext/ZarrCoreExt.jl @@ -32,8 +32,12 @@ function YAB.get_var_dims(ds::ZarrDataset, name) if dn !== nothing && all(!isnothing, dn) return collect(String, dn) end - haskey(a.attrs, "_ARRAY_DIMENSIONS") || throw(ArgumentError("Zarr array $name has no dimension names")) - reverse(a.attrs["_ARRAY_DIMENSIONS"]) + if haskey(a.attrs, "_ARRAY_DIMENSIONS") + return reverse(a.attrs["_ARRAY_DIMENSIONS"]) + end + # Scalars have no dimensions to name, Zarr v3 allows omitting dimension_names for them + ndims(a) == 0 && return String[] + throw(ArgumentError("Zarr array $name has no dimension names")) end YAB.get_varnames(ds::ZarrDataset) = collect(keys(ds.g.arrays)) function YAB.get_var_attrs(ds::ZarrDataset, name) diff --git a/test/datasets.jl b/test/datasets.jl index d9765db..5f17db9 100644 --- a/test/datasets.jl +++ b/test/datasets.jl @@ -240,8 +240,13 @@ end zcreate(Float32, g, "a", 3, 4; dimension_names=("x", "y")) zcreate(Float32, g, "partial", 3, 4; dimension_names=("x", nothing), attrs=Dict("_ARRAY_DIMENSIONS" => ["y", "x"])) zcreate(Float32, g, "unnamed", 3, 4) + scalar = zcreate(Float64, g, "scalar") + scalar[] = 2.5 ds = to_dataset(path) @test get_var_dims(ds, "a") == ["x", "y"] @test get_var_dims(ds, "partial") == ["x", "y"] @test_throws ArgumentError get_var_dims(ds, "unnamed") + @test !occursin("dimension_names", read(joinpath(path, "scalar", "zarr.json"), String)) + @test get_var_dims(ds, "scalar") == [] + @test get_var_handle(ds, "scalar")[] == 2.5 end From c7b446de3e78ce8546b82172d64061d25e457f53 Mon Sep 17 00:00:00 2001 From: Felix Cremer Date: Fri, 9 Oct 2026 10:54:45 +0200 Subject: [PATCH 5/7] Rename zarr_format keyword to format and use zarr_format function from Zarr --- ext/ZarrCoreExt.jl | 17 ++++++++--------- test/datasets.jl | 24 ++++++++++++------------ 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/ext/ZarrCoreExt.jl b/ext/ZarrCoreExt.jl index 98f922e..eb56244 100644 --- a/ext/ZarrCoreExt.jl +++ b/ext/ZarrCoreExt.jl @@ -1,6 +1,6 @@ module ZarrCoreExt using YAXArrayBase -using ZarrCore: ZarrCore, ZArray, ZGroup, zgroup, zcreate, zopen, NoCompressor, CRC32cV3Codec +using ZarrCore: ZarrCore, ZArray, ZGroup, zgroup, zcreate, zopen, NoCompressor, CRC32cV3Codec, zarr_format import YAXArrayBase: YAXArrayBase as YAB export ZarrDataset @@ -59,15 +59,15 @@ Base.haskey(ds::ZarrDataset, k) = haskey(ds.g, k) # end function YAB.add_var(p::ZarrDataset, T::Type, varname, s, dimnames, attr; - chunksize=s, fill_as_missing=false, zarr_format=nothing, kwargs...) + chunksize=s, fill_as_missing=false, format=nothing, kwargs...) # The format is set by the group in create_empty - if zarr_format !== nothing && zarr_format != _zarr_format(p) - throw(ArgumentError("Can not create a Zarr v$zarr_format array in a Zarr v$(_zarr_format(p)) group")) + if format !== nothing && format != zarr_format(p) + throw(ArgumentError("Can not create a Zarr v$format array in a Zarr v$(zarr_format(p)) group")) end if !haskey(kwargs, :compressor) && ZarrCore.default_compressor() isa NoCompressor @info "No Zarr compressor package is loaded, so data will be written uncompressed. Load e.g. ZarrBlosc or Zarr to enable compression." maxlog = 1 end - if _zarr_format(p) == 2 + if zarr_format(p) == 2 attr2 = merge(attr, Dict("_ARRAY_DIMENSIONS" => reverse(collect(String, dimnames)))) else attr2 = attr @@ -91,10 +91,9 @@ function YAB.add_var(p::ZarrDataset, a::AbstractArray, varname, dimnames, attr; a end -YAB.create_empty(::Type{ZarrDataset}, path, gatts=Dict(); zarr_format=2, kwargs...) = - ZarrDataset(zgroup(ZarrCore.storefromstring(path, true)..., zarr_format; attrs=gatts)) -_zarr_format(ds::ZarrDataset) = _zarr_format(ds.g.zarr_format) -_zarr_format(::ZarrCore.ZarrFormat{N}) where N = N +YAB.create_empty(::Type{ZarrDataset}, path, gatts=Dict(); format=2, kwargs...) = + ZarrDataset(zgroup(ZarrCore.storefromstring(path, true)..., format; attrs=gatts)) +zarr_format(ds::ZarrDataset) = zarr_format(ds.g) diff --git a/test/datasets.jl b/test/datasets.jl index 5f17db9..c5e4f44 100644 --- a/test/datasets.jl +++ b/test/datasets.jl @@ -73,16 +73,16 @@ end h = get_var_handle(ds_loaded, "tas") @test h[:, :] == reshape(1:12, 3, 4) @test !YAXArrayBase.iscompressed(h) - @test !YAXArrayBase.iscompressed(zcreate(Float32, 3, 4, zarr_format=3)) + @test !YAXArrayBase.iscompressed(zcreate(Float32, 3, 4, format=3)) @test_throws "ZarrZip" to_dataset(tempname() * ".zarr.zip") end using ZarrHTTP, ZarrBlosc, ZarrZip @testset "Zarr iscompressed" begin - for zarr_format in (2, 3) - @test YAXArrayBase.iscompressed(zcreate(Float32, 3, 4; zarr_format)) - @test !YAXArrayBase.iscompressed(zcreate(Float32, 3, 4; zarr_format, compressor=ZarrCore.NoCompressor())) + for format in (2, 3) + @test YAXArrayBase.iscompressed(zcreate(Float32, 3, 4; format)) + @test !YAXArrayBase.iscompressed(zcreate(Float32, 3, 4; format, compressor=ZarrCore.NoCompressor())) end end @@ -185,25 +185,25 @@ end end end -@testset "Writing Zarr v$zarr_format" for zarr_format in (2, 3) - test_write(YAXArrayBase.backendlist[:zarr]; zarr_format) +@testset "Writing Zarr v$format" for format in (2, 3) + test_write(YAXArrayBase.backendlist[:zarr]; format) end import JSON -@testset "create_dataset Zarr v$zarr_format" for zarr_format in (2, 3) +@testset "create_dataset Zarr v$format" for format in (2, 3) ZD = YAXArrayBase.backendlist[:zarr] path = tempname() * ".zarr" ds = YAXArrayBase.create_dataset(ZD, path, Dict("title" => "test"), ["lon", "lat"], [0.5:1:2.5, 10.0:-1:7], [Dict("units" => "degrees_east"), Dict("units" => "degrees_north")], [Float32, Int], ["tas", "count"], [["lon", "lat"], ["lat"]], [Dict{String,Any}(), Dict{String,Any}()], - [(3, 4), (4,)]; zarr_format) + [(3, 4), (4,)]; format) add_var(ds, fill(1), "scalar", (), Dict{String,Any}()) - if zarr_format == 3 + if format == 3 nodes = [joinpath(r, f) for (r, _, fs) in walkdir(path) for f in fs if f == "zarr.json"] @test length(nodes) == 6 for n in nodes j = JSON.parsefile(n) - @test j["zarr_format"] == 3 + @test j["format"] == 3 if j["node_type"] == "array" @test haskey(j, "dimension_names") @test !haskey(get(j, "attributes", Dict()), "_ARRAY_DIMENSIONS") @@ -222,14 +222,14 @@ import JSON @test get_var_dims(ds2, "lon") == ["lon"] @test get_var_dims(ds2, "scalar") == [] @test YAXArrayBase.get_global_attrs(ds2)["title"] == "test" - @test_throws ArgumentError add_var(ds, Float32, "bad", (3,), ("lon",), Dict{String,Any}(); zarr_format=5 - zarr_format) + @test_throws ArgumentError add_var(ds, Float32, "bad", (3,), ("lon",), Dict{String,Any}(); format=5 - format) @test all(n -> YAXArrayBase.iscompressed(ds2[n]), ["tas", "lon", "lat"]) # The compressor also applies to the coordinate arrays path = tempname() * ".zarr" ds = YAXArrayBase.create_dataset(ZD, path, Dict(), ["lon"], [0.5:1:2.5], [Dict()], [Float32], ["tas"], [["lon"]], [Dict{String,Any}()], [(3,)]; - zarr_format, compressor=ZarrCore.NoCompressor()) + format, compressor=ZarrCore.NoCompressor()) @test !YAXArrayBase.iscompressed(ds["tas"]) @test !YAXArrayBase.iscompressed(ds["lon"]) end From 0dbc0093c9ace29f6674034dce7a476c84bdcceb Mon Sep 17 00:00:00 2001 From: Felix Cremer Date: Fri, 9 Oct 2026 11:15:13 +0200 Subject: [PATCH 6/7] Import zarr_format to extend it --- ext/ZarrCoreExt.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/ZarrCoreExt.jl b/ext/ZarrCoreExt.jl index eb56244..5c22ea0 100644 --- a/ext/ZarrCoreExt.jl +++ b/ext/ZarrCoreExt.jl @@ -1,7 +1,8 @@ module ZarrCoreExt using YAXArrayBase -using ZarrCore: ZarrCore, ZArray, ZGroup, zgroup, zcreate, zopen, NoCompressor, CRC32cV3Codec, zarr_format +using ZarrCore: ZarrCore, ZArray, ZGroup, zgroup, zcreate, zopen, NoCompressor, CRC32cV3Codec import YAXArrayBase: YAXArrayBase as YAB +import ZarrCore: zarr_format export ZarrDataset function __init__() From 0fdbe4e2d9760851d8d29b7d47a810175bb90d4f Mon Sep 17 00:00:00 2001 From: Felix Cremer Date: Fri, 9 Oct 2026 12:06:00 +0200 Subject: [PATCH 7/7] Access metadata of wrapped group for zarr_format Stop gap till change landed in Zarr.jl --- ext/ZarrCoreExt.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/ZarrCoreExt.jl b/ext/ZarrCoreExt.jl index 5c22ea0..1d6e100 100644 --- a/ext/ZarrCoreExt.jl +++ b/ext/ZarrCoreExt.jl @@ -94,7 +94,7 @@ end YAB.create_empty(::Type{ZarrDataset}, path, gatts=Dict(); format=2, kwargs...) = ZarrDataset(zgroup(ZarrCore.storefromstring(path, true)..., format; attrs=gatts)) -zarr_format(ds::ZarrDataset) = zarr_format(ds.g) +zarr_format(ds::ZarrDataset) = zarr_format(ds.g.metadata)