diff --git a/Project.toml b/Project.toml index a7501ad..e805238 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "YAXArrayBase" uuid = "90b8fcef-0c2d-428d-9c56-5f86629e9d14" authors = ["Fabian Gans "] -version = "0.7.11" +version = "0.8.0" [deps] DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" @@ -15,8 +15,10 @@ ArchGDAL = "0.10" AxisArrays = "0.4" AxisKeys = "0.2" DimensionalData = "0.27, 0.28, 0.29, 0.30" +DiskArrays = "0.4" NetCDF = "0.11, 0.12" -Zarr = "0.10" +ZarrCore = "0.11.1" +ZarrZip = "0.11" [extensions] @@ -26,16 +28,16 @@ AxisKeysExt = "AxisKeys" DimensionalDataExt = "DimensionalData" NamedDimsExt = "NamedDims" NetCDFExt = "NetCDF" -ZarrExt = ["Zarr", "ZipArchives", "DiskArrays"] +ZarrCoreExt = "ZarrCore" +ZarrZipExt = ["ZarrCore", "ZarrZip", "DiskArrays"] [weakdeps] ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" -Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" NamedDims = "356022a1-0364-5f58-8944-0da4b18d706f" NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9" -Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" -ZipArchives = "49080126-0e18-4c2a-b176-c102e4b3760c" +ZarrCore = "77f5b75c-4c08-499f-ba13-550b0a0af171" +ZarrZip = "54bcf1c8-e61d-4e4f-b375-463581846cca" diff --git a/README.md b/README.md index 8808df7..09e374b 100644 --- a/README.md +++ b/README.md @@ -83,3 +83,13 @@ dimvals(a,1) ```` Look at `src/axisinterface` to get a full description of the interface. + +## Zarr backend + +The Zarr dataset backend is enabled by loading `ZarrCore`. ZarrCore on its own +only handles local stores and writes uncompressed data. Load the subpackages +you need, or simply `using Zarr` to get all of them: + +- `ZarrHTTP`, `ZarrS3`, `ZarrGCS` for remote stores +- `ZarrBlosc`, `ZarrZlib`, `ZarrZstd` for compression (`ZarrBlosc` makes Blosc the default compressor) +- `ZarrZip` for reading `.zarr.zip` files diff --git a/ext/NetCDFExt.jl b/ext/NetCDFExt.jl index 2480a14..a6523cd 100644 --- a/ext/NetCDFExt.jl +++ b/ext/NetCDFExt.jl @@ -70,7 +70,14 @@ function readblock!(v::NetCDFVariable, aout, r::AbstractUnitRange...) aout .= aouttemp end end -YAB.iscompressed(v::NetCDFVariable) = NetCDF.open(v->v.compress > 0, v.filename, v.varname) +# NcVar.compress is not read back from the file, so ask the library for the deflate setting +function YAB.iscompressed(v::NetCDFVariable) + NetCDF.open(v.filename, v.varname) do nv + shuffle, deflate, level = Ref{Cint}(0), Ref{Cint}(0), Ref{Cint}(0) + NetCDF.nc_inq_var_deflate(nv.ncid, nv.varid, shuffle, deflate, level) + deflate[] != 0 + end +end Base.size(v::NetCDFVariable) = v.size @@ -95,7 +102,7 @@ function YAB.add_var(p::NetCDFDataset, T::Type, varname, s, dimnames, attr; NetCDFVariable{T,length(s)}(p.filename,varname,(s...,)) end -function YAB.create_empty(::Type{NetCDFDataset}, path, gatts=Dict()) +function YAB.create_empty(::Type{NetCDFDataset}, path, gatts=Dict(); kwargs...) NetCDF.create(_->nothing, path, NcVar[], gatts = gatts) NetCDFDataset(path) end diff --git a/ext/ZarrCoreExt.jl b/ext/ZarrCoreExt.jl new file mode 100644 index 0000000..1d6e100 --- /dev/null +++ b/ext/ZarrCoreExt.jl @@ -0,0 +1,110 @@ +module ZarrCoreExt +using YAXArrayBase +using ZarrCore: ZarrCore, ZArray, ZGroup, zgroup, zcreate, zopen, NoCompressor, CRC32cV3Codec +import YAXArrayBase: YAXArrayBase as YAB +import ZarrCore: zarr_format +export ZarrDataset + +function __init__() + @debug "new driver key :zarr, updating backendlist." + YAB.backendlist[:zarr] = ZarrDataset + push!(YAB.backendregex, r"(.zarr$)|(.zarr/$)|(zarr.zip$)" => ZarrDataset) +end + +struct ZarrDataset + g::ZGroup +end +function ZarrDataset(g::Union{String,ZGroup}; mode="r", path="", kwargs...) + if g isa ZGroup + return ZarrDataset(g) + end + store = if endswith(g, "zip") + YAB.zarr_zipstore(g) + else + g + end + ZarrDataset(zopen(store, mode, fill_as_missing=false, path=path)) +end + +function YAB.get_var_dims(ds::ZarrDataset, name) + a = ds[name] + # Zarr v3 stores dimension names in the array metadata, already in Julia order + dn = ZarrCore.dimension_names(a) + if dn !== nothing && all(!isnothing, dn) + return collect(String, dn) + end + if haskey(a.attrs, "_ARRAY_DIMENSIONS") + return reverse(a.attrs["_ARRAY_DIMENSIONS"]) + end + # Scalars have no dimensions to name, Zarr v3 allows omitting dimension_names for them + ndims(a) == 0 && return String[] + throw(ArgumentError("Zarr array $name has no dimension names")) +end +YAB.get_varnames(ds::ZarrDataset) = collect(keys(ds.g.arrays)) +function YAB.get_var_attrs(ds::ZarrDataset, name) + #We add the fill value to the attributes to be consistent with NetCDF + a = ds[name] + if a.metadata.fill_value !== nothing + merge(ds[name].attrs, Dict("_FillValue" => a.metadata.fill_value)) + else + ds[name].attrs + end +end +YAB.get_global_attrs(ds::ZarrDataset) = ds.g.attrs +Base.getindex(ds::ZarrDataset, i) = ds.g[i] +Base.haskey(ds::ZarrDataset, k) = haskey(ds.g, k) + +# function add_var(p::ZarrDataset, T::Type{>:Missing}, varname, s, dimnames, attr; kwargs...) +# S = Base.nonmissingtype(T) +# add_var(p,S, varname, s, dimnames, attr; fill_value = defaultfillval(S), fill_as_missing=true, kwargs...) +# end + +function YAB.add_var(p::ZarrDataset, T::Type, varname, s, dimnames, attr; + chunksize=s, fill_as_missing=false, format=nothing, kwargs...) + # The format is set by the group in create_empty + if format !== nothing && format != zarr_format(p) + throw(ArgumentError("Can not create a Zarr v$format array in a Zarr v$(zarr_format(p)) group")) + end + if !haskey(kwargs, :compressor) && ZarrCore.default_compressor() isa NoCompressor + @info "No Zarr compressor package is loaded, so data will be written uncompressed. Load e.g. ZarrBlosc or Zarr to enable compression." maxlog = 1 + end + if zarr_format(p) == 2 + attr2 = merge(attr, Dict("_ARRAY_DIMENSIONS" => reverse(collect(String, dimnames)))) + else + attr2 = attr + kwargs = (; kwargs..., dimension_names=Tuple(collect(String, dimnames))) + end + fv = get(attr, "_FillValue", get(attr, "missing_value", YAB.defaultfillval(T))) + attr3 = filter(attr2) do (k, v) + !isa(v, AbstractFloat) || !isnan(v) + end + za = zcreate(T, p.g, varname, s...; fill_value=fv, fill_as_missing, attrs=attr3, chunks=chunksize, kwargs...) + za +end + +#Special case for init with Arrays +function YAB.add_var(p::ZarrDataset, a::AbstractArray, varname, dimnames, attr; + kwargs...) + # to_zarrtype is not public in ZarrCore + T = ZarrCore.to_zarrtype(a) + b = add_var(p, T, varname, size(a), dimnames, attr; kwargs...) + b .= a + a +end + +YAB.create_empty(::Type{ZarrDataset}, path, gatts=Dict(); format=2, kwargs...) = + ZarrDataset(zgroup(ZarrCore.storefromstring(path, true)..., format; attrs=gatts)) +zarr_format(ds::ZarrDataset) = zarr_format(ds.g.metadata) + + + +YAB.allow_parallel_write(::ZarrDataset) = true +YAB.allow_missings(::ZarrDataset) = false +YAB.to_dataset(g::ZGroup; kwargs...) = ZarrDataset(g; kwargs...) +# get_pipeline, V2Pipeline and V3Pipeline are not public in ZarrCore +YAB.iscompressed(a::ZArray) = _iscompressed(ZarrCore.get_pipeline(a.metadata)) +_iscompressed(p::ZarrCore.V2Pipeline) = !isa(p.compressor, NoCompressor) +# Checksum codecs are bytes-to-bytes codecs too, but do not compress +_iscompressed(p::ZarrCore.V3Pipeline) = any(c -> !isa(c, CRC32cV3Codec), p.bytes_bytes) + +end diff --git a/ext/ZarrExt.jl b/ext/ZarrExt.jl deleted file mode 100644 index b599233..0000000 --- a/ext/ZarrExt.jl +++ /dev/null @@ -1,113 +0,0 @@ -module ZarrExt -using YAXArrayBase -using Zarr: ZArray, ZGroup, zgroup, zcreate, to_zarrtype, zopen, Compressor, ZipStore, NoCompressor, MetadataV2, MetadataV3 -import DiskArrays: AbstractDiskArray, DiskArrays, Unchunked, Chunked, GridChunks -using ZipArchives: ZipReader -import YAXArrayBase: YAXArrayBase as YAB -export ZarrDataset - -function __init__() - @debug "new driver key :zarr, updating backendlist." - YAB.backendlist[:zarr] = ZarrDataset - push!(YAB.backendregex, r"(.zarr$)|(.zarr/$)|(zarr.zip$)" => ZarrDataset) -end - -struct ZarrDataset - g::ZGroup -end -function ZarrDataset(g::Union{String,ZGroup}; mode="r", path="", kwargs...) - if g isa ZGroup - return ZarrDataset(g) - end - store = if endswith(g, "zip") - ZipStore(ZipReader(SimpleFileDiskArray(g))) - else - g - end - ZarrDataset(zopen(store, mode, fill_as_missing=false, path=path)) -end - -YAB.get_var_dims(ds::ZarrDataset, name) = reverse(ds[name].attrs["_ARRAY_DIMENSIONS"]) -YAB.get_varnames(ds::ZarrDataset) = collect(keys(ds.g.arrays)) -function YAB.get_var_attrs(ds::ZarrDataset, name) - #We add the fill value to the attributes to be consistent with NetCDF - a = ds[name] - if a.metadata.fill_value !== nothing - merge(ds[name].attrs, Dict("_FillValue" => a.metadata.fill_value)) - else - ds[name].attrs - end -end -YAB.get_global_attrs(ds::ZarrDataset) = ds.g.attrs -Base.getindex(ds::ZarrDataset, i) = ds.g[i] -Base.haskey(ds::ZarrDataset, k) = haskey(ds.g, k) - -# function add_var(p::ZarrDataset, T::Type{>:Missing}, varname, s, dimnames, attr; kwargs...) -# S = Base.nonmissingtype(T) -# add_var(p,S, varname, s, dimnames, attr; fill_value = defaultfillval(S), fill_as_missing=true, kwargs...) -# end - -function YAB.add_var(p::ZarrDataset, T::Type, varname, s, dimnames, attr; - chunksize=s, fill_as_missing=false, kwargs...) - attr2 = merge(attr, Dict("_ARRAY_DIMENSIONS" => reverse(collect(String, dimnames)))) - fv = get(attr, "_FillValue", get(attr, "missing_value", YAB.defaultfillval(T))) - attr3 = filter(attr2) do (k, v) - !isa(v, AbstractFloat) || !isnan(v) - end - za = zcreate(T, p.g, varname, s...; fill_value=fv, fill_as_missing, attrs=attr3, chunks=chunksize, kwargs...) - za -end - -#Special case for init with Arrays -function YAB.add_var(p::ZarrDataset, a::AbstractArray, varname, dimnames, attr; - kwargs...) - T = to_zarrtype(a) - b = add_var(p, T, varname, size(a), dimnames, attr; kwargs...) - b .= a - a -end - -YAB.create_empty(::Type{ZarrDataset}, path, gatts=Dict()) = ZarrDataset(zgroup(path, attrs=gatts)) - - - -YAB.allow_parallel_write(::ZarrDataset) = true -YAB.allow_missings(::ZarrDataset) = false -YAB.to_dataset(g::ZGroup; kwargs...) = ZarrDataset(g; kwargs...) -YAB.iscompressed(a::ZArray) = _iscompressed(a.metadata) -_iscompressed(m::MetadataV3) = !isempty(m.codec.byte_to_byte) -_iscompressed(m::MetadataV2) = !isa(m.compressor, NoCompressor) - - -#Add ability to read zipped zarrs - - -struct SimpleFileDiskArray{C<:Union{Int,Nothing}} <: AbstractDiskArray{UInt8,1} - file::String - s::Int - chunksize::C -end -Base.size(s::SimpleFileDiskArray) = (s.s,) -function SimpleFileDiskArray(filename; chunksize=nothing) - isfile(filename) || throw(ArgumentError("File $filename does not exist")) - s = filesize(filename) - SimpleFileDiskArray(filename, s, chunksize) -end -function DiskArrays.readblock!(a::SimpleFileDiskArray, aout, i::AbstractUnitRange) - open(a.file) do f - seek(f, first(i) - 1) - read!(f, aout) - end -end -DiskArrays.haschunks(a::SimpleFileDiskArray) = a.chunksize === nothing ? Unchunked() : Chunked() -function DiskArrays.eachchunk(a::SimpleFileDiskArray) - if a.chunksize === nothing - DiskArrays.estimate_chunksize(a) - else - GridChunks((a.s,), (a.chunksize,)) - end -end - - - -end \ No newline at end of file diff --git a/ext/ZarrZipExt.jl b/ext/ZarrZipExt.jl new file mode 100644 index 0000000..e2cfe12 --- /dev/null +++ b/ext/ZarrZipExt.jl @@ -0,0 +1,35 @@ +module ZarrZipExt +using ZarrZip: ZipStore +import DiskArrays: AbstractDiskArray, DiskArrays, Unchunked, Chunked, GridChunks +import YAXArrayBase: YAXArrayBase as YAB + +#Add ability to read zipped zarrs +YAB.zarr_zipstore(path::AbstractString) = ZipStore(SimpleFileDiskArray(path)) + +struct SimpleFileDiskArray{C<:Union{Int,Nothing}} <: AbstractDiskArray{UInt8,1} + file::String + s::Int + chunksize::C +end +Base.size(s::SimpleFileDiskArray) = (s.s,) +function SimpleFileDiskArray(filename; chunksize=nothing) + isfile(filename) || throw(ArgumentError("File $filename does not exist")) + s = filesize(filename) + SimpleFileDiskArray(filename, s, chunksize) +end +function DiskArrays.readblock!(a::SimpleFileDiskArray, aout, i::AbstractUnitRange) + open(a.file) do f + seek(f, first(i) - 1) + read!(f, aout) + end +end +DiskArrays.haschunks(a::SimpleFileDiskArray) = a.chunksize === nothing ? Unchunked() : Chunked() +function DiskArrays.eachchunk(a::SimpleFileDiskArray) + if a.chunksize === nothing + DiskArrays.estimate_chunksize(a) + else + GridChunks((a.s,), (a.chunksize,)) + end +end + +end diff --git a/src/datasets/datasetinterface.jl b/src/datasets/datasetinterface.jl index d2a65c2..ccb4353 100644 --- a/src/datasets/datasetinterface.jl +++ b/src/datasets/datasetinterface.jl @@ -15,7 +15,12 @@ function get_var_attrs end function get_global_attrs end #Functions to be implemented for Dataset sinks -"Initialize and return a handle to a new empty dataset" +""" + create_empty(T, path, gatts; kwargs...) + +Initialize and return a handle to a new empty dataset. Backends ignore keyword +arguments they do not support. +""" function create_empty end "Apply a function `f` on a dataset `ds` while keeping possible file handles open during the operations" @@ -56,10 +61,13 @@ function add_var(ds,x::AbstractArray,name,dimlist,atts;kwargs...) end function create_dataset(T::Type, path, gatts, dimnames, dimvals, dimattrs, vartypes, varnames, vardims, varattrs, varchunks; kwargs...) - ds = create_empty(T, path, gatts) + ds = create_empty(T, path, gatts; kwargs...) axlengths = Dict{String, Int}() + # Coordinates share the compression settings (Zarr compressor, NetCDF compress), + # other options like filters may be type specific + coordkwargs = (; (k => v for (k, v) in pairs(kwargs) if k in (:compressor, :compress))...) for (dname, dval, dattr) in zip(dimnames, dimvals, dimattrs) - add_var(ds, dval, dname, (dname,), dattr) + add_var(ds, dval, dname, (dname,), dattr; coordkwargs...) axlengths[dname] = length(dval) end for (T, vn, vd, va, vc) in zip(vartypes, varnames, vardims, varattrs, varchunks) @@ -81,6 +89,14 @@ backendlist = OrderedDict{Symbol, Any}( backendregex = Pair[] +""" + zarr_zipstore(path) + +Open the zipped Zarr store at `path`. Used by the Zarr backend and implemented +when `ZarrZip` is loaded. +""" +zarr_zipstore(path) = throw(ArgumentError("Reading zipped Zarr stores requires ZarrZip. Run `using ZarrZip` (or `using Zarr`) first.")) + function backendfrompath(g::String; driver = :all) if driver == :all isempty(backendregex) && throw("No backend found. Load a backend by using the corresponding package.") diff --git a/test/Project.toml b/test/Project.toml index e845daa..336d296 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -4,6 +4,7 @@ AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" NamedDims = "356022a1-0364-5f58-8944-0da4b18d706f" NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" @@ -11,4 +12,10 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" TestItems = "1c621080-faea-4a02-84b6-bbd5e436b8fe" YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14" -Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" +ZarrBlosc = "5717d1f5-dd42-40cd-8e77-00dfad1079f5" +ZarrCore = "77f5b75c-4c08-499f-ba13-550b0a0af171" +ZarrHTTP = "904a97b2-8abf-4ac8-b211-6540db5c2c8c" +ZarrZip = "54bcf1c8-e61d-4e4f-b375-463581846cca" + +[compat] +ZarrCore = "0.11.1" diff --git a/test/datasets.jl b/test/datasets.jl index f89ae04..c5e4f44 100644 --- a/test/datasets.jl +++ b/test/datasets.jl @@ -3,7 +3,7 @@ using YAXArrayBase, Test @test_throws "No backend found." YAXArrayBase.backendfrompath("test.zarr") end -using NetCDF, Zarr +using NetCDF, ZarrCore using Pkg.Artifacts import Downloads @@ -63,6 +63,43 @@ YAXArrayBase.open_dataset_handle(ds_nc2) do ds_nc end end +@testset "Zarr with bare ZarrCore" begin + path = tempname() * ".zarr" + ds = create_empty(YAXArrayBase.backendlist[:zarr], path) + v = @test_logs (:info, r"uncompressed") add_var(ds, Float32, "tas", (3, 4), ("lon", "lat"), Dict{String,Any}()) + v[:, :] = reshape(1:12, 3, 4) + ds_loaded = to_dataset(path) + @test get_var_dims(ds_loaded, "tas") == ["lon", "lat"] + h = get_var_handle(ds_loaded, "tas") + @test h[:, :] == reshape(1:12, 3, 4) + @test !YAXArrayBase.iscompressed(h) + @test !YAXArrayBase.iscompressed(zcreate(Float32, 3, 4, format=3)) + @test_throws "ZarrZip" to_dataset(tempname() * ".zarr.zip") +end + +using ZarrHTTP, ZarrBlosc, ZarrZip + +@testset "Zarr iscompressed" begin + for format in (2, 3) + @test YAXArrayBase.iscompressed(zcreate(Float32, 3, 4; format)) + @test !YAXArrayBase.iscompressed(zcreate(Float32, 3, 4; format, compressor=ZarrCore.NoCompressor())) + end +end + +@testset "Reading zipped Zarr" begin + path = tempname() * ".zarr" + ds = create_empty(YAXArrayBase.backendlist[:zarr], path) + add_var(ds, reshape(1.0:12.0, 3, 4), "tas", ("lon", "lat"), Dict{String,Any}("units" => "K")) + zippath = path * ".zip" + open(io -> ZarrZip.writezip(io, ds.g), zippath, "w") + ds_zip = to_dataset(zippath) + @test ds_zip isa YAXArrayBase.backendlist[:zarr] + @test get_varnames(ds_zip) == ["tas"] + @test get_var_dims(ds_zip, "tas") == ["lon", "lat"] + @test get_var_attrs(ds_zip, "tas")["units"] == "K" + @test get_var_handle(ds_zip, "tas")[:, :] == reshape(1.0:12.0, 3, 4) +end + @testset "Reading Zarr" begin p = "https://s3.bgc-jena.mpg.de:9000/esdl-esdc-v3.0.2/esdc-16d-2.5deg-46x72x1440-3.0.2.zarr" for ds_zarr in [to_dataset(p,driver=:zarr), to_dataset(zopen(p))] @@ -103,9 +140,9 @@ end @test allow_parallel_write(ds_tif) == false @test allow_missings(ds_tif) == true end -function test_write(T) +function test_write(T; kwargs...) p = tempname() - ds = create_empty(T, p) + ds = create_empty(T, p; kwargs...) add_var(ds, 0.5:1:9.5, "lon", ("lon",), Dict("units"=>"degrees_east")) add_var(ds, 20:-1.0:1, "lat", ("lat",), Dict("units"=>"degrees_north")) v = add_var(ds, Float32, "tas", (10,20), ("lon", "lat"), Dict{String,Any}("units"=>"Celsius")) @@ -136,6 +173,80 @@ end test_write(YAXArrayBase.backendlist[:netcdf]) end -@testset "Writing Zarr" begin - test_write(YAXArrayBase.backendlist[:zarr]) +@testset "create_dataset NetCDF compress" begin + ND = YAXArrayBase.backendlist[:netcdf] + for (compress, compressed) in ((-1, false), (5, true)) + path = tempname() * ".nc" + YAXArrayBase.create_dataset(ND, path, Dict(), ["lon"], [0.5:1:2.5], [Dict()], + [Float32], ["tas"], [["lon"]], [Dict{String,Any}()], [(3,)]; compress) + ds = to_dataset(path, driver=:netcdf) + @test YAXArrayBase.iscompressed(get_var_handle(ds, "tas")) == compressed + @test YAXArrayBase.iscompressed(get_var_handle(ds, "lon")) == compressed + end +end + +@testset "Writing Zarr v$format" for format in (2, 3) + test_write(YAXArrayBase.backendlist[:zarr]; format) +end + +import JSON +@testset "create_dataset Zarr v$format" for format in (2, 3) + ZD = YAXArrayBase.backendlist[:zarr] + path = tempname() * ".zarr" + ds = YAXArrayBase.create_dataset(ZD, path, Dict("title" => "test"), + ["lon", "lat"], [0.5:1:2.5, 10.0:-1:7], [Dict("units" => "degrees_east"), Dict("units" => "degrees_north")], + [Float32, Int], ["tas", "count"], [["lon", "lat"], ["lat"]], [Dict{String,Any}(), Dict{String,Any}()], + [(3, 4), (4,)]; format) + add_var(ds, fill(1), "scalar", (), Dict{String,Any}()) + if format == 3 + nodes = [joinpath(r, f) for (r, _, fs) in walkdir(path) for f in fs if f == "zarr.json"] + @test length(nodes) == 6 + for n in nodes + j = JSON.parsefile(n) + @test j["format"] == 3 + if j["node_type"] == "array" + @test haskey(j, "dimension_names") + @test !haskey(get(j, "attributes", Dict()), "_ARRAY_DIMENSIONS") + end + end + @test !any(f -> f in (".zgroup", ".zarray", ".zattrs"), (f for (_, _, fs) in walkdir(path) for f in fs)) + else + @test isfile(joinpath(path, ".zgroup")) + @test !isfile(joinpath(path, "zarr.json")) + @test ds["tas"].attrs["_ARRAY_DIMENSIONS"] == ["lat", "lon"] + @test ZarrCore.dimension_names(ds["tas"]) === nothing + end + ds2 = to_dataset(path) + @test get_var_dims(ds2, "tas") == ["lon", "lat"] + @test get_var_dims(ds2, "count") == ["lat"] + @test get_var_dims(ds2, "lon") == ["lon"] + @test get_var_dims(ds2, "scalar") == [] + @test YAXArrayBase.get_global_attrs(ds2)["title"] == "test" + @test_throws ArgumentError add_var(ds, Float32, "bad", (3,), ("lon",), Dict{String,Any}(); format=5 - format) + @test all(n -> YAXArrayBase.iscompressed(ds2[n]), ["tas", "lon", "lat"]) + + # The compressor also applies to the coordinate arrays + path = tempname() * ".zarr" + ds = YAXArrayBase.create_dataset(ZD, path, Dict(), ["lon"], [0.5:1:2.5], [Dict()], + [Float32], ["tas"], [["lon"]], [Dict{String,Any}()], [(3,)]; + format, compressor=ZarrCore.NoCompressor()) + @test !YAXArrayBase.iscompressed(ds["tas"]) + @test !YAXArrayBase.iscompressed(ds["lon"]) +end + +@testset "Reading Zarr v3 with only dimension_names" begin + path = tempname() * ".zarr" + g = zgroup(ZarrCore.storefromstring(path, true)..., 3) + zcreate(Float32, g, "a", 3, 4; dimension_names=("x", "y")) + zcreate(Float32, g, "partial", 3, 4; dimension_names=("x", nothing), attrs=Dict("_ARRAY_DIMENSIONS" => ["y", "x"])) + zcreate(Float32, g, "unnamed", 3, 4) + scalar = zcreate(Float64, g, "scalar") + scalar[] = 2.5 + ds = to_dataset(path) + @test get_var_dims(ds, "a") == ["x", "y"] + @test get_var_dims(ds, "partial") == ["x", "y"] + @test_throws ArgumentError get_var_dims(ds, "unnamed") + @test !occursin("dimension_names", read(joinpath(path, "scalar", "zarr.json"), String)) + @test get_var_dims(ds, "scalar") == [] + @test get_var_handle(ds, "scalar")[] == 2.5 end