Add size, ndims, length, etc methods to Dataspace (#758)

* Also `copy`, `==`, `hash`, `isempty,` and `isnull`.
* Merge `size`-like methods for `Dataspace` and `Union{Dataset,Attribute}`
* Start systematically testing `Dataspace` methods
* Close temporary dataspace in get_dims

Author: jmert

docs/src/api_bindings.md

@@ -193,6 +193,7 @@ h5s_combine_select(space1_id::hid_t, op::Cint, space2_id::hid_t)
 h5s_copy(space_id::hid_t)
 h5s_create(class::Cint)
 h5s_create_simple(rank::Cint, current_dims::Ptr{hsize_t}, maximum_dims::Ptr{hsize_t})
+h5s_extent_equal(space1_id::hid_t, space2_id::hid_t)
 h5s_get_regular_hyperslab(space_id::hid_t, start::Ptr{hsize_t}, stride::Ptr{hsize_t}, count::Ptr{hsize_t}, block::Ptr{hsize_t})
 h5s_get_select_hyper_nblocks(space_id::hid_t)
 h5s_get_select_npoints(space_id::hid_t)

gen/api_defs.jl

@@ -203,6 +203,7 @@
 @bind h5s_copy(space_id::hid_t)::hid_t "Error copying dataspace"
 @bind h5s_create(class::Cint)::hid_t "Error creating dataspace"
 @bind h5s_create_simple(rank::Cint, current_dims::Ptr{hsize_t}, maximum_dims::Ptr{hsize_t})::hid_t "Error creating simple dataspace"
+@bind h5s_extent_equal(space1_id::hid_t, space2_id::hid_t)::htri_t "Error comparing dataspaces"
 @bind h5s_get_regular_hyperslab(space_id::hid_t, start::Ptr{hsize_t}, stride::Ptr{hsize_t}, count::Ptr{hsize_t}, block::Ptr{hsize_t})::herr_t "Error getting regular hyperslab selection"
 @bind h5s_get_simple_extent_dims(space_id::hid_t, dims::Ptr{hsize_t}, maxdims::Ptr{hsize_t})::Cint "Error getting the dimensions for a dataspace"
 @bind h5s_get_simple_extent_ndims(space_id::hid_t)::Cint "Error getting the number of dimensions for a dataspace"

src/HDF5.jl

@@ -264,6 +264,9 @@ mutable struct Dataspace
     end
 end
 Base.cconvert(::Type{hid_t}, dspace::Dataspace) = dspace.id
+Base.:(==)(dspace1::Dataspace, dspace2::Dataspace) = h5s_extent_equal(checkvalid(dspace1), checkvalid(dspace2))
+Base.hash(dspace::Dataspace, h::UInt) = hash(dspace.id, hash(Dataspace, h))
+Base.copy(dspace::Dataspace) = Dataspace(h5s_copy(checkvalid(dspace)))
 
 mutable struct Attribute
     id::hid_t
@@ -963,23 +966,8 @@ object_info(obj::Union{File,Object}) = h5o_get_info(checkvalid(obj))
 Base.length(obj::Union{Group,File}) = h5g_get_num_objs(checkvalid(obj))
 Base.length(x::Attributes) = object_info(x.parent).num_attrs
 
-Base.isempty(x::Union{Dataset,Group,File}) = length(x) == 0
-function Base.size(obj::Union{Dataset,Attribute})
-    dspace = dataspace(obj)
-    dims, maxdims = get_dims(dspace)
-    close(dspace)
-    return dims
-end
-Base.size(dset::Union{Dataset,Attribute}, d) = d > ndims(dset) ? 1 : size(dset)[d]
-Base.length(dset::Union{Dataset,Attribute}) = prod(size(dset))
-Base.ndims(dset::Union{Dataset,Attribute}) = length(size(dset))
+Base.isempty(x::Union{Group,File}) = length(x) == 0
 Base.eltype(dset::Union{Dataset,Attribute}) = get_jl_type(dset)
-function isnull(obj::Union{Dataset,Attribute})
-    dspace = dataspace(obj)
-    ret = h5s_get_simple_extent_type(dspace) == H5S_NULL
-    close(dspace)
-    ret
-end
 
 # filename and name
 filename(obj::Union{File,Group,Dataset,Attribute,Datatype}) = h5f_get_name(checkvalid(obj))
@@ -1094,6 +1082,61 @@ dataspace(sz::Dims{N}; max_dims::Union{Dims{N},Tuple{}}=()) where {N} = _dataspa
 dataspace(sz1::Int, sz2::Int, sz3::Int...; max_dims::Union{Dims,Tuple{}}=()) = _dataspace(tuple(sz1, sz2, sz3...), max_dims)
 
 
+function Base.ndims(obj::Union{Dataspace,Dataset,Attribute})
+    dspace = obj isa Dataspace ? checkvalid(obj) : dataspace(obj)
+    ret = Int(h5s_get_simple_extent_ndims(dspace))
+    obj isa Dataspace || close(dspace)
+    return ret
+end
+function Base.size(obj::Union{Dataspace,Dataset,Attribute})
+    dspace = obj isa Dataspace ? checkvalid(obj) : dataspace(obj)
+    h5_dims = h5s_get_simple_extent_dims(dspace, nothing)
+    N = length(h5_dims)
+    ret = ntuple(i -> @inbounds(Int(h5_dims[N-i+1])), N)
+    obj isa Dataspace || close(dspace)
+    return ret
+end
+function Base.size(obj::Union{Dataspace,Dataset,Attribute}, d::Integer)
+    d > 0 || throw(ArgumentError("invalid dimension d; must be positive integer"))
+    N = ndims(obj)
+    d > N && return 1
+    dspace = obj isa Dataspace ? obj : dataspace(obj)
+    h5_dims = h5s_get_simple_extent_dims(dspace, nothing)
+    ret = @inbounds Int(h5_dims[N - d + 1])
+    obj isa Dataspace || close(dspace)
+    return ret
+end
+function Base.length(obj::Union{Dataspace,Dataset,Attribute})
+    isnull(obj) && return 0
+    dspace = obj isa Dataspace ? obj : dataspace(obj)
+    h5_dims = h5s_get_simple_extent_dims(dspace, nothing)
+    ret = Int(prod(h5_dims))
+    obj isa Dataspace || close(dspace)
+    return ret
+end
+Base.isempty(dspace::Union{Dataspace,Dataset,Attribute}) = length(dspace) == 0
+
+"""
+    isnull(dspace::Union{Dataspace, Dataset, Attribute})
+
+Determines whether the given object has no size (consistent with the `H5S_NULL` dataspace).
+
+# Examples
+```julia-repl
+julia> HDF5.isnull(dataspace(HDF5.EmptyArray{Float64}()))
+true
+
+julia> HDF5.isnull(dataspace((0,)))
+false
+```
+"""
+function isnull(obj::Union{Dataspace,Dataset,Attribute})
+    dspace = obj isa Dataspace ? checkvalid(obj) : dataspace(obj)
+    ret = h5s_get_simple_extent_type(dspace) == H5S_NULL
+    obj isa Dataspace || close(dspace)
+    return ret
+end
+
 function get_dims(dspace::Dataspace)
     h5_dims, h5_maxdims = h5s_get_simple_extent_dims(dspace)
     # reverse dimensions since hdf5 uses C-style order
@@ -1111,11 +1154,16 @@ function get_regular_hyperslab(dspace::Dataspace)
 end
 
 """
-    get_dims(dset::HDF5.Dataset)
+    HDF5.get_dims(obj::Union{HDF5.Dataset, HDF5.Attribute})
 
-Get the array dimensions from a dataset and return a tuple of dims and maxdims.
+Get the array dimensions from a dataset or attribute and return a tuple of dims and maxdims.
 """
-get_dims(dset::Dataset) = get_dims(dataspace(checkvalid(dset)))
+function get_dims(dset::Union{Dataset,Attribute})
+    dspace = dataspace(dset)
+    ret = get_dims(dspace)
+    close(dspace)
+    return ret
+end
 
 """
     set_dims!(dset::HDF5.Dataset, new_dims::Dims)
@@ -1222,7 +1270,7 @@ function Base.read(obj::DatasetOrAttribute, ::Type{T}, I...) where T
         sz = (1,)
         scalar = true
     elseif isempty(I)
-        sz, _ = get_dims(dspace)
+        sz = size(dspace)
     else
         sz = map(length, filter(i -> !isa(i, Int), indices))
         if isempty(sz)
@@ -1376,7 +1424,7 @@ function readmmap(obj::Dataset, ::Type{T}) where {T}
     dspace = dataspace(obj)
     stype = h5s_get_simple_extent_type(dspace)
     (stype != H5S_SIMPLE) && error("can only mmap simple dataspaces")
-    dims, _ = get_dims(dspace)
+    dims = size(dspace)
 
     if isempty(dims)
         return T[]
@@ -1596,7 +1644,7 @@ end
 function hyperslab(dspace::Dataspace, I::Union{AbstractRange{Int},Int}...)
     local dsel_id
     try
-        dims, maxdims = get_dims(dspace)
+        dims = size(dspace)
         n_dims = length(dims)
         if length(I) != n_dims
             error("Wrong number of indices supplied, supplied length $(length(I)) but expected $(n_dims).")

src/api.jl

@@ -782,6 +782,12 @@ function h5s_create_simple(rank, current_dims, maximum_dims)
     return var"#status#"
 end
 
+function h5s_extent_equal(space1_id, space2_id)
+    var"#status#" = ccall((:H5Sextent_equal, libhdf5), htri_t, (hid_t, hid_t), space1_id, space2_id)
+    var"#status#" < 0 && error("Error comparing dataspaces")
+    return var"#status#" > 0
+end
+
 function h5s_get_regular_hyperslab(space_id, start, stride, count, block)
     var"#status#" = ccall((:H5Sget_regular_hyperslab, libhdf5), herr_t, (hid_t, Ptr{hsize_t}, Ptr{hsize_t}, Ptr{hsize_t}, Ptr{hsize_t}), space_id, start, stride, count, block)
     var"#status#" < 0 && error("Error getting regular hyperslab selection")

src/api_helpers.jl

@@ -254,6 +254,12 @@ function h5s_get_simple_extent_dims(space_id)
     h5s_get_simple_extent_dims(space_id, dims, maxdims)
     return dims, maxdims
 end
+function h5s_get_simple_extent_dims(space_id, ::Nothing)
+    n = h5s_get_simple_extent_ndims(space_id)
+    dims = Vector{hsize_t}(undef, n)
+    h5s_get_simple_extent_dims(space_id, dims, C_NULL)
+    return dims
+end
 
 
 ###

test/dataspace.jl

@@ -0,0 +1,137 @@
+@testset "Dataspaces" begin
+    hsize_t = HDF5.hsize_t
+    # Reference objects without using high-level API
+    ds_null   = HDF5.Dataspace(HDF5.h5s_create(HDF5.H5S_NULL))
+    ds_scalar = HDF5.Dataspace(HDF5.h5s_create(HDF5.H5S_SCALAR))
+    ds_zerosz = HDF5.Dataspace(HDF5.h5s_create_simple(1, hsize_t[0], hsize_t[0]))
+    ds_vector = HDF5.Dataspace(HDF5.h5s_create_simple(1, hsize_t[5], hsize_t[5]))
+    ds_matrix = HDF5.Dataspace(HDF5.h5s_create_simple(2, hsize_t[7, 5], hsize_t[7, 5]))
+    ds_maxdim = HDF5.Dataspace(HDF5.h5s_create_simple(2, hsize_t[7, 5], hsize_t[20, 20]))
+    ds_unlim  = HDF5.Dataspace(HDF5.h5s_create_simple(1, hsize_t[1], [HDF5.H5S_UNLIMITED]))
+
+    # Testing basic property accessors of dataspaces
+
+    @test isvalid(ds_scalar)
+
+    @test ndims(ds_null)   === 0
+    @test ndims(ds_scalar) === 0
+    @test ndims(ds_zerosz) === 1
+    @test ndims(ds_vector) === 1
+    @test ndims(ds_matrix) === 2
+
+    # Test that properties of existing datasets can be extracted.
+    # Note: Julia reverses the order of dimensions when using the high-level API versus
+    #       the dimensions used above to create the reference objects.
+    @test size(ds_null)   === ()
+    @test size(ds_scalar) === ()
+    @test size(ds_zerosz) === (0,)
+    @test size(ds_vector) === (5,)
+    @test size(ds_matrix) === (5, 7)
+    @test size(ds_maxdim) === (5, 7)
+
+    @test size(ds_null, 5)   === 1
+    @test size(ds_scalar, 5) === 1
+    @test size(ds_zerosz, 5) === 1
+    @test size(ds_vector, 5) === 1
+    @test size(ds_matrix, 5) === 1
+    @test size(ds_maxdim, 5) === 1
+    @test_throws ArgumentError("invalid dimension d; must be positive integer") size(ds_null, 0)
+    @test_throws ArgumentError("invalid dimension d; must be positive integer") size(ds_scalar, -1)
+
+    @test length(ds_null)   === 0
+    @test length(ds_scalar) === 1
+    @test length(ds_zerosz) === 0
+    @test length(ds_vector) === 5
+    @test length(ds_matrix) === 35
+    @test length(ds_maxdim) === 35
+
+    @test isempty(ds_null)
+    @test !isempty(ds_scalar)
+    @test isempty(ds_zerosz)
+    @test !isempty(ds_vector)
+
+    @test HDF5.isnull(ds_null)
+    @test !HDF5.isnull(ds_scalar)
+    @test !HDF5.isnull(ds_zerosz)
+    @test !HDF5.isnull(ds_vector)
+
+    @test HDF5.get_dims(ds_null)   === ((), ())
+    @test HDF5.get_dims(ds_scalar) === ((), ())
+    @test HDF5.get_dims(ds_zerosz) === ((0,), (0,))
+    @test HDF5.get_dims(ds_vector) === ((5,), (5,))
+    @test HDF5.get_dims(ds_matrix) === ((5, 7), (5, 7))
+    @test HDF5.get_dims(ds_maxdim) === ((5, 7), (20, 20))
+    @test HDF5.get_dims(ds_unlim)  === ((1,), (-1,))
+
+    # Can create new copies
+    ds_tmp  = copy(ds_maxdim)
+    ds_tmp2 = HDF5.Dataspace(ds_tmp.id) # copy of ID, but new Julia object
+    @test ds_tmp.id == ds_tmp2.id != ds_maxdim.id
+    # Equality and hashing
+    @test ds_tmp  == ds_maxdim
+    @test ds_tmp !== ds_maxdim
+    @test hash(ds_tmp) != hash(ds_maxdim)
+    @test ds_tmp  == ds_tmp2
+    @test ds_tmp !== ds_tmp2
+    @test hash(ds_tmp) == hash(ds_tmp2)
+
+    # Behavior of closing dataspace objects
+    close(ds_tmp)
+    @test ds_tmp.id == -1
+    @test !isvalid(ds_tmp)
+    @test !isvalid(ds_tmp2)
+
+    # Validity checking in high-level operations
+    @test_throws ErrorException("File or object has been closed") copy(ds_tmp)
+    @test_throws ErrorException("File or object has been closed") ndims(ds_tmp)
+    @test_throws ErrorException("File or object has been closed") size(ds_tmp)
+    @test_throws ErrorException("File or object has been closed") size(ds_tmp, 1)
+    @test_throws ErrorException("File or object has been closed") length(ds_tmp)
+    @test_throws ErrorException("File or object has been closed") ds_tmp == ds_tmp2
+    @test close(ds_tmp) === nothing # no error
+
+    # Test ability to create explicitly-sized dataspaces
+
+    @test dataspace(()) == ds_scalar
+    @test dataspace((5,)) == ds_vector
+    @test dataspace((5, 7)) == ds_matrix != ds_maxdim
+    @test dataspace((5, 7), max_dims = (20, 20)) == ds_maxdim != ds_matrix
+    @test dataspace((1,), max_dims = (-1,)) == ds_unlim
+    # for ≥ 2 numbers, same as single tuple argument
+    @test dataspace(5, 7) == ds_matrix
+    @test dataspace(5, 7, 1) == dataspace((5, 7, 1))
+
+    # Test dataspaces derived from data
+
+    @test dataspace(nothing) == ds_null
+    @test dataspace(HDF5.EmptyArray{Bool}()) == ds_null
+
+    @test dataspace(fill(1.0)) == ds_scalar
+    @test dataspace(1) == ds_scalar
+    @test dataspace(1 + 1im) == ds_scalar
+    @test dataspace("string") == ds_scalar
+
+    @test dataspace(zeros(0)) == ds_zerosz
+    @test dataspace(zeros(0, 0)) != ds_zerosz
+    @test dataspace(zeros(5, 7)) == ds_matrix
+    @test dataspace(HDF5.VLen([[1]])) == dataspace((1,))
+    @test dataspace(HDF5.VLen([[1], [2]])) == dataspace((2,))
+
+    # Constructing dataspace for/from HDF5 dataset or attribute
+
+    mktemp() do path, io
+        close(io)
+        h5open(path, "w") do hid
+            dset = create_dataset(hid, "dset", datatype(Int), ds_matrix)
+            attr = create_attribute(dset, "attr", datatype(Bool), ds_vector)
+            @test dataspace(dset)  == ds_matrix
+            @test dataspace(dset) !== ds_matrix
+            @test dataspace(attr)  == ds_vector
+            @test dataspace(attr) !== ds_vector
+            close(dset)
+            close(attr)
+            @test_throws ErrorException("File or object has been closed") dataspace(dset)
+            @test_throws ErrorException("File or object has been closed") dataspace(attr)
+        end
+    end
+end

test/runtests.jl

@@ -10,6 +10,7 @@ include("plain.jl")
 include("compound.jl")
 include("custom.jl")
 include("reference.jl")
+include("dataspace.jl")
 include("hyperslab.jl")
 include("readremote.jl")
 include("extend_test.jl")