Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for reading H5T_ARRAY #55

Closed
wants to merge 1 commit into from
Closed

Add support for reading H5T_ARRAY #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions src/fixedarrays.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Create a type that encodes array dimensions as parameters.
# This is an intermediate needed to encode the H5T_ARRAY types.
# The data are eventually read as plain Arrays.

abstract FixedArray{T,N} <: AbstractArray{T,N}

import Base.size

function fixedarray_definetype(N::Int)
sz = [symbol(string("n", i)) for i = 1:N]
typename = symbol(string("FixedArray", N))
extype = Expr(:curly, typename, :T, sz...)
excreate = Expr(:type, false, Expr(:<:, extype, :(FixedArray{T, $N})), quote end)
end

const fixedarray_isdefined = [false]
const fixedarray_dict = Dict{(DataType,Dims),DataType}()
function fixedarray_type(T::DataType, sz::Dims)
t = (T,sz)
if haskey(fixedarray_dict, t)
return fixedarray_dict[t]
end
N = length(sz)
if N > length(fixedarray_isdefined)
append!(fixedarray_isdefined, falses(N-length(fixedarray_isdefined)))
end
if !fixedarray_isdefined[N]
eval(fixedarray_definetype(N))
fixedarray_isdefined[N] = true
end
typename = symbol(string("FixedArray", N))
ret = eval(Expr(:curly, typename, T, sz...))
fixedarray_dict[t] = ret
ret
end

size{T<:FixedArray}(::Type{T}) = T.parameters[2:end]
49 changes: 45 additions & 4 deletions src/plain.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@

module HDF5

include("datafile.jl")

## Add methods to...
import Base: close, convert, done, dump, endof, flush, getindex, has, isempty, isvalid, length, names, ndims, next, read, setindex!, show, size, start, write

include("datafile.jl")
include("fixedarrays.jl")

## C types
typealias C_time_t Int

Expand Down Expand Up @@ -965,9 +966,7 @@ function read{T<:HDF5BitsKind}(obj::DatasetOrAttribute, ::Type{T})
end
# Read array of BitsKind
function read{T<:HDF5BitsKind}(obj::DatasetOrAttribute, ::Type{Array{T}})
local data
dims = size(obj)
dspace = dataspace(obj)
data = Array(T, dims...)
readarray(obj, hdf5_type_id(T), data)
data
Expand All @@ -976,6 +975,34 @@ end
function read{T<:BitsKindOrByteString}(obj::DatasetOrAttribute, ::Type{EmptyArray{T}})
Array(T, 0)
end
# Fixed-size arrays (H5T_ARRAY)
function read{A<:FixedArray}(obj::DatasetOrAttribute, ::Type{A})
T = eltype(A)
sz = size(A)
data = Array(T, sz)
readarray(obj, hdf5_type_id(T), data)
data
end
function read{A<:FixedArray}(obj::DatasetOrAttribute, ::Type{Array{A}})
T = eltype(A)
sz = size(A)
dims = size(obj)
dataptr = Base.c_malloc(prod(sz)*prod(dims)*sizeof(T))
nd = length(sz)
hsz = Hsize[convert(Hsize,sz[nd-i+1]) for i = 1:nd]
memtype_id = h5t_array_create(hdf5_type_id(T), convert(Cuint, length(sz)), hsz)
try
h5d_read(obj.id, memtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, dataptr)
finally
h5t_close(memtype_id)
end
ret = Array(Array{T}, dims)
L = prod(sz)
for i = 1:prod(dims)
ret[i] = pointer_to_array(convert(Ptr{T},dataptr+L*sizeof(T)*(i-1)), sz, i == 1)
end
ret
end

# Clean up string buffer according to padding mode
function unpad(s::ByteString, pad::Cint)
Expand Down Expand Up @@ -1496,6 +1523,8 @@ function hdf5_to_julia_eltype(objtype)
T = HDF5Vlen{hdf5_to_julia_eltype(HDF5Datatype(super_id))}
elseif class_id == H5T_COMPOUND
T = HDF5Compound
elseif class_id == H5T_ARRAY
T = hdf5array(objtype)
else
error("Class id ", class_id, " is not yet supported")
end
Expand Down Expand Up @@ -1723,8 +1752,11 @@ for (jlname, h5name, outtype, argtypes, argsyms, ex_error) in
(:h5s_get_simple_extent_dims, :H5Sget_simple_extent_dims, Cint, (Hid, Ptr{Hsize}, Ptr{Hsize}), (:space_id, :dims, :maxdims), :(error("Error getting the dimensions for a dataspace"))),
(:h5s_get_simple_extent_ndims, :H5Sget_simple_extent_ndims, Cint, (Hid,), (:space_id,), :(error("Error getting the number of dimensions for a dataspace"))),
(:h5s_get_simple_extent_type, :H5Sget_simple_extent_type, Cint, (Hid,), (:space_id,), :(error("Error getting the dataspace type"))),
(:h5t_array_create, :H5Tarray_create2, Hid, (Hid, Cuint, Ptr{Hsize}), (:basetype_id, :ndims, :sz), :(error("Error creating H5T_ARRAY of id ", basetype_id, " and size ", sz))),
(:h5t_copy, :H5Tcopy, Hid, (Hid,), (:dtype_id,), :(error("Error copying datatype"))),
(:h5t_create, :H5Tcreate, Hid, (Cint, Csize_t), (:class_id, :sz), :(error("Error creating datatype of id ", classid))),
(:h5t_get_array_dims, :H5Tget_array_dims2, Cint, (Hid, Ptr{Hsize}), (:dtype_id, :dims), :(error("Error getting dimensions of array"))),
(:h5t_get_array_ndims, :H5Tget_array_ndims, Cint, (Hid,), (:dtype_id,), :(error("Error getting ndims of array"))),
(:h5t_get_class, :H5Tget_class, Cint, (Hid,), (:dtype_id,), :(error("Error getting class"))),
(:h5t_get_cset, :H5Tget_cset, Cint, (Hid,), (:dtype_id,), :(error("Error getting character set encoding"))),
(:h5t_get_member_class, :H5Tget_member_class, Cint, (Hid, Cuint), (:dtype_id, :index), :(error("Error getting class of compound datatype member #", index))),
Expand Down Expand Up @@ -1854,6 +1886,15 @@ function vlen_get_buf_size(dset::HDF5Dataset, dtype::HDF5Datatype, dspace::HDF5D
sz[1]
end

function hdf5array(objtype)
nd = h5t_get_array_ndims(objtype.id)
dims = Array(Hsize, nd)
h5t_get_array_dims(objtype.id, dims)
eltyp = HDF5Datatype(h5t_get_super(objtype.id))
T = hdf5_to_julia_eltype(eltyp)
fixedarray_type(T, ntuple(nd, i->int(dims[nd-i+1])))
end

### Property manipulation ###
get_create_properties(dset::HDF5Dataset) = HDF5Properties(h5d_get_create_plist(dset.id))
get_create_properties(g::HDF5Group) = HDF5Properties(h5g_get_create_plist(dset.id))
Expand Down
29 changes: 29 additions & 0 deletions test/readremote.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,29 @@ opq = Array{Uint8}[[0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x30],
[0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x31],
[0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x32],
[0x4f, 0x50, 0x41, 0x51, 0x55, 0x45, 0x33]]
# For H5T_ARRAY
AA = Array{Int,2}[
[ 0 0 0;
0 -1 -2;
0 -2 -4;
0 -3 -6;
0 -4 -8],
[ 0 1 2;
1 1 1;
2 1 0;
3 1 -1;
4 1 -2],
[ 0 2 4;
2 3 4;
4 4 4;
6 5 4;
8 6 4],
[ 0 3 6;
3 5 7;
6 7 8;
9 9 9;
12 11 10]]


const savedir = joinpath(tempdir(), "h5")
if !isdir(savedir)
Expand Down Expand Up @@ -132,3 +155,9 @@ d = read(fid, "DS1")
@assert d.tag == "Character array"
@assert d.data == opq
close(fid)

file = getfile("h5ex_t_array.h5")
fid = h5open(file, "r")
A = read(fid, "DS1")
@assert A == AA
close(fid)