From a7903304518553177c741fb6af6fdb4896aa3e95 Mon Sep 17 00:00:00 2001 From: Tobias Knopp Date: Wed, 17 Apr 2019 22:43:20 +0200 Subject: [PATCH 1/3] get explicite filedescriptor on windows --- src/HDF5.jl | 25 +++++++++++++++---------- test/runtests.jl | 4 ++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/HDF5.jl b/src/HDF5.jl index 55c98e274..5b0bfec13 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -1598,14 +1598,19 @@ function readmmap(obj::HDF5Dataset, ::Type{Array{T}}) where {T} if isempty(dims) return T[] end - local fd - prop = h5d_get_access_plist(checkvalid(obj).id) - try - ret = Ptr{Cint}[0] - h5f_get_vfd_handle(obj.file.id, prop, ret) - fd = unsafe_load(ret[1]) - finally - HDF5.h5p_close(prop) + if !Sys.iswindows() + local fdint + prop = h5d_get_access_plist(checkvalid(obj).id) + try + ret = Ptr{Cint}[0] + h5f_get_vfd_handle(obj.file.id, prop, ret) + fdint = unsafe_load(ret[1]) + finally + HDF5.h5p_close(prop) + end + fd = fdio(fdint) + else + fd = open(obj.file.filename,"r") # which acces rights? end offset = h5d_get_offset(obj.id) @@ -1613,9 +1618,9 @@ function readmmap(obj::HDF5Dataset, ::Type{Array{T}}) where {T} error("Error mmapping array") end if offset % Base.datatype_alignment(T) == 0 - return Mmap.mmap(fdio(fd), Array{T,length(dims)}, dims, offset) + return Mmap.mmap(fd, Array{T,length(dims)}, dims, offset) else - A = Mmap.mmap(fdio(fd), Array{UInt8,1}, prod(dims)*sizeof(T), offset) + A = Mmap.mmap(fd, Array{UInt8,1}, prod(dims)*sizeof(T), offset) return reshape(reinterpret(T,A),dims) end end diff --git a/test/runtests.jl b/test/runtests.jl index 16239776b..5b5c34171 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,9 +10,9 @@ include("extend_test.jl") include("gc.jl") include("external.jl") include("swmr.jl") -if !Sys.iswindows() # Mmap needs to be fixed on windows +#if !Sys.iswindows() # Mmap needs to be fixed on windows include("mmap.jl") -end +#end if get(Pkg.installed(), "MPI", nothing) !== nothing # basic MPI tests, for actual parallel tests we need to run in MPI mode include("mpio.jl") From f47bd65c7c7e4c694cf5c01aadbdc0d3d84ce1af Mon Sep 17 00:00:00 2001 From: Tobias Knopp Date: Thu, 18 Apr 2019 01:45:30 +0200 Subject: [PATCH 2/3] get permissions right --- src/HDF5.jl | 26 ++++++++++++++++++++++---- test/mmap.jl | 17 ++++++++++++----- test/runtests.jl | 4 +--- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/HDF5.jl b/src/HDF5.jl index 5b0bfec13..e55d9866a 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -1610,7 +1610,18 @@ function readmmap(obj::HDF5Dataset, ::Type{Array{T}}) where {T} end fd = fdio(fdint) else - fd = open(obj.file.filename,"r") # which acces rights? + # This is a workaround since h5f_get_vfd_handle gives a wrong result on + # windows. We therefore open a new file handle. + + # Check permissions + intent = Ref{Cuint}() + h5f_get_intend(obj.file, intent) + if intent[] == HDF5.H5F_ACC_RDONLY || intent[] == HDF5.H5F_ACC_RDONLY + flag = "r" + else + flag = "r+" + end + fd = open(obj.file.filename, flag) end offset = h5d_get_offset(obj.id) @@ -1618,11 +1629,17 @@ function readmmap(obj::HDF5Dataset, ::Type{Array{T}}) where {T} error("Error mmapping array") end if offset % Base.datatype_alignment(T) == 0 - return Mmap.mmap(fd, Array{T,length(dims)}, dims, offset) + A = Mmap.mmap(fd, Array{T,length(dims)}, dims, offset) else - A = Mmap.mmap(fd, Array{UInt8,1}, prod(dims)*sizeof(T), offset) - return reshape(reinterpret(T,A),dims) + Aflat = Mmap.mmap(fd, Array{UInt8,1}, prod(dims)*sizeof(T), offset) + A = reshape(reinterpret(T, Aflat), dims) end + + if Sys.iswindows() + close(fd) + end + + return A end function readmmap(obj::HDF5Dataset) @@ -2103,6 +2120,7 @@ for (jlname, h5name, outtype, argtypes, argsyms, msg) in (:h5f_flush, :H5Fflush, Herr, (Hid, Cint), (:object_id, :scope,), "Error flushing object to file"), (:hf5start_swmr_write, :H5Fstart_swmr_write, Herr, (Hid,), (:id,), "Error starting SWMR write"), (:h5f_get_vfd_handle, :H5Fget_vfd_handle, Herr, (Hid, Hid, Ptr{Ptr{Cint}}), (:file_id, :fapl_id, :file_handle), "Error getting VFD handle"), + (:h5f_get_intend, :H5Fget_intent, Herr, (Hid, Ptr{Cuint}), (:file_id, :intent), "Error getting file intent"), (:h5g_close, :H5Gclose, Herr, (Hid,), (:group_id,), "Error closing group"), (:h5g_get_info, :H5Gget_info, Herr, (Hid, Ptr{H5Ginfo}), (:group_id, :buf), "Error getting group info"), (:h5o_get_info, :H5Oget_info1, Herr, (Hid, Ptr{H5Oinfo}), (:object_id, :buf), "Error getting object info"), diff --git a/test/mmap.jl b/test/mmap.jl index f68e2372f..66d27b79c 100644 --- a/test/mmap.jl +++ b/test/mmap.jl @@ -12,12 +12,19 @@ f = h5open(fn, "w") hdf5_A = d_create(f,"A",datatype(Int64),dataspace(3,3)); A = rand(Int64,3,3) hdf5_A[:,:] = A -flush(f); -close(f); +flush(f) +close(f) # Read HDF5 file & MMAP -f = h5open(fn,"r"); -A_mmaped = readmmap(f["A"]); - +f = h5open(fn,"r") +A_mmaped = readmmap(f["A"]) @test all(A .== A_mmaped) +# Check that it is read only +@test_throws ReadOnlyMemoryError A_mmaped[1,1] = 33 +close(f) +# Now check if we can write +f = h5open(fn,"r+") +A_mmaped = readmmap(f["A"]) +A_mmaped[1,1] = 33 +close(f) end diff --git a/test/runtests.jl b/test/runtests.jl index 5b5c34171..ac17d2e00 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,9 +10,7 @@ include("extend_test.jl") include("gc.jl") include("external.jl") include("swmr.jl") -#if !Sys.iswindows() # Mmap needs to be fixed on windows - include("mmap.jl") -#end +include("mmap.jl") if get(Pkg.installed(), "MPI", nothing) !== nothing # basic MPI tests, for actual parallel tests we need to run in MPI mode include("mpio.jl") From 42f21a64b7c172eb3e0907ea2dff6db26e36feb6 Mon Sep 17 00:00:00 2001 From: Tobias Knopp Date: Tue, 23 Apr 2019 10:55:16 +0200 Subject: [PATCH 3/3] improve documentation on mmap windows issue --- src/HDF5.jl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/HDF5.jl b/src/HDF5.jl index e55d9866a..da5b37615 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -1610,8 +1610,18 @@ function readmmap(obj::HDF5Dataset, ::Type{Array{T}}) where {T} end fd = fdio(fdint) else - # This is a workaround since h5f_get_vfd_handle gives a wrong result on - # windows. We therefore open a new file handle. + # This is a workaround since the regular code path does not work on windows + # (see #89 for background). The error is that "Mmap.mmap(fd, ...)" cannot + # create create a valid file mapping. The question is if the handler + # returned by "h5f_get_vfd_handle" has + # the correct format as required by the "fdio" function. The former + # calls + # https://gitlabext.iag.uni-stuttgart.de/libs/hdf5/blob/develop/src/H5FDcore.c#L1209 + # + # The workaround is to create a new file handle, which should actually + # not make any problems. Since we need to know the permissions of the + # original file handle, we first retrieve them using the "h5f_get_intend" + # function # Check permissions intent = Ref{Cuint}()