From e77a4562afcd8ed5210b00aff4779754543d3ab5 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 20 Apr 2021 13:28:40 +0100 Subject: [PATCH 1/8] try saving interpolated stats every time we save --- src/Output.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Output.jl b/src/Output.jl index f7061aa9..cf7b5fa3 100644 --- a/src/Output.jl +++ b/src/Output.jl @@ -306,7 +306,6 @@ end function (o::HDF5Output)(y, t, dt, yfun) o.readonly && error("Cannot add data to read-only output!") save, ts = o.save_cond(y, t, dt, o.saved) - push!(o.stats_tmp, o.statsfun(y, t, dt)) if save @hlock HDF5.h5open(o.fpath, "r+") do file !HDF5.exists(file, o.yname) && initialise(o, y) @@ -321,7 +320,9 @@ function (o::HDF5Output)(y, t, dt, yfun) s[end] += 1 HDF5.set_dims!(file[o.yname], Tuple(s)) end - file[o.yname][idcs..., o.saved+1] = yfun(ts) + ys = yfun(ts) + push!(o.stats_tmp, o.statsfun(ys, ts, dt)) + file[o.yname][idcs..., o.saved+1] = ys s = collect(size(file[o.tname])) if s[end] < o.saved+1 s[end] += 1 @@ -341,6 +342,7 @@ function (o::HDF5Output)(y, t, dt, yfun) end end end + push!(o.stats_tmp, o.statsfun(y, t, dt)) end function append_stats!(parent, a::Array{Dict{String,Any},1}) From 9403d375b822696812c60254f2cc1d13d69768f3 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 20 Apr 2021 13:34:59 +0100 Subject: [PATCH 2/8] Add interpolated stats for Memory Output too --- src/Output.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Output.jl b/src/Output.jl index cf7b5fa3..28fc61da 100644 --- a/src/Output.jl +++ b/src/Output.jl @@ -61,21 +61,23 @@ show(io::IO, o::MemoryOutput) = print(io, "MemoryOutput$(collect(keys(o.data)))" """ function (o::MemoryOutput)(y, t, dt, yfun) save, ts = o.save_cond(y, t, dt, o.saved) - append_stats!(o, o.statsfun(y, t, dt)) !haskey(o.data, o.yname) && initialise(o, y) while save + ys = yfun(ts) + append_stats!(o, o.statsfun(ys, ts, dt)) s = size(o.data[o.yname]) if s[end] < o.saved+1 - o.data[o.yname] = fastcat(o.data[o.yname], yfun(ts)) + o.data[o.yname] = fastcat(o.data[o.yname], ys) push!(o.data[o.tname], ts) else idcs = fill(:, ndims(y)) - o.data[o.yname][idcs..., o.saved+1] = yfun(ts) + o.data[o.yname][idcs..., o.saved+1] = ys o.data[o.tname][o.saved+1] = ts end o.saved += 1 save, ts = o.save_cond(y, t, dt, o.saved) end + append_stats!(o, o.statsfun(y, t, dt)) end function append_stats!(o::MemoryOutput, d) From f7391bf66fcd26de4632e41b293d15e65ac7bf58 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 20 Apr 2021 13:54:14 +0100 Subject: [PATCH 3/8] fix HDF5Ouput --- src/Output.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Output.jl b/src/Output.jl index 28fc61da..865da1cd 100644 --- a/src/Output.jl +++ b/src/Output.jl @@ -310,11 +310,15 @@ function (o::HDF5Output)(y, t, dt, yfun) save, ts = o.save_cond(y, t, dt, o.saved) if save @hlock HDF5.h5open(o.fpath, "r+") do file - !HDF5.exists(file, o.yname) && initialise(o, y) - statsnames = sort(collect(keys(o.stats_tmp[end]))) - cachehash = hash((statsnames, size(y))) - cachehash == o.cachehash || error( - "the hash for this propagation does not agree with cache in file") + if !HDF5.exists(file, o.yname) + push!(o.stats_tmp, o.statsfun(y, t, dt)) + initialise(o, y) + statsnames = sort(collect(keys(o.stats_tmp[end]))) + cachehash = hash((statsnames, size(y))) + cachehash == o.cachehash || error( + "the hash for this propagation does not agree with cache in file") + o.stats_tmp = Vector{Dict{String, Any}}() + end while save s = collect(size(file[o.yname])) idcs = fill(:, length(s)-1) From 96b87d16219dff6a969dc75f7a1c530f79d6e2ef Mon Sep 17 00:00:00 2001 From: John Travers Date: Sat, 24 Apr 2021 21:39:32 +0100 Subject: [PATCH 4/8] review corrections --- src/Output.jl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Output.jl b/src/Output.jl index 865da1cd..d4ee35d8 100644 --- a/src/Output.jl +++ b/src/Output.jl @@ -211,7 +211,7 @@ function HDF5Output(fpath::AbstractString) HDF5Output(fpath, 0, 0, 1; readonly=true) end -function initialise(o::HDF5Output, y) +function initialise(o::HDF5Output, y, statsnames) ydims = size(y) idims = init_dims(ydims, o.save_cond) cdims = collect(idims) @@ -230,7 +230,6 @@ function initialise(o::HDF5Output, y) end HDF5.d_create(file, o.tname, HDF5.datatype(Float64), ((dims[end],), (-1,)), "chunk", (1,)) - statsnames = sort(collect(keys(o.stats_tmp[end]))) o.cachehash = hash((statsnames, size(y))) file["meta"]["cachehash"] = o.cachehash if o.cache @@ -311,13 +310,13 @@ function (o::HDF5Output)(y, t, dt, yfun) if save @hlock HDF5.h5open(o.fpath, "r+") do file if !HDF5.exists(file, o.yname) - push!(o.stats_tmp, o.statsfun(y, t, dt)) - initialise(o, y) - statsnames = sort(collect(keys(o.stats_tmp[end]))) + statsnames = sort(collect(keys(o.statsfun(y, t, dt)))) + initialise(o, y, statsnames) + else + statsnames = sort(collect(keys(o.stats_tmp[end]))) cachehash = hash((statsnames, size(y))) cachehash == o.cachehash || error( "the hash for this propagation does not agree with cache in file") - o.stats_tmp = Vector{Dict{String, Any}}() end while save s = collect(size(file[o.yname])) From e474719b9121bf5ce6cf89f72689fc7f54c665a0 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 13 Sep 2021 14:38:31 +0100 Subject: [PATCH 5/8] addres review --- src/Output.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Output.jl b/src/Output.jl index 73b7541e..c44cb3b3 100644 --- a/src/Output.jl +++ b/src/Output.jl @@ -77,7 +77,9 @@ function (o::MemoryOutput)(y, t, dt, yfun) o.saved += 1 save, ts = o.save_cond(y, t, dt, o.saved) end - append_stats!(o, o.statsfun(y, t, dt)) + if ts != t + append_stats!(o, o.statsfun(y, t, dt)) + end end function append_stats!(o::MemoryOutput, d) From 172b1eccf2774153cda0bbaca5dae78c15dfd5fa Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 13 Sep 2021 14:41:47 +0100 Subject: [PATCH 6/8] prevent double save --- src/Output.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Output.jl b/src/Output.jl index c44cb3b3..9fcd2f6c 100644 --- a/src/Output.jl +++ b/src/Output.jl @@ -343,7 +343,9 @@ function (o::HDF5Output)(y, t, dt, yfun) end end end - push!(o.stats_tmp, o.statsfun(y, t, dt)) + if ts != t + push!(o.stats_tmp, o.statsfun(y, t, dt)) + end end function append_stats!(parent, a::Array{Dict{String,Any},1}) From 23968e5d73813b800ad4fe8149baca7c347848ef Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 13 Sep 2021 15:13:46 +0100 Subject: [PATCH 7/8] add tests --- test/test_output.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_output.jl b/test/test_output.jl index ca5ccdae..0d992cc2 100644 --- a/test/test_output.jl +++ b/test/test_output.jl @@ -193,6 +193,11 @@ fpath_comp = joinpath(dirpath, "test_comp.h5") ω, Eω, zac = Processing.getEω(o, 5e-2) @test (ω, Eω[:, 1]) == Processing.getEω(grid, mem["Eω"][:, 51]) @test zac[1] == 5e-2 + # test stat save locations + @test length(unique(mem["stats"]["z"])) == length(mem["stats"]["z"]) + @test length(unique(o["stats"]["z"])) == length(o["stats"]["z"]) + @test all([z in o["stats"]["z"] for z in o["z"]]) + @test all([z in mem["stats"]["z"] for z in mem["z"]]) end rm(fpath) rm(fpath_comp) From 244c3abaa765d53f59a79af53cce3169b7966e2a Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 13 Sep 2021 15:29:56 +0100 Subject: [PATCH 8/8] fix element access --- src/Output.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Output.jl b/src/Output.jl index 9fcd2f6c..776fd324 100644 --- a/src/Output.jl +++ b/src/Output.jl @@ -305,11 +305,10 @@ function (o::HDF5Output)(y, t, dt, yfun) save, ts = o.save_cond(y, t, dt, o.saved) if save @hlock HDF5.h5open(o.fpath, "r+") do file + statsnames = sort(collect(keys(o.statsfun(y, t, dt)))) if !HDF5.haskey(file, o.yname) - statsnames = sort(collect(keys(o.statsfun(y, t, dt)))) initialise(o, y, statsnames) - else - statsnames = sort(collect(keys(o.stats_tmp[end]))) + else cachehash = hash((statsnames, size(y))) cachehash == o.cachehash || error( "the hash for this propagation does not agree with cache in file")