From 189e4c5596d2927fd2346431ef1ab8861d56bafb Mon Sep 17 00:00:00 2001 From: Guido Kraemer Date: Sat, 4 Aug 2018 22:31:54 +0200 Subject: [PATCH] 0.7 compat --- src/JLD2.jl | 13 ++++++------- src/bufferedio.jl | 2 +- src/data.jl | 23 +++++++++++++---------- src/datasets.jl | 2 +- src/mmapio.jl | 3 ++- test/rw.jl | 10 +++++----- 6 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/JLD2.jl b/src/JLD2.jl index 3e40af3b..a58f6a9b 100644 --- a/src/JLD2.jl +++ b/src/JLD2.jl @@ -1,5 +1,3 @@ -__precompile__() - module JLD2 using DataStructures, CodecZlib, FileIO import Base.sizeof @@ -151,8 +149,8 @@ mutable struct JLDFile{T<:IO} datatype_locations::OrderedDict{RelOffset,CommittedDatatype} datatypes::Vector{H5Datatype} datatype_wsession::JLDWriteSession{Dict{UInt,RelOffset}} - jlh5type::ObjectIdDict - h5jltype::ObjectIdDict + jlh5type::IdDict + h5jltype::IdDict jloffset::Dict{RelOffset,WeakRef} end_of_data::Int64 global_heaps::Dict{RelOffset,GlobalHeap} @@ -166,10 +164,10 @@ mutable struct JLDFile{T<:IO} compress::Bool, mmaparrays::Bool) where T f = new(io, path, writable, written, compress, mmaparrays, 1, OrderedDict{RelOffset,CommittedDatatype}(), H5Datatype[], - JLDWriteSession(), ObjectIdDict(), ObjectIdDict(), Dict{RelOffset,WeakRef}(), + JLDWriteSession(), IdDict(), IdDict(), Dict{RelOffset,WeakRef}(), Int64(FILE_HEADER_LENGTH + sizeof(Superblock)), Dict{RelOffset,GlobalHeap}(), GlobalHeap(0, 0, 0, Int64[]), Dict{RelOffset,Group}(), UNDEFINED_ADDRESS) - finalizer(f, jld_finalizer) + finalizer(jld_finalizer, f) f end end @@ -196,7 +194,8 @@ h5offset(f::JLDFile, x::Int64) = RelOffset(x - FILE_HEADER_LENGTH) # openfile(::Type{IOStream}, fname, wr, create, truncate) = - open(fname, true, wr, create, truncate, false) + open(fname, read = true, write = wr, create = create, + truncate = truncate, append = false) openfile(::Type{MmapIO}, fname, wr, create, truncate) = MmapIO(fname, wr, create, truncate) diff --git a/src/bufferedio.jl b/src/bufferedio.jl index 16c2b919..6adbbb2d 100644 --- a/src/bufferedio.jl +++ b/src/bufferedio.jl @@ -75,7 +75,7 @@ Base.show(io::IO, ::BufferedReader) = print(io, "BufferedReader") function readmore!(io::BufferedReader, n::Int) f = io.f - amount = max(nb_available(f), n) + amount = max(bytesavailable(f), n) buffer = io.buffer oldlen = length(buffer) resize!(buffer, oldlen + amount) diff --git a/src/data.jl b/src/data.jl index 0d451f92..dd6b20f2 100644 --- a/src/data.jl +++ b/src/data.jl @@ -761,7 +761,7 @@ function h5convert!(out::Pointers, fls::FixedLengthString, f::JLDFile, x, ::JLDW (unsafe_copyto!(convert(Ptr{UInt8}, out), pointer(x), fls.length); nothing) end h5convert!(out::Pointers, ::Type{Vlen{String}}, f::JLDFile, x, wsession::JLDWriteSession) = - store_vlen!(out, UInt8, f, Vector{UInt8}(x), wsession) + store_vlen!(out, UInt8, f, unsafe_wrap(Array{UInt8}, pointer(x), ncodeunits(x)), wsession) jlconvert(::ReadRepresentation{String,Vlen{String}}, f::JLDFile, ptr::Ptr, ::RelOffset) = String(jlconvert(ReadRepresentation{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE)) @@ -788,8 +788,10 @@ fieldodr(::Type{Symbol}, ::Bool) = Vlen{String} h5type(f::JLDFile, ::Type{Symbol}, x) = h5fieldtype(f, Symbol, typeof(x), Val{true}) odr(::Type{Symbol}) = Vlen{String} -h5convert!(out::Pointers, ::Type{Vlen{String}}, f::JLDFile, x::Symbol, ::JLDWriteSession) = - store_vlen!(out, UInt8, f, Vector{UInt8}(String(x)), f.datatype_wsession) +function h5convert!(out::Pointers, ::Type{Vlen{String}}, f::JLDFile, x::Symbol, ::JLDWriteSession) + s = String(x) + store_vlen!(out, UInt8, f, unsafe_wrap(Array{UInt8}, pointer(s), ncodeunits(s)), f.datatype_wsession) +end constructrr(::JLDFile, ::Type{Symbol}, dt::VariableLengthDatatype{FixedPointDatatype}, ::Vector{ReadAttribute}) = dt == H5TYPE_VLEN_UTF8 ? (ReadRepresentation{Symbol,Vlen{String}}(), true) : @@ -801,9 +803,9 @@ jlconvert(::ReadRepresentation{Symbol,Vlen{String}}, f::JLDFile, ptr::Ptr, ::Rel ## BigInts and BigFloats writeas(::Union{Type{BigInt},Type{BigFloat}}) = String -wconvert(::Type{String}, x::BigInt) = base(62, x) +wconvert(::Type{String}, x::BigInt) = string(x, base = 62) wconvert(::Type{String}, x::BigFloat) = string(x) -rconvert(::Type{BigInt}, x::String) = parse(BigInt, x, 62) +rconvert(::Type{BigInt}, x::String) = parse(BigInt, x, base = 62) rconvert(::Type{BigFloat}, x::String) = parse(BigFloat, x) ## DataTypes @@ -849,7 +851,7 @@ function typename(T::DataType) tn = Symbol[] m = T.name.module while m != parentmodule(m) - push!(tn, module_name(m)) + push!(tn, nameof(m)) m = parentmodule(m) end reverse!(tn) @@ -858,11 +860,12 @@ function typename(T::DataType) end function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::DataType, wsession::JLDWriteSession) - store_vlen!(out, UInt8, f, Vector{UInt8}(typename(T)), f.datatype_wsession) + t = typename(T) + store_vlen!(out, UInt8, f, unsafe_wrap(Array{UInt8}, pointer(t), ncodeunits(t)), f.datatype_wsession) if isempty(T.parameters) h5convert_uninitialized!(out+odr_sizeof(Vlen{UInt8}), Vlen{UInt8}) else - refs = RelOffset[begin + refs = RelOffset[ if isa(x, DataType) # The heuristic here is that, if the field type is a committed data type, # then we commit the datatype and write it as a reference to the committed @@ -879,7 +882,7 @@ function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::DataType, wsess else write_ref(f, x, wsession) end - end for x in T.parameters] + for x in T.parameters] store_vlen!(out+odr_sizeof(Vlen{UInt8}), RelOffset, f, refs, f.datatype_wsession) end nothing @@ -1031,7 +1034,7 @@ rconvert(::Type{Core.SimpleVector}, x::Vector{Any}) = Core.svec(x...) ## Dicts writeas(::Type{Dict{K,V}}) where {K,V} = Vector{Pair{K,V}} -writeas(::Type{ObjectIdDict}) = Vector{Pair{Any,Any}} +writeas(::Type{IdDict}) = Vector{Pair{Any,Any}} wconvert(::Type{Vector{Pair{K,V}}}, x::AbstractDict{K,V}) where {K,V} = collect(x) function rconvert(::Type{T}, x::Vector{Pair{K,V}}) where {T<:AbstractDict,K,V} d = T() diff --git a/src/datasets.jl b/src/datasets.jl index 6ea8ae36..77e447a0 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -285,7 +285,7 @@ end Construct array by reading `ndims` dimensions from `io`. Assumes `io` has already been seeked to the correct position. """ -function construct_array{T}(io::IO, ::Type{T}, ndims::Int)::Array{T} +function construct_array(io::IO, ::Type{T}, ndims::Int)::Array{T} where {T} if ndims == 1 n = read(io, Int64) Vector{T}(undef, n) diff --git a/src/mmapio.jl b/src/mmapio.jl index 182be3c0..9e76931a 100644 --- a/src/mmapio.jl +++ b/src/mmapio.jl @@ -97,7 +97,8 @@ end function MmapIO(fname::AbstractString, write::Bool, create::Bool, truncate::Bool) truncate && !write && throw(ArgumentError("cannot truncate file that is not writable")) - f = open(fname, true, write, create, truncate, false) + f = open(fname, read = true, write = write, + create = create, truncate = truncate, append = false) initialsz = truncate ? Int64(0) : filesize(fname) @static if Int == Int32 initialsz > typemax(Int) && error("cannot read a file greater than 2GB on a 32-bit system") diff --git a/test/rw.jl b/test/rw.jl index f3cea3b4..88a40983 100644 --- a/test/rw.jl +++ b/test/rw.jl @@ -37,7 +37,7 @@ AB = Any[A, B] t = (3, "cat") c = Complex{Float32}(3,7) cint = 1+im # issue 108 -C = reinterpret(Complex{Float64}, B, (4,)) +C = reshape(reinterpret(Complex{Float64}, B), (4,)) emptyA = zeros(0,2) emptyB = zeros(2,0) try @@ -55,7 +55,7 @@ msempty = MyStruct(5, Float64[]) sym = :TestSymbol syms = [:a, :b] d = Dict([(syms[1],"aardvark"), (syms[2], "banana")]) -oidd = ObjectIdDict([(syms[1],"aardvark"), (syms[2], "banana")]) +oidd = IdDict([(syms[1],"aardvark"), (syms[2], "banana")]) ex = quote function incrementby1(x::Int) x+1 @@ -237,9 +237,9 @@ bitsparamint16 = BitsParams{Int16(1)}() tuple_of_tuples = (1, 2, (3, 4, [5, 6]), [7, 8]) # Zero-dimensional arrays -zerod = Array{Int}() +zerod = Array{Int}(undef) zerod[] = 1 -zerod_any = Array{Any}() +zerod_any = Array{Any}(undef) zerod_any[] = 1.0+1.0im # Type with None typed field @@ -313,7 +313,7 @@ macro check(fid, sym) tmp = read($fid, $(string(sym))) catch e @show e - Base.show_backtrace(STDOUT, catch_backtrace()) + Base.show_backtrace(stdout, catch_backtrace()) error("error reading ", $(string(sym))) end written_type = typeof($sym)