diff --git a/src/FileIO.jl b/src/FileIO.jl index b53bad46..9e2c1eda 100644 --- a/src/FileIO.jl +++ b/src/FileIO.jl @@ -42,9 +42,9 @@ include("registry.jl") - `load([filename|stream])`: read data in formatted file, inferring the format - `load(File(format"PNG",filename))`: specify the format manually -- `loadstreaming(f)`: similar to `load`, except that it returns an object that can be read from +- `loadstreaming([filename|stream])`: similar to `load`, except that it returns an object that can be read from - `save(filename, data...)` for similar operations involving saving data -- `savestreaming(f)`: similar to `save`, except that it returns an object that can be written to +- `savestreaming([filename|stream])`: similar to `save`, except that it returns an object that can be written to - `io = open(f::File, args...)` opens a file - `io = stream(s::Stream)` returns the IOStream from the query object `s` diff --git a/src/loadsave.jl b/src/loadsave.jl index 71d8e3ba..66259ba7 100644 --- a/src/loadsave.jl +++ b/src/loadsave.jl @@ -87,16 +87,19 @@ trying to infer the format from `filename`. """ savestreaming +# if a bare filename or IO stream are given, query for the format and dispatch +# to the formatted handlers below for fn in (:load, :loadstreaming, :save, :savestreaming) - @eval $fn(s::@compat(Union{AbstractString,IO}), args...; options...) = + @eval $fn(s::Union{AbstractString,IO}, args...; options...) = $fn(query(s), args...; options...) end +# return a save function, so you can do `thing_to_save |> save("filename.ext")` function save(s::Union{AbstractString,IO}; options...) data -> save(s, data; options...) end -# Forced format +# Allow format to be overridden with first argument function save{sym}(df::Type{DataFormat{sym}}, f::AbstractString, data...; options...) libraries = applicable_savers(df) checked_import(libraries[1]) @@ -137,7 +140,7 @@ for fn in (:loadstreaming, :savestreaming) end end -# Fallbacks +# Handlers for formatted files/streams # TODO: this definitely should be refactored to reduce duplication function load{F}(q::Formatted{F}, args...; options...) @@ -218,6 +221,8 @@ function savestreaming{F}(q::Formatted{F}, data...; options...) handle_exceptions(failures, "opening \"$(filename(q))\" for streamed saving") end +# returns true if the given method table includes a method defined by the given +# module, false otherwise function has_method_from(mt, Library) for m in mt if getmodule(m) == Library diff --git a/src/query.jl b/src/query.jl index 4330d4a8..26467a00 100644 --- a/src/query.jl +++ b/src/query.jl @@ -358,7 +358,8 @@ unknown{F}(::Stream{F}) = unknown(F) """ `query(filename)` returns a `File` object with information about the -format inferred from the file's extension and/or magic bytes.""" +format inferred from the file's extension and/or magic bytes. +""" function query(filename::AbstractString) _, ext = splitext(filename) if haskey(ext2sym, ext) @@ -394,7 +395,8 @@ hasfunction(s::Tuple) = false #has magic """ `query(io, [filename])` returns a `Stream` object with information about the -format inferred from the magic bytes.""" +format inferred from the magic bytes. +""" query(io::IO, filename) = query(io, Nullable(String(filename))) function query(io::IO, filename::Nullable{String}=Nullable{String}()) diff --git a/test/loadsave.jl b/test/loadsave.jl index 57dc7678..86880f26 100644 --- a/test/loadsave.jl +++ b/test/loadsave.jl @@ -61,15 +61,76 @@ module Dummy using FileIO +mutable struct DummyReader{IOtype} + stream::IOtype + ownstream::Bool + bytesleft::Int64 +end + +function DummyReader(stream, ownstream) + read(stream, 5) == magic(format"DUMMY") || error("wrong magic bytes") + DummyReader(stream, ownstream, read(stream, Int64)) +end + +function Base.read(stream::DummyReader, n) + toread = min(n, stream.bytesleft) + buf = read(stream.stream, toread) + stream.bytesleft -= length(buf) + buf +end + +Base.eof(stream::DummyReader) = stream.bytesleft == 0 || eof(stream.stream) +Base.close(stream::DummyReader) = stream.ownstream && close(stream.stream) + +mutable struct DummyWriter{IOtype} + stream::IOtype + ownstream::Bool + headerpos::Int + byteswritten::Int +end + +function DummyWriter(stream, ownstream) + write(stream, magic(format"DUMMY")) # Write the magic bytes + # store the position where we'll need to write the length + pos = position(stream) + # write a dummy length value + write(stream, 0xffffffffffffffff) + DummyWriter(stream, ownstream, pos, 0) +end + +function Base.write(stream::DummyWriter, data) + udata = convert(Vector{UInt8}, data) + n = write(stream.stream, udata) + stream.byteswritten += n + + n +end + +function Base.close(stream::DummyWriter) + here = position(stream.stream) + # go back and write the header + seek(stream.stream, stream.headerpos) + write(stream.stream, convert(Int64, stream.byteswritten)) + seek(stream.stream, here) + stream.ownstream && close(stream.stream) + + nothing +end + +loadstreaming(s::Stream{format"DUMMY"}) = DummyReader(s, false) +loadstreaming(file::File{format"DUMMY"}) = DummyReader(open(file), true) +savestreaming(s::Stream{format"DUMMY"}) = DummyWriter(s, false) +savestreaming(file::File{format"DUMMY"}) = DummyWriter(open(file, "w"), true) + +# we could implement `load` and `save` in terms of their streaming versions function FileIO.load(file::File{format"DUMMY"}) open(file) do s - skipmagic(s) load(s) end end function FileIO.load(s::Stream{format"DUMMY"}) - # We're already past the magic bytes + skipmagic(s) n = read(s, Int64) out = Vector{UInt8}(n) read!(s, out) @@ -133,7 +194,6 @@ add_saver(format"DUMMY", :Dummy) @test a == b b = open(query(fn)) do s - skipmagic(s) load(s) end @test a == b