From ef6d7a4aa97d39fed92721e0db263170c5cbb2ce Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 5 Apr 2020 08:53:35 +1000 Subject: [PATCH 1/2] Version check for serialized data headers Check that the serialization data format is compatible before attempting to read a serialized stream. New versions of Serialization are assumed to be able to read old serialized data, but attempting to read newer data with an older version of Serialization will fail with an error. --- stdlib/Serialization/src/Serialization.jl | 34 +++++++++++++++++++++-- stdlib/Serialization/test/runtests.jl | 25 +++++++++++++++-- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index bf6c995b708b8..a459a0b2d5346 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -685,6 +685,36 @@ function writeheader(s::AbstractSerializer) nothing end +function readheader(s::AbstractSerializer) + # Tag already read + io = s.io + m1 = read(io, UInt8) + m2 = read(io, UInt8) + if m1 != UInt8('J') || m2 != UInt8('L') + error("Unsupported serialization format (got header magic bytes $m1 $m2)") + end + version = read(io, UInt8) + flags = read(io, UInt8) + reserved1 = read(io, UInt8) + reserved2 = read(io, UInt8) + reserved3 = read(io, UInt8) + endianflag = flags & 0x3 + wordflag = (flags >> 2) & 0x3 + wordsize = wordflag == 0 ? 4 : + wordflag == 1 ? 8 : + error("Unknown word size flag in header") + endian_bom = endianflag == 0 ? 0x04030201 : + endianflag == 1 ? 0x01020304 : + error("Unknown endianness flag in header") + # Check protocol compatibility. + endian_bom == ENDIAN_BOM || error("Serialized byte order mismatch ($(repr(endian_bom)))") + wordsize == sizeof(Int) || error("Serialized word size mismatch ($wordsize)") + if version > ser_version + error("""Cannot read stream serialized with a newer version of Julia. + Got data version $version > current version $ser_version""") + end +end + """ serialize(stream::IO, value) @@ -843,9 +873,7 @@ function handle_deserialize(s::AbstractSerializer, b::Int32) elseif b == LONGSYMBOL_TAG return deserialize_symbol(s, Int(read(s.io, Int32)::Int32)) elseif b == HEADER_TAG - for _ = 1:7 - read(s.io, UInt8) - end + readheader(s) return deserialize(s) elseif b == INT8_TAG return read(s.io, Int8) diff --git a/stdlib/Serialization/test/runtests.jl b/stdlib/Serialization/test/runtests.jl index c6c28517adee6..5b631b645ed2c 100644 --- a/stdlib/Serialization/test/runtests.jl +++ b/stdlib/Serialization/test/runtests.jl @@ -529,8 +529,8 @@ let x = T20324[T20324(1) for i = 1:2] @test y == x end -# serializer header -let io = IOBuffer() +@testset "serializer header" begin + io = IOBuffer() serialize(io, ()) seekstart(io) b = read(io) @@ -541,6 +541,27 @@ let io = IOBuffer() @test ((b[5] & 0xc)>>2) == (sizeof(Int) == 8) @test (b[5] & 0xf0) == 0 @test all(b[6:8] .== 0) + + # Detection of incompatible binary serializations + function corrupt_header(bytes, offset, val) + b = copy(bytes) + b[offset] = val + IOBuffer(b) + end + @test_throws( + ErrorException("""Cannot read stream serialized with a newer version of Julia. + Got data version 255 > current version $(Serialization.ser_version)"""), + deserialize(corrupt_header(b, 4, 0xff))) + @test_throws(ErrorException("Unknown word size flag in header"), + deserialize(corrupt_header(b, 5, 2<<2))) + @test_throws(ErrorException("Unknown endianness flag in header"), + deserialize(corrupt_header(b, 5, 2))) + other_wordsize = sizeof(Int) == 8 ? 4 : 8 + @test_throws(ErrorException("Serialized word size mismatch ($other_wordsize)"), + deserialize(corrupt_header(b, 5, UInt8((sizeof(Int) != 8)<<2)))) + other_endianness = bswap(ENDIAN_BOM) + @test_throws(ErrorException("Serialized byte order mismatch ($(repr(other_endianness)))"), + deserialize(corrupt_header(b, 5, UInt8(ENDIAN_BOM != 0x01020304)))) end # issue #26979 From 77c202fedf55c265dfc00f934de038dcb6c4805c Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 7 Apr 2020 15:27:30 +1000 Subject: [PATCH 2/2] Remove wordsize check --- stdlib/Serialization/src/Serialization.jl | 4 +++- stdlib/Serialization/test/runtests.jl | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index a459a0b2d5346..d1665a73b2fa8 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -708,7 +708,9 @@ function readheader(s::AbstractSerializer) error("Unknown endianness flag in header") # Check protocol compatibility. endian_bom == ENDIAN_BOM || error("Serialized byte order mismatch ($(repr(endian_bom)))") - wordsize == sizeof(Int) || error("Serialized word size mismatch ($wordsize)") + # We don't check wordsize == sizeof(Int) here, as Int is encoded concretely + # as Int32 or Int64, which should be enough to correctly deserialize a range + # of data structures between Julia versions. if version > ser_version error("""Cannot read stream serialized with a newer version of Julia. Got data version $version > current version $ser_version""") diff --git a/stdlib/Serialization/test/runtests.jl b/stdlib/Serialization/test/runtests.jl index 5b631b645ed2c..24bd08145b261 100644 --- a/stdlib/Serialization/test/runtests.jl +++ b/stdlib/Serialization/test/runtests.jl @@ -557,8 +557,6 @@ end @test_throws(ErrorException("Unknown endianness flag in header"), deserialize(corrupt_header(b, 5, 2))) other_wordsize = sizeof(Int) == 8 ? 4 : 8 - @test_throws(ErrorException("Serialized word size mismatch ($other_wordsize)"), - deserialize(corrupt_header(b, 5, UInt8((sizeof(Int) != 8)<<2)))) other_endianness = bswap(ENDIAN_BOM) @test_throws(ErrorException("Serialized byte order mismatch ($(repr(other_endianness)))"), deserialize(corrupt_header(b, 5, UInt8(ENDIAN_BOM != 0x01020304))))