de-duplicate strings in serialization (#35056)

fixes #35030
JuliaLang · Mar 19, 2020 · d33c5a5 · d33c5a5 · nanosoldier · Mar 20, 2020
1 parent 64d31fe
commit d33c5a5
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 2 deletions.
diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl
@@ -31,7 +31,7 @@ Serializer(io::IO) = Serializer{typeof(io)}(io)
 
 const n_int_literals = 33
 const n_reserved_slots = 24
-const n_reserved_tags = 11
+const n_reserved_tags = 10
 
 const TAGS = Any[
     Symbol, Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Int128, UInt128,
@@ -57,6 +57,7 @@ const TAGS = Any[
     Symbol, # FULL_GLOBALREF_TAG
     Symbol, # HEADER_TAG
     Symbol, # IDDICT_TAG
+    Symbol, # SHARED_REF_TAG
     fill(Symbol, n_reserved_tags)...,
 
     (), Bool, Any, Bottom, Core.TypeofBottom, Type, svec(), Tuple{}, false, true, nothing,
@@ -76,7 +77,7 @@ const TAGS = Any[
 
 @assert length(TAGS) == 255
 
-const ser_version = 9 # do not make changes without bumping the version #!
+const ser_version = 10 # do not make changes without bumping the version #!
 
 const NTAGS = length(TAGS)
 
@@ -135,6 +136,7 @@ const REF_OBJECT_TAG       = Int32(o0+13)
 const FULL_GLOBALREF_TAG   = Int32(o0+14)
 const HEADER_TAG           = Int32(o0+15)
 const IDDICT_TAG           = Int32(o0+16)
+const SHARED_REF_TAG       = Int32(o0+17)
 
 writetag(s::IO, tag) = (write(s, UInt8(tag)); nothing)
 
@@ -284,6 +286,10 @@ end
 
 function serialize(s::AbstractSerializer, ss::String)
     len = sizeof(ss)
+    if len > 7
+        serialize_cycle(s, ss) && return
+        writetag(s.io, SHARED_REF_TAG)
+    end
     if len <= 255
         writetag(s.io, STRING_TAG)
         write(s.io, UInt8(len))
@@ -802,6 +808,11 @@ function handle_deserialize(s::AbstractSerializer, b::Int32)
         push!(s.pending_refs, slot)
         t = deserialize(s)
         return deserialize(s, t)
+    elseif b == SHARED_REF_TAG
+        slot = s.counter; s.counter += 1
+        obj = deserialize(s)
+        s.table[slot] = obj
+        return obj
     elseif b == SYMBOL_TAG
         return deserialize_symbol(s, Int(read(s.io, UInt8)::UInt8))
     elseif b == SHORTINT64_TAG

diff --git a/stdlib/Serialization/test/runtests.jl b/stdlib/Serialization/test/runtests.jl
@@ -582,3 +582,11 @@ let d = IdDict([1] => 2, [3] => 4), io = IOBuffer()
     @test Dict(d) == Dict(ds)
     @test all([k in keys(ds) for k in keys(ds)])
 end
+
+# issue #35030, shared references to Strings
+let s = join(rand('a':'z', 1024)), io = IOBuffer()
+    serialize(io, (s, s))
+    seekstart(io)
+    s2 = deserialize(io)
+    @test Base.summarysize(s2) < 2*sizeof(s)
+end