Elide fields with default value; also supports Proto2 non-zero defaults.

Also now supports inner message as a Some type.

Core/Serializer.fs

@@ -19,18 +19,43 @@ module Utility =
     /// Decode SInt64
     let zagZig64 (n:int64) = int64(uint64 n >>> 1) ^^^ (if n&&&1L = 0L then 0L else -1L)
 
-    /// Calculate length when encoded as a Varint
+    /// Calculate length when encoded as a Varint; if value is default, then length is 0
+    let varIntLenDefaulted d (v:uint64) =
+        if v = d
+        then 0
+        else
+            let rec loop acc len =
+                let bMore = acc > 0x7FUL
+                if bMore
+                then loop (acc >>> 7) (len+1)
+                else len
+            loop v 1
+
+    /// Calculate length when encoded as a Varint; if value is default, then length is 0
     let varIntLen (v:uint64) =
+        varIntLenDefaulted 0UL v
+
+    /// Calculate length when encoded as a Varint without a default; e.g., a packed field
+    let varIntLenNoDefault (v:uint64) =
         let rec loop acc len =
             let bMore = acc > 0x7FUL
             if bMore
             then loop (acc >>> 7) (len+1)
             else len
         loop v 1
 
-    // Calculate field number length when encoded in a tag
+
+    /// Calculate field number length when encoded in a tag
     let tagLen (t:int32) =
-        varIntLen ((uint64 t) <<< 3)
+        varIntLenNoDefault ((uint64 t) <<< 3)
+
+    /// Apply a function to a Option value, if the value is Some.
+    let inline IfSome f opt =
+        match opt with
+        | None -> id
+        | Some(v) -> f v
+
+
 
 ///
 /// Serialization and Deserialization.
@@ -133,8 +158,8 @@ module Serializer =
 
     let hydrateString = helper_bytes toString
     let hydrateBytes  = helper_bytes toByteArray
-    // TODO: Can the following be made type safe?
     let hydrateMessage messageCtor = helper_bytes messageCtor
+    let hydrateOptionalMessage messageCtor = hydrateMessage (messageCtor >> Some)
 
     /// Helper to deserialize Packed Repeated from LengthDelimited.
     /// Since this is used by an inline function (hydratePackedEnum),
@@ -165,25 +190,47 @@ module Serializer =
     let hydratePackedSingle   = helper_packed decodeSingle
     let hydratePackedDouble   = helper_packed decodeDouble
 
+//---- Serialization
+
+    /// If value = default, then elide the field (don't serialize)
+    let inline elided d v f =
+        if v = d
+        then id
+        else f
+
     /// Generic Dehydrate for all varint types, excepted for signed & bool:
     ///   int32, int64, uint32, uint64, enum
-    let inline dehydrateVarint fldNum i = WireFormat.encodeFieldVarint fldNum (uint64 i)
+    let inline dehydrateDefaultedVarint d fldNum v = elided d v <| WireFormat.encodeFieldVarint fldNum (uint64 v)
+    let inline dehydrateNondefaultedVarint fldNum v = WireFormat.encodeFieldVarint fldNum (uint64 v)
 
-//---- Serialization
+    let dehydrateDefaultedSInt32 d fldNum v = elided d v <| WireFormat.encodeFieldVarint fldNum (Utility.zigZag32 v |> uint64)
+    let dehydrateDefaultedSInt64 d fldNum v = elided d v <| WireFormat.encodeFieldVarint fldNum (Utility.zigZag64 v |> uint64)
+    let dehydrateDefaultedBool   d fldNum v = elided d v <| dehydrateNondefaultedVarint fldNum (fromBool v)
+
+    let inline dehydrateDefaultedFixed32  d fldNum v = elided d v <| WireFormat.encodeFieldFixed32 fldNum (uint32 v)
+    let inline dehydrateDefaultedFixed64  d fldNum v = elided d v <| WireFormat.encodeFieldFixed64 fldNum (uint64 v)
+
+    let dehydrateDefaultedSingle d fldNum v = elided d v <| WireFormat.encodeFieldSingle fldNum v
+    let dehydrateDefaultedDouble d fldNum v = elided d v <| WireFormat.encodeFieldDouble fldNum v
 
-    let dehydrateSInt32 fldNum i = WireFormat.encodeFieldVarint fldNum (Utility.zigZag32 i |> uint64)
-    let dehydrateSInt64 fldNum i = WireFormat.encodeFieldVarint fldNum (Utility.zigZag64 i |> uint64)
+    let dehydrateDefaultedString d fldNum v = elided d v <| WireFormat.encodeFieldString fldNum v
+    let dehydrateDefaultedBytes  d fldNum v = elided d v <| WireFormat.encodeFieldBytes fldNum v
 
-    let dehydrateBool fldNum b = dehydrateVarint fldNum (fromBool b)
+    let inline dehydrateVarint fldNum (v:'a) = dehydrateDefaultedVarint (Unchecked.defaultof<'a>) fldNum v
 
-    let inline dehydrateFixed32  fldNum i = WireFormat.encodeFieldFixed32 fldNum (uint32 i)
-    let inline dehydrateFixed64  fldNum i = WireFormat.encodeFieldFixed64 fldNum (uint64 i)
+    let dehydrateSInt32 fldNum v = dehydrateDefaultedSInt32 0 fldNum v
+    let dehydrateSInt64 fldNum v = dehydrateDefaultedSInt64 0L fldNum v
+    let dehydrateBool   fldNum v = dehydrateDefaultedBool false fldNum v
 
-    let dehydrateSingle fldNum i = WireFormat.encodeFieldSingle fldNum i
-    let dehydrateDouble fldNum i = WireFormat.encodeFieldDouble fldNum i
+    let inline dehydrateFixed32 fldNum v = dehydrateDefaultedFixed32 0 fldNum v
+    let inline dehydrateFixed64 fldNum v = dehydrateDefaultedFixed64 0 fldNum v
+
+    let dehydrateSingle fldNum v = dehydrateDefaultedSingle 0.0f fldNum v
+    let dehydrateDouble fldNum v = dehydrateDefaultedDouble 0.0 fldNum v
+
+    let dehydrateString fldNum v = dehydrateDefaultedString "" fldNum v
+    let dehydrateBytes  fldNum v = dehydrateDefaultedBytes (ArraySegment ([||]:byte array)) fldNum v
 
-    let dehydrateString fldNum s = WireFormat.encodeFieldString fldNum s
-    let dehydrateBytes  fldNum buf = WireFormat.encodeFieldBytes fldNum buf
 
     (* Dehydrate Repeated Packed Numeric Values *)
 
@@ -196,7 +243,7 @@ module Serializer =
         >> flip (List.fold (fun buf x -> buf |> encFn x )) xs
 
     let inline varIntListPackedLen encode (xs:'a list) =
-        List.sumBy (encode >> Utility.varIntLen) xs
+        List.sumBy (encode >> Utility.varIntLenNoDefault) xs
 
     /// Generic Dehydrate for all packed varint types, excepted for bool & signed:
     ///   int32, int64, uint32, uint64, enum
@@ -262,6 +309,9 @@ module Serializer =
         WireFormat.encodeTag fieldNum WireType.LengthDelimited
         >> serializeMsg
 
+    let inline dehydrateOptionalMessage fieldNum (o:^msg option when ^msg : (member SerializeLengthDelimited : ZeroCopyWriteBuffer -> ZeroCopyWriteBuffer)) =
+        o |> Utility.IfSome (fun o -> dehydrateMessage fieldNum o)
+
     (* Repeated Field Helpers *)
     let hydrateRepeated<'a> (hydrater:'a ref -> RawField -> unit) propRef rawField =
         let element = Unchecked.defaultof<'a>
@@ -452,7 +502,7 @@ type MessageBase () =
     /// length of the object as a varint.
     member x.SerializedLengthDelimitedLength =
         let len = x.SerializedLength
-        let lenlen = Utility.varIntLen (uint64 len)
+        let lenlen = Utility.varIntLenNoDefault (uint64 len)
         (uint32 lenlen) + len
 
     /// Serialize first the length as a varint, followed by the serialized

Froto.Core.Test/ExampleProtoClass.fs

@@ -4,10 +4,11 @@ open Froto.Core.Encoding
 
 type InnerMessage () =
     inherit MessageBase()
+    let ETestDefault = ETest.One  // NOTE: Non-zero default is only supported in Proto2
     let m_id = ref 0
     let m_name = ref ""
     let m_option = ref false
-    let m_test = ref ETest.Nada
+    let m_test = ref ETestDefault
     let m_packedFixed32 = ref List.empty
     let m_repeatedInt32 = ref List.empty
 
@@ -22,7 +23,7 @@ type InnerMessage () =
         m_id := 0
         m_name := ""
         m_option := false
-        m_test := ETest.Nada
+        m_test := ETestDefault
         m_packedFixed32 := List.empty
         m_repeatedInt32 := List.empty
 
@@ -42,7 +43,7 @@ type InnerMessage () =
             (!m_id            |> Serializer.dehydrateVarint 1) >>
             (!m_name          |> Serializer.dehydrateString 2) >>
             (!m_option        |> Serializer.dehydrateBool 3) >>
-            (!m_test          |> Serializer.dehydrateVarint 4) >>
+            (!m_test          |> Serializer.dehydrateDefaultedVarint ETestDefault 4) >>
             (!m_packedFixed32 |> Serializer.dehydratePackedFixed32 5) >>
             (!m_repeatedInt32 |> Serializer.dehydrateRepeated Serializer.dehydrateVarint 6)
         encode zcb
@@ -59,22 +60,22 @@ and ETest =
 
 type OuterMessage() =
     inherit MessageBase()
-    let m_inner = ref <| InnerMessage()
+    let m_inner = ref None
 
     member x.Inner with get() = !m_inner and set(v) = m_inner := v
 
     override x.Clear() =
-        (!m_inner).Clear()
+        m_inner := None
 
     override x.DecoderRing =
         [
-            42, m_inner |> Serializer.hydrateMessage (InnerMessage.FromArraySegment)
+            42, m_inner |> Serializer.hydrateOptionalMessage (InnerMessage.FromArraySegment)
         ]
         |> Map.ofList
 
     override x.EncoderRing(zcb) =
         let encode =
-            (!m_inner |> Serializer.dehydrateMessage 42)
+            (!m_inner |> Serializer.dehydrateOptionalMessage 42)
         encode zcb
 
     static member FromArraySegment (buf:System.ArraySegment<byte>) =

Froto.Core.Test/TestMessageSerialization.fs

@@ -11,9 +11,7 @@ module MessageSerialization =
 
 (* TODO:
     Write tests (and implementation) for:
-    - Empty values are elided from serialized form
     - Proto2 required field missing causes exception
-    - Proto2 enum has correct default (done via code-gen/Clear())
 *)
 
     let toArray (seg:ArraySegment<'a>) =
@@ -83,7 +81,7 @@ module MessageSerialization =
     type OuterMessage () =
         inherit MessageBase()
         let m_id        = ref 0
-        let m_inner     = ref <| InnerMessage()
+        let m_inner     = ref None
         let m_hasMore   = ref false
 
         member x.ID         with get() = !m_id and set(v) = m_id := v
@@ -92,20 +90,20 @@ module MessageSerialization =
 
         override x.Clear() =
             m_id := 0
-            (!m_inner).Clear()
+            m_inner := None
             m_hasMore := false
 
         override x.DecoderRing =
             [  1, m_id      |> Serializer.hydrateInt32;
-              42, m_inner   |> Serializer.hydrateMessage (InnerMessage.FromArraySegment);
+              42, m_inner   |> Serializer.hydrateOptionalMessage (InnerMessage.FromArraySegment);
               43, m_hasMore |> Serializer.hydrateBool;
             ]
             |> Map.ofList
 
         override x.EncoderRing zcb =
             let encode =
                 (!m_id         |> Serializer.dehydrateVarint 1) >>
-                (!m_inner      |> Serializer.dehydrateMessage 42) >>
+                (!m_inner      |> Serializer.dehydrateOptionalMessage 42) >>
                 (!m_hasMore    |> Serializer.dehydrateBool 43)
             encode zcb
 
@@ -133,16 +131,17 @@ module MessageSerialization =
             |] |> ArraySegment
         let msg = OuterMessage.FromArraySegment(buf)
         msg.ID |> should equal 21
-        msg.Inner.ID |> should equal 99
-        msg.Inner.Name |> should equal "Test message"
+        msg.Inner.IsSome |> should equal true
+        msg.Inner.Value.ID |> should equal 99
+        msg.Inner.Value.Name |> should equal "Test message"
 
     [<Fact>]
     let ``Serialize compound message`` () =
         let msg = OuterMessage()
-        msg.Inner <- InnerMessage()
+        msg.Inner <- Some(InnerMessage())
         msg.ID <- 5
-        msg.Inner.ID <- 6
-        msg.Inner.Name <- "ABC0"
+        msg.Inner.Value.ID <- 6
+        msg.Inner.Value.Name <- "ABC0"
         msg.HasMore <- true
         msg.Serialize()
         |> toArray

Froto.Core.Test/TestSerializer.fs

@@ -48,7 +48,7 @@ module Utility =
     [<Fact>]
     let ``Length when encoded to Varint`` () =
 
-        varIntLen      0UL |> should equal 1
+        varIntLen      0UL |> should equal 0
         varIntLen   0x7FUL |> should equal 1
         varIntLen   0x80UL |> should equal 2
         varIntLen 0x3FFFUL |> should equal 2
@@ -298,6 +298,11 @@ module Serialize =
 
     [<Fact>]
     let ``Dehydrate integer varint`` () =
+        ZCB(2)
+        |> dehydrateVarint fid 0UL
+        |> toArray
+        |> should equal Array.empty
+
         ZCB(2)
         |> dehydrateVarint fid 2UL
         |> toArray
@@ -308,7 +313,7 @@ module Serialize =
         ZCB(2)
         |> dehydrateSInt32 fid 0
         |> toArray
-        |> should equal [| 0x08uy; 0uy |]
+        |> should equal Array.empty
 
         ZCB(2)
         |> dehydrateSInt32 fid -1
@@ -325,7 +330,7 @@ module Serialize =
         ZCB(2)
         |> dehydrateSInt64 fid 0L
         |> toArray
-        |> should equal [| 0x08uy; 0uy |]
+        |> should equal Array.empty
 
         ZCB(2)
         |> dehydrateSInt64 fid -1L
@@ -342,7 +347,7 @@ module Serialize =
         ZCB(2)
         |> dehydrateBool fid false
         |> toArray
-        |> should equal [| 0x08uy; 0uy |]
+        |> should equal Array.empty
 
         ZCB(2)
         |> dehydrateBool fid true
@@ -391,6 +396,32 @@ module Serialize =
         |> toArray
         |> should equal [| 0x08uy ||| 2uy; 4uy; 3uy; 4uy; 5uy; 6uy |]
 
+
+    let ``Dehydrate with default value`` () =
+        // Verify non-default value results in an actual value
+        ZCB(2)
+        |> dehydrateDefaultedVarint 0UL fid 2UL
+        |> toArray
+        |> should equal [| 0x08uy; 2uy |]
+
+        // Now, check that default value result in an empty array (value is elided)
+        let checkGetsElided f =
+            ZCB(16)
+            |> f
+            |> toArray
+            |> should equal Array.empty
+
+        checkGetsElided <| dehydrateDefaultedVarint 1UL fid 1UL
+        checkGetsElided <| dehydrateDefaultedSInt32 2 fid 2
+        checkGetsElided <| dehydrateDefaultedSInt64 3L fid 3L
+        checkGetsElided <| dehydrateDefaultedBool true fid true
+        checkGetsElided <| dehydrateDefaultedFixed32 4 fid 4
+        checkGetsElided <| dehydrateDefaultedFixed64 5L fid 5L
+        checkGetsElided <| dehydrateDefaultedSingle 0.60f fid 0.60f
+        checkGetsElided <| dehydrateDefaultedDouble 0.70 fid 0.70
+        checkGetsElided <| dehydrateDefaultedString "Hello" fid "Hello"
+        checkGetsElided <| dehydrateDefaultedBytes (ArraySegment([|8uy;9uy|])) fid (ArraySegment([|8uy;9uy|]))
+
     [<Fact>]
     let ``Dehydrate Packed Varint`` () =
         ZCB(8)