diff --git a/Core/EncodingTypes.fs b/Core/EncodingTypes.fs new file mode 100644 index 0000000..8b72667 --- /dev/null +++ b/Core/EncodingTypes.fs @@ -0,0 +1,36 @@ +(* + * Types used for encoding and decoding raw fields, at the wire format level. + *) +namespace Froto.Core.Encoding + +open System + +/// Protobuf field wire-type +type WireType = + | Varint = 0 + | Fixed64 = 1 + | LengthDelimited = 2 + | StartGroup = 3 + | EndGroup = 4 + | Fixed32 = 5 + +/// Protobuf field number +type FieldNum = int32 + +/// Raw unencoded wire-format field +type RawField = + | Varint of FieldNum * uint64 + | Fixed32 of FieldNum * uint32 + | Fixed64 of FieldNum * uint64 + | LengthDelimited of FieldNum * ArraySegment +// | Group is deprecated & not supported + with + member x.FieldNum = + match x with + | Varint(n,_) -> n + | Fixed32(n,_) -> n + | Fixed64(n,_) -> n + | LengthDelimited(n,_) -> n + // Need a place to hang this, because it cannot go on FieldNum + // (a type alias cannot have methods or properties). + static member MaxFieldNum = (2<<<28) - 1 diff --git a/Core/Exceptions.fs b/Core/Exceptions.fs new file mode 100644 index 0000000..0cb7b9d --- /dev/null +++ b/Core/Exceptions.fs @@ -0,0 +1,12 @@ +namespace Froto.Core + +/// Exception encoding or decoding a Protobuf wireformat + +type ProtobufException(message:string, ?innerException:exn) = + inherit System.ApplicationException( message, defaultArg innerException null ) + +type ProtobufWireFormatException (message:string, ?innerException:exn) = + inherit ProtobufException(message, defaultArg innerException null) + +type ProtobufSerializerException (message:string, ?innerException:exn) = + inherit ProtobufException(message, defaultArg innerException null) diff --git a/Core/Froto.Core.fsproj b/Core/Froto.Core.fsproj index 11f68cb..c8638f4 100644 --- a/Core/Froto.Core.fsproj +++ b/Core/Froto.Core.fsproj @@ -29,13 +29,19 @@ TRACE 3 + + + + + + + + + + - - - - - + \ No newline at end of file diff --git a/Core/Froto.Core.nuspec b/Core/Froto.Core.nuspec new file mode 100644 index 0000000..a833cc9 --- /dev/null +++ b/Core/Froto.Core.nuspec @@ -0,0 +1,18 @@ + + + + Froto.Core + 0.0.0 + Cameron Taggart, James Hugard + https://opensource.org/licenses/MIT + https://github.com/ctaggart/froto + false + A serailization library for Protocol Buffers. + @dependencies@ + froto protobuf binary protocol buffers serialization deserialization + + + + + + \ No newline at end of file diff --git a/Core/IO.fs b/Core/IO.fs deleted file mode 100644 index 516f6d0..0000000 --- a/Core/IO.fs +++ /dev/null @@ -1,302 +0,0 @@ -module Froto.IO - -open System -open System.IO - -let zigZag32 (n:int32) = (n <<< 1) ^^^ (n >>> 31) -let zigZag64 (n:int64) = (n <<< 1) ^^^ (n >>> 63) - -type WireType = - | Varint = 0 - | Fixed64 = 1 - | LengthDelimited = 2 - | StartGroup = 3 - | EndGroup = 4 - | Fixed32 = 5 - -//type ProtoType = -// | Double of double option -// | Float of float option -// | Int32 of int32 option -// | Int64 of int64 option -// | UInt32 of uint32 option -// | SInt32 of int32 option -// | SInt64 of int64 option -// | Fixed32 of uint32 option -// | Fixed64 of uint64 option -// | SFixed32 of int32 option -// | SFixed64 of int64 option -// | Bool of bool option -// | String of string option -// | Bytes of byte[] option - -let decodeVarintUInt32 (b:byte[]) : uint32 = - match b.Length with - | 0 -> 0u - | 1 -> - uint32 b.[0] - | 2 -> - let b0 = uint32 (b.[0] &&& 0x7Fuy) - let b1 = uint32 (b.[1]) - b0 ||| (b1 <<< 7) - | 3 -> - let b0 = uint32 (b.[0] &&& 0x7Fuy) - let b1 = uint32 (b.[1] &&& 0x7Fuy) - let b2 = uint32 (b.[2]) - b0 ||| (b1 <<< 7) ||| (b2 <<< 14) - | 4 -> - let b0 = uint32 (b.[0] &&& 0x7Fuy) - let b1 = uint32 (b.[1] &&& 0x7Fuy) - let b2 = uint32 (b.[2] &&& 0x7Fuy) - let b3 = uint32 (b.[3]) - b0 ||| (b1 <<< 7) ||| (b2 <<< 14) ||| (b3 <<< 21) - | _ -> // 5 - let b0 = uint32 (b.[0] &&& 0x7Fuy) - let b1 = uint32 (b.[1] &&& 0x7Fuy) - let b2 = uint32 (b.[2] &&& 0x7Fuy) - let b3 = uint32 (b.[3] &&& 0x7Fuy) - let b4 = uint32 (b.[4]) - b0 ||| (b1 <<< 7) ||| (b2 <<< 14) ||| (b3 <<< 21) ||| (b4 <<< 28) - -let decodeVarintUInt64 (b:byte[]) : uint64 = - match b.Length with - | 0 -> 0UL - | 1 | 2 | 3 | 4 -> uint64 (decodeVarintUInt32 b) - | 5 -> - let b0 = uint64 (b.[0] &&& 0x7Fuy) - let b1 = uint64 (b.[1] &&& 0x7Fuy) - let b2 = uint64 (b.[2] &&& 0x7Fuy) - let b3 = uint64 (b.[3] &&& 0x7Fuy) - let b4 = uint64 (b.[4]) - b0 ||| (b1 <<< 7) ||| (b2 <<< 14) ||| (b3 <<< 21) ||| (b4 <<< 28) - | 6 -> - let b0 = uint64 (b.[0] &&& 0x7Fuy) - let b1 = uint64 (b.[1] &&& 0x7Fuy) - let b2 = uint64 (b.[2] &&& 0x7Fuy) - let b3 = uint64 (b.[3] &&& 0x7Fuy) - let b4 = uint64 (b.[4] &&& 0x7Fuy) - let b5 = uint64 (b.[5]) - b0 ||| (b1 <<< 7) ||| (b2 <<< 14) ||| (b3 <<< 21) ||| (b4 <<< 28) ||| (b5 <<< 35) - | 7 -> - let b0 = uint64 (b.[0] &&& 0x7Fuy) - let b1 = uint64 (b.[1] &&& 0x7Fuy) - let b2 = uint64 (b.[2] &&& 0x7Fuy) - let b3 = uint64 (b.[3] &&& 0x7Fuy) - let b4 = uint64 (b.[4] &&& 0x7Fuy) - let b5 = uint64 (b.[5] &&& 0x7Fuy) - let b6 = uint64 (b.[6]) - b0 ||| (b1 <<< 7) ||| (b2 <<< 14) ||| (b3 <<< 21) ||| (b4 <<< 28) ||| (b5 <<< 35) ||| (b6 <<< 42) - | 8 -> - let b0 = uint64 (b.[0] &&& 0x7Fuy) - let b1 = uint64 (b.[1] &&& 0x7Fuy) - let b2 = uint64 (b.[2] &&& 0x7Fuy) - let b3 = uint64 (b.[3] &&& 0x7Fuy) - let b4 = uint64 (b.[4] &&& 0x7Fuy) - let b5 = uint64 (b.[5] &&& 0x7Fuy) - let b6 = uint64 (b.[6] &&& 0x7Fuy) - let b7 = uint64 (b.[7]) - b0 ||| (b1 <<< 7) ||| (b2 <<< 14) ||| (b3 <<< 21) ||| (b4 <<< 28) ||| (b5 <<< 35) ||| (b6 <<< 42) ||| (b7 <<< 49) - | 9 -> - let b0 = uint64 (b.[0] &&& 0x7Fuy) - let b1 = uint64 (b.[1] &&& 0x7Fuy) - let b2 = uint64 (b.[2] &&& 0x7Fuy) - let b3 = uint64 (b.[3] &&& 0x7Fuy) - let b4 = uint64 (b.[4] &&& 0x7Fuy) - let b5 = uint64 (b.[5] &&& 0x7Fuy) - let b6 = uint64 (b.[6] &&& 0x7Fuy) - let b7 = uint64 (b.[7] &&& 0x7Fuy) - let b8 = uint64 (b.[8]) - b0 ||| (b1 <<< 7) ||| (b2 <<< 14) ||| (b3 <<< 21) ||| (b4 <<< 28) ||| (b5 <<< 35) ||| (b6 <<< 42) ||| (b7 <<< 49) ||| (b8 <<< 56) - | _ -> // 10 - let b0 = uint64 (b.[0] &&& 0x7Fuy) - let b1 = uint64 (b.[1] &&& 0x7Fuy) - let b2 = uint64 (b.[2] &&& 0x7Fuy) - let b3 = uint64 (b.[3] &&& 0x7Fuy) - let b4 = uint64 (b.[4] &&& 0x7Fuy) - let b5 = uint64 (b.[5] &&& 0x7Fuy) - let b6 = uint64 (b.[6] &&& 0x7Fuy) - let b7 = uint64 (b.[7] &&& 0x7Fuy) - let b8 = uint64 (b.[8] &&& 0x7Fuy) - let b9 = uint64 (b.[9]) - b0 ||| (b1 <<< 7) ||| (b2 <<< 14) ||| (b3 <<< 21) ||| (b4 <<< 28) ||| (b5 <<< 35) ||| (b6 <<< 42) ||| (b7 <<< 49) ||| (b8 <<< 56) ||| (b8 <<< 63) - -let encodeVarintUInt32 (i:uint32) : byte[] = - if i < (2u <<< 6) then // fits in 7 bits - let b0 = byte (i) - [| b0 |] - else if i < (2u <<< 13) then // fits in 14 bits - let b0 = byte (i) ||| 0x80uy - let b1 = byte (i >>> 7) - [| b0; b1 |] - else if i < (2u <<< 20) then // fits in 21 bits - let b0 = byte (i) ||| 0x80uy - let b1 = byte (i >>> 7) ||| 0x80uy - let b2 = byte (i >>> 14) - [| b0; b1; b2 |] - else if i < (2u <<< 27) then // fits in 28 bits - let b0 = byte (i) ||| 0x80uy - let b1 = byte (i >>> 7) ||| 0x80uy - let b2 = byte (i >>> 14) ||| 0x80uy - let b3 = byte (i >>> 21) - [| b0; b1; b2; b3 |] - else // fits in 35 bits - let b0 = byte (i) ||| 0x80uy - let b1 = byte (i >>> 7) ||| 0x80uy - let b2 = byte (i >>> 14) ||| 0x80uy - let b3 = byte (i >>> 21) ||| 0x80uy - let b4 = byte (i >>> 28) - [| b0; b1; b2; b3; b4 |] - -let encodeVarintUInt64 (i:uint64) : byte[] = - if i < (2UL <<< 31) then // fits in 32 bits - encodeVarintUInt32 (uint32 i) - else if i < (2UL <<< 34) then // fits in 35 bits - let b0 = byte (i) ||| 0x80uy - let b1 = byte (i >>> 7) ||| 0x80uy - let b2 = byte (i >>> 14) ||| 0x80uy - let b3 = byte (i >>> 21) ||| 0x80uy - let b4 = byte (i >>> 28) - [| b0; b1; b2; b3; b4 |] - else if i < (2UL <<< 41) then // fits in 42 bits - let b0 = byte (i) ||| 0x80uy - let b1 = byte (i >>> 7) ||| 0x80uy - let b2 = byte (i >>> 14) ||| 0x80uy - let b3 = byte (i >>> 21) ||| 0x80uy - let b4 = byte (i >>> 28) ||| 0x80uy - let b5 = byte (i >>> 35) - [| b0; b1; b2; b3; b4; b5 |] - else if i < (2UL <<< 48) then // fits in 49 bits - let b0 = byte (i) ||| 0x80uy - let b1 = byte (i >>> 7) ||| 0x80uy - let b2 = byte (i >>> 14) ||| 0x80uy - let b3 = byte (i >>> 21) ||| 0x80uy - let b4 = byte (i >>> 28) ||| 0x80uy - let b5 = byte (i >>> 35) ||| 0x80uy - let b6 = byte (i >>> 42) - [| b0; b1; b2; b3; b4; b5; b6 |] - else if i < (2UL <<< 55) then // fits in 56 bits - let b0 = byte (i) ||| 0x80uy - let b1 = byte (i >>> 7) ||| 0x80uy - let b2 = byte (i >>> 14) ||| 0x80uy - let b3 = byte (i >>> 21) ||| 0x80uy - let b4 = byte (i >>> 28) ||| 0x80uy - let b5 = byte (i >>> 35) ||| 0x80uy - let b6 = byte (i >>> 42) ||| 0x80uy - let b7 = byte (i >>> 49) - [| b0; b1; b2; b3; b4; b5; b6; b7 |] - else if i < (2UL <<< 62) then // fits in 63 bits - let b0 = byte (i) ||| 0x80uy - let b1 = byte (i >>> 7) ||| 0x80uy - let b2 = byte (i >>> 14) ||| 0x80uy - let b3 = byte (i >>> 21) ||| 0x80uy - let b4 = byte (i >>> 28) ||| 0x80uy - let b5 = byte (i >>> 35) ||| 0x80uy - let b6 = byte (i >>> 42) ||| 0x80uy - let b7 = byte (i >>> 49) ||| 0x80uy - let b8 = byte (i >>> 56) - [| b0; b1; b2; b3; b4; b5; b6; b7; b8 |] - else // fits in 70 bits - let b0 = byte (i) ||| 0x80uy - let b1 = byte (i >>> 7) ||| 0x80uy - let b2 = byte (i >>> 14) ||| 0x80uy - let b3 = byte (i >>> 21) ||| 0x80uy - let b4 = byte (i >>> 28) ||| 0x80uy - let b5 = byte (i >>> 35) ||| 0x80uy - let b6 = byte (i >>> 42) ||| 0x80uy - let b7 = byte (i >>> 49) ||| 0x80uy - let b8 = byte (i >>> 56) ||| 0x80uy - let b9 = byte (i >>> 63) - [| b0; b1; b2; b3; b4; b5; b6; b7; b8; b9 |] - -let encodeVarintInt32 (i:int32) = encodeVarintUInt32 (uint32 i) -let encodeVarintInt64 (i:int64) = encodeVarintUInt64 (uint64 i) -let encodeVarintSInt32 (i:int32) = encodeVarintUInt32 (uint32 (zigZag32 i)) -let encodeVarintSInt64 (i:int64) = encodeVarintUInt64 (uint64 (zigZag64 i)) -let encodeVarintBool (b:bool) = encodeVarintUInt32 (if b then 1u else 0u) - -let encodeUInt32 (i:uint32) = - let b0 = byte i - let b1 = byte (i >>> 8) - let b2 = byte (i >>> 16) - let b3 = byte (i >>> 24) - [| b0; b1; b2; b3 |] - -let encodeInt32 (i:int32) = encodeUInt32 (uint32 i) - -let decodeUInt32 (b:byte[]) = - let b0 = uint32 b.[0] - let b1 = uint32 b.[1] - let b2 = uint32 b.[2] - let b3 = uint32 b.[3] - b0 ||| (b1 <<< 8) ||| (b2 <<< 16) ||| (b3 <<< 24) - -let decodeInt32 b = int32 (decodeUInt32 b) - -//type Varint = -// | Int32 of int32 -// | Int64 of int64 -// | UInt32 of uint32 -// | UInt64 of uint64 -// | SInt32 of int32 -// | SInt64 of int64 -// | Bool of bool -// -//let encodeVarint (i:Varint) : byte[] = -// [||] - -// a tag is a field number and a wire type -/// gets the field number from a tag -let parseTag (b:byte[]) = - let i = decodeVarintUInt32 b - let fieldNumber = i >>> 3 - let wireType = i &&& 0x7u - fieldNumber, wireType - -let writeBytes (s:Stream) (b:byte[]) = - s.Write(b, 0, b.Length) - -let writeTag s (fieldNumber:uint32) (wireType:WireType) = - let tag = (fieldNumber <<< 3) ||| (uint32 wireType) - writeBytes s (encodeVarintUInt32 tag) - -let writeFieldString s n (v:string) = - writeTag s n WireType.LengthDelimited - let b = Text.Encoding.UTF8.GetBytes v - writeBytes s (encodeVarintUInt32 (uint32 b.Length)) - writeBytes s b - -//let readFieldString (s:Stream) - -let writeFieldInt32 s n (v:int32) = - writeTag s n WireType.Fixed32 - writeBytes s (encodeInt32 v) - -let writeFieldUInt32 s n (v:uint32) = - writeTag s n WireType.Fixed32 - writeBytes s (encodeUInt32 v) - -let readFieldUInt32 (s:Stream) = - let mutable b = Array.zeroCreate 4 - s.Read(b, 0, 4) |> ignore - decodeUInt32 b - -/// Make the stream a sequence of bytes. -let streamAsBytes (s:Stream) = - seq { - let i = ref 0 - while (i := s.ReadByte(); !i <> -1) do - yield byte !i - } - -// Is the most significant bit set? -let msb (b:byte) = Convert.ToBoolean (b &&& 0x80uy) - -let readVarintBytes (s:Stream) = - let bytes = seq { - use e = (streamAsBytes s).GetEnumerator() - let b = ref 0uy - while e.MoveNext() && (b := e.Current; msb !b) do - yield !b - yield !b - } - Seq.toArray bytes diff --git a/Core/Serializer.fs b/Core/Serializer.fs new file mode 100644 index 0000000..1ad92ca --- /dev/null +++ b/Core/Serializer.fs @@ -0,0 +1,576 @@ +namespace Froto.Core.Encoding + +open System +open Froto.Core +open Froto.Core.WireFormat + +/// +/// Utility functions used by the serializer +/// +module Utility = + + /// Encode SInt32 + let zigZag32 (n:int32) = (n <<< 1) ^^^ (n >>> 31) + /// Encode SInt64 + let zigZag64 (n:int64) = (n <<< 1) ^^^ (n >>> 63) + + /// Decode SInt32 + let zagZig32 (n:int32) = int32(uint32 n >>> 1) ^^^ (if n&&&1 = 0 then 0 else -1 ) + /// Decode SInt64 + let zagZig64 (n:int64) = int64(uint64 n >>> 1) ^^^ (if n&&&1L = 0L then 0L else -1L) + + /// Calculate length when encoded as a Varint; if value is default, then length is 0 + let varIntLenDefaulted d (v:uint64) = + if v = d + then 0 + else + let rec loop acc len = + let bMore = acc > 0x7FUL + if bMore + then loop (acc >>> 7) (len+1) + else len + loop v 1 + + /// Calculate length when encoded as a Varint; if value is default, then length is 0 + let varIntLen (v:uint64) = + varIntLenDefaulted 0UL v + + /// Calculate length when encoded as a Varint without a default; e.g., a packed field + let varIntLenNoDefault (v:uint64) = + let rec loop acc len = + let bMore = acc > 0x7FUL + if bMore + then loop (acc >>> 7) (len+1) + else len + loop v 1 + + + /// Calculate field number length when encoded in a tag + let tagLen (t:int32) = + varIntLenNoDefault ((uint64 t) <<< 3) + + /// Apply a function to a Option value, if the value is Some. + let inline IfSome f opt = + match opt with + | None -> id + | Some(v) -> f v + + + +/// +/// Serialization and Deserialization. +/// +/// This module contains support for writing (or generating) code which +/// performs deserialization (hydration) and serialization (dehydration) of +/// individual properties. These can be used directly to create methods +/// on classes, records, discriminated unions, etc. +/// +/// In addition, an abstract base-class (MessageBase) is provided which +/// defines a simple DSL for constructing Classes which model Protobuf +/// Messages. +/// +module Serializer = + + let inline internal flip f a b = f b a + let inline internal toBool (u:uint64) = not (u=0UL) + let inline internal fromBool b = if b then 1UL else 0UL + + let internal raiseMismatch expected actual = + let extractNumAndType = function + | RawField.Varint (n,_) -> n, "Varint" + | RawField.Fixed32 (n,_) -> n, "Fixed32" + | RawField.Fixed64 (n,_) -> n, "Fixed64" + | RawField.LengthDelimited (n,_) -> n, "LengthDelimited" + let (n, found) = actual |> extractNumAndType + let s = sprintf "Deserialize failure: wiretype mismatch for field %d: expected %s, found %s" n expected found + raise <| ProtobufSerializerException(s) + +//---- Deserialization + + /// Helper to deserialize from Varint. + /// Since this is used by an inline function (hydrateEnum), + /// it cannot be marked "internal". + let helper_vi f fld = function + | RawField.Varint (n, v) -> + fld := f v + | raw -> + raiseMismatch "Varint" raw + + let hydrateInt32 = helper_vi int32 + let hydrateInt64 = helper_vi int64 + let hydrateUInt32 = helper_vi uint32 + let hydrateUInt64 = helper_vi uint64 + let hydrateSInt32 = helper_vi (int32 >> Utility.zagZig32) + let hydrateSInt64 = helper_vi (int64 >> Utility.zagZig64) + let hydrateBool = helper_vi toBool + let inline hydrateEnum x = helper_vi (int32 >> enum) x + + /// Helper to deserialize from Fixed32 + let internal helper_fx32 f fld = function + | RawField.Fixed32 (_, v) -> + fld := f v + | raw -> + raiseMismatch "Fixed32" raw + + let hydrateFixed32 = helper_fx32 uint32 + let hydrateSFixed32 = helper_fx32 int32 + let hydrateSingle = + let aux (u:uint32) = + // TODO: eliminate the Array allocation, + // perhaps using CIL (MSIL) to load float from a register + let bytes = BitConverter.GetBytes(u) + if not BitConverter.IsLittleEndian then Array.Reverse bytes + BitConverter.ToSingle(bytes,0) + helper_fx32 aux + + /// Helper to deserialize from Fixed64 + let internal helper_fx64 f fld = function + | RawField.Fixed64 (_, v) -> + fld := f v + | raw -> + raiseMismatch "Fixed64" raw + + let hydrateFixed64 = helper_fx64 uint64 + let hydrateSFixed64 = helper_fx64 int64 + let hydrateDouble = + let aux (u:uint64) = + // TODO: eliminate the Array allocation, + // perhaps using CIL (MSIL) to load float from a register + let bytes = BitConverter.GetBytes(u) + if not BitConverter.IsLittleEndian then Array.Reverse bytes + BitConverter.ToDouble(bytes,0) + helper_fx64 aux + + + /// Helper to deserialize from a LengthDelimited + let internal helper_bytes f fld = function + | RawField.LengthDelimited (_, v) -> + fld := f v + | raw -> + raiseMismatch "LengthDelimited" raw + + let internal toString (a:System.ArraySegment) = + let utf8 = System.Text.Encoding.UTF8 + utf8.GetString(a.Array, a.Offset, a.Count) + + let internal toByteArray (a:System.ArraySegment) = + a.Array.[ a.Offset .. a.Offset + (a.Count-1)] + + let hydrateString = helper_bytes toString + let hydrateBytes = helper_bytes toByteArray + let hydrateMessage messageCtor = helper_bytes messageCtor + let hydrateOptionalMessage messageCtor = hydrateMessage (messageCtor >> Some) + + /// Helper to deserialize Packed Repeated from LengthDelimited. + /// Since this is used by an inline function (hydratePackedEnum), + /// it cannot be marked "internal". + let helper_packed f fld = function + | RawField.LengthDelimited (_,v) -> + fld := + [ + let s = ZeroCopyReadBuffer(v) + while not s.IsEof do + yield (f s) + ] + | raw -> + raiseMismatch "LengthDelimited" raw + + let hydratePackedInt32 = helper_packed (decodeVarint >> int32) + let hydratePackedInt64 = helper_packed (decodeVarint >> int64) + let hydratePackedUInt32 = helper_packed (decodeVarint >> uint32) + let hydratePackedUInt64 = helper_packed (decodeVarint >> uint64) + let hydratePackedSInt32 = helper_packed (decodeVarint >> int32 >> Utility.zagZig32) + let hydratePackedSInt64 = helper_packed (decodeVarint >> int64 >> Utility.zagZig64) + let hydratePackedBool = helper_packed (decodeVarint >> toBool) + let inline hydratePackedEnum x = helper_packed (decodeVarint >> int32 >> enum) x + let hydratePackedFixed32 = helper_packed decodeFixed32 + let hydratePackedFixed64 = helper_packed decodeFixed64 + let hydratePackedSFixed32 = helper_packed (decodeFixed32 >> int32) + let hydratePackedSFixed64 = helper_packed (decodeFixed64 >> int64) + let hydratePackedSingle = helper_packed decodeSingle + let hydratePackedDouble = helper_packed decodeDouble + +//---- Serialization + + /// If value = default, then elide the field (don't serialize) + let inline elided d v f = + if v = d + then id + else f + + /// Generic Dehydrate for all varint types, excepted for signed & bool: + /// int32, int64, uint32, uint64, enum + let inline dehydrateDefaultedVarint d fldNum v = elided d v <| WireFormat.encodeFieldVarint fldNum (uint64 v) + let inline dehydrateNondefaultedVarint fldNum v = WireFormat.encodeFieldVarint fldNum (uint64 v) + + let dehydrateDefaultedSInt32 d fldNum v = elided d v <| WireFormat.encodeFieldVarint fldNum (Utility.zigZag32 v |> uint64) + let dehydrateDefaultedSInt64 d fldNum v = elided d v <| WireFormat.encodeFieldVarint fldNum (Utility.zigZag64 v |> uint64) + let dehydrateDefaultedBool d fldNum v = elided d v <| dehydrateNondefaultedVarint fldNum (fromBool v) + + let inline dehydrateDefaultedFixed32 d fldNum v = elided d v <| WireFormat.encodeFieldFixed32 fldNum (uint32 v) + let inline dehydrateDefaultedFixed64 d fldNum v = elided d v <| WireFormat.encodeFieldFixed64 fldNum (uint64 v) + + let dehydrateDefaultedSingle d fldNum v = elided d v <| WireFormat.encodeFieldSingle fldNum v + let dehydrateDefaultedDouble d fldNum v = elided d v <| WireFormat.encodeFieldDouble fldNum v + + let dehydrateDefaultedString d fldNum v = elided d v <| WireFormat.encodeFieldString fldNum v + let dehydrateDefaultedBytes d fldNum v = elided d v <| WireFormat.encodeFieldBytes fldNum v + + let inline dehydrateVarint fldNum (v:'a) = dehydrateDefaultedVarint (Unchecked.defaultof<'a>) fldNum v + + let dehydrateSInt32 fldNum v = dehydrateDefaultedSInt32 0 fldNum v + let dehydrateSInt64 fldNum v = dehydrateDefaultedSInt64 0L fldNum v + let dehydrateBool fldNum v = dehydrateDefaultedBool false fldNum v + + let inline dehydrateFixed32 fldNum v = dehydrateDefaultedFixed32 0 fldNum v + let inline dehydrateFixed64 fldNum v = dehydrateDefaultedFixed64 0 fldNum v + + let dehydrateSingle fldNum v = dehydrateDefaultedSingle 0.0f fldNum v + let dehydrateDouble fldNum v = dehydrateDefaultedDouble 0.0 fldNum v + + let dehydrateString fldNum v = dehydrateDefaultedString "" fldNum v + let dehydrateBytes fldNum v = dehydrateDefaultedBytes (ArraySegment ([||]:byte array)) fldNum v + + + (* Dehydrate Repeated Packed Numeric Values *) + + let dehydratePackedHelper lenFn encFn fieldNum xs = + let xslen = xs + |> lenFn + |> uint64 + WireFormat.encodeTag fieldNum WireType.LengthDelimited + >> WireFormat.encodeVarint xslen + >> flip (List.fold (fun buf x -> buf |> encFn x )) xs + + let inline varIntListPackedLen encode (xs:'a list) = + List.sumBy (encode >> Utility.varIntLenNoDefault) xs + + /// Generic Dehydrate for all packed varint types, excepted for bool & signed: + /// int32, int64, uint32, uint64, enum + let inline dehydratePackedVarint fieldNum xs = + let encode = uint64 + dehydratePackedHelper + (varIntListPackedLen encode) + (encode >> WireFormat.encodeVarint) + fieldNum xs + + let dehydratePackedBool fieldNum xs = + let boolPackedLen = List.length + dehydratePackedHelper + boolPackedLen (* encodes to 1 byte per bool *) + (fromBool >> WireFormat.encodeVarint) + fieldNum xs + + let dehydratePackedSInt32 fieldNum xs = + let encode = Utility.zigZag32 >> uint64 + dehydratePackedHelper + (varIntListPackedLen encode) + (encode >> WireFormat.encodeVarint) + fieldNum xs + + let dehydratePackedSInt64 fieldNum xs = + let encode = Utility.zigZag64 >> uint64 + dehydratePackedHelper + (varIntListPackedLen encode) + (encode >> WireFormat.encodeVarint) + fieldNum xs + + let inline fixedListPackedLen size = (List.length >> ((*) size)) + let inline fixed32ListPackedLen xs = fixedListPackedLen 4 xs + let inline fixed64ListPackedLen xs = fixedListPackedLen 8 xs + + let inline dehydratePackedFixed32 fieldNum xs = + dehydratePackedHelper + fixed32ListPackedLen + (uint32 >> WireFormat.encodeFixed32) + fieldNum xs + + let inline dehydratePackedFixed64 fieldNum xs = + dehydratePackedHelper + fixed64ListPackedLen + (uint64 >> WireFormat.encodeFixed64) + fieldNum xs + + let dehydratePackedSingle fieldNum xs = + dehydratePackedHelper + fixed32ListPackedLen + WireFormat.encodeSingle + fieldNum xs + + let dehydratePackedDouble fieldNum xs = + dehydratePackedHelper + fixed64ListPackedLen + WireFormat.encodeDouble + fieldNum xs + + (* Dehydrate Message *) + let inline dehydrateMessage fieldNum (o:^msg when ^msg : (member SerializeLengthDelimited : ZeroCopyWriteBuffer -> ZeroCopyWriteBuffer)) = + let serializeMsg zcb = (^msg : (member SerializeLengthDelimited : ZeroCopyWriteBuffer -> ZeroCopyWriteBuffer) (o,zcb)) + WireFormat.encodeTag fieldNum WireType.LengthDelimited + >> serializeMsg + + let inline dehydrateOptionalMessage fieldNum (o:^msg option when ^msg : (member SerializeLengthDelimited : ZeroCopyWriteBuffer -> ZeroCopyWriteBuffer)) = + o |> Utility.IfSome (fun o -> dehydrateMessage fieldNum o) + + (* Repeated Field Helpers *) + let hydrateRepeated<'a> (hydrater:'a ref -> RawField -> unit) propRef rawField = + let element = Unchecked.defaultof<'a> + hydrater (ref element) rawField + propRef := element :: !propRef + + let dehydrateRepeated<'a> (dehydrater:FieldNum -> 'a -> ZeroCopyWriteBuffer -> ZeroCopyWriteBuffer) (fldNum:int32) (vs:'a list) : (ZeroCopyWriteBuffer -> ZeroCopyWriteBuffer) = + let dh = flip (dehydrater fldNum) + let wrapperFn (zcb:ZeroCopyWriteBuffer) = + vs + |> List.iter (dh zcb >> ignore) + zcb + wrapperFn + +/// +/// Abstract base class for Protobuf-serializable Messages. +/// +/// This class provides a simple DSL for deriving your own serializable +/// classes. Such classes are best written in F#, due to the strong +/// dependance on function currying (partial function application), +/// especially in the DecoderRing and EncoderRing properties. +/// +[] +type MessageBase () = + + let mutable m_unknownFields = List.empty + + let asArraySegment (zcb:ZeroCopyWriteBuffer) = + zcb.AsArraySegment + + let remainder (zcb:ZeroCopyReadBuffer) = + zcb.Remainder + + /// Derrived classes must provide a Clear function, which resets all + /// members to their default values; generally, Zero(0) or Empty. + abstract Clear : Unit -> Unit + + /// Derrived classes must provide a DecoderRing property, which is used + /// to map from a field number to a deserialization (hydration) function. + /// This is a map, because Protobuf fields can appear in any order. + abstract DecoderRing : Mapunit)> + + /// Derrived classes must provide an EncoderRing property, which is used + /// to serialize the class. Generally, this single function chains + /// the serialization functions for all known members. + abstract EncoderRing : ZeroCopyWriteBuffer -> ZeroCopyWriteBuffer + + /// Derrived classes MAY provide a list of required fields; when either + /// merging or deserializing, all of these fields MUST be present in the + /// serialized message, otherwise an exception will be thrown. + abstract RequiredFields : Set + override x.RequiredFields = Set.empty + + /// List of fields provided in the protobuf, but which were not found on + /// the DecoderRing. All these fields will be serialized to the buffer + /// after all fields on the EncoderRing. + member x.UnknownFields + with get() = m_unknownFields + and set(v) = m_unknownFields <- v + + // Internal helper + member private x.MergeWhile (predicate:ZeroCopyReadBuffer->bool) (zcb:ZeroCopyReadBuffer) = + if Set.isEmpty x.RequiredFields then + seq { + while predicate zcb do + yield WireFormat.decodeField zcb + } + |> Seq.iter x.DeserializeField + zcb + else + let mutable foundFields = Set.empty + seq { + while predicate zcb do + let rawField = WireFormat.decodeField zcb + let fieldId = rawField.FieldNum + foundFields <- foundFields |> Set.add fieldId + yield rawField + } + |> Seq.iter x.DeserializeField + let missingFields = (x.RequiredFields - foundFields) + if Set.isEmpty missingFields + then zcb + else raise <| ProtobufSerializerException(sprintf "Missing required fields %A" missingFields) + + + /// Merge from a serialized buffer. + /// + /// Does not Clear() the object before merging, so fields which do + /// not appear in the buffer will remain untouched, and repeated + /// fields in the buffer will be added to any existing values. + /// + /// The entire buffer will be consumed, so must be of the right + /// length to exactly contain the message. + member x.Merge (zcb:ZeroCopyReadBuffer) : ZeroCopyReadBuffer = + zcb + |> x.MergeWhile (fun zcb -> not zcb.IsEof) + + /// Merge from a buffer whose first value is a varint + /// specifying the length of the message. + /// + /// Does not Clear() the object before merging, so fields which do + /// not appear in the buffer will remain untouched, and repeated + /// fields in the buffer will be added to any existing values. + /// + /// Returns the remaining bytes in the buffer as an ArraySegment. + member x.MergeLengthDelimited (zcb:ZeroCopyReadBuffer) = + let len = zcb |> WireFormat.decodeVarint |> uint32 + let end_ = zcb.Position + len + zcb + |> x.MergeWhile (fun zcb -> zcb.Position < end_) + + /// Merge from an ArraySegment. + /// + /// Does not Clear() the object before merging, so fields which do + /// not appear in the buffer will remain untouched, and repeated + /// fields in the buffer will be added to any existing values. + /// + /// The entire ArraySegment will be consumed, so must be of the right + /// length to exactly contain the message. + member x.Merge (buf:System.ArraySegment) = + ZeroCopyReadBuffer(buf) + |> x.Merge + |> remainder + + /// Merge from an ArraySegment whose first value is a varint + /// specifying the length of the message. + /// + /// Does not Clear() the object before merging, so fields which do + /// not appear in the buffer will remain untouched, and repeated + /// fields in the buffer will be added to any existing values. + /// + /// Returns the remaining bytes in the buffer as an ArraySegment. + member x.MergeLengthDelimited (buf:System.ArraySegment) = + ZeroCopyReadBuffer(buf) + |> x.MergeLengthDelimited + |> remainder + + /// Deserialize from a serialized buffer. + /// + /// Clear()'s the object before deserializing. + /// + /// The entire buffer will be consumed, so must be of the right + /// length to exactly contain the message. + member x.Deserialize (zcb:ZeroCopyReadBuffer) : ZeroCopyReadBuffer = + x.Clear() + x.Merge(zcb) + + /// Deserialize from a buffer whose first value is a varint + /// specifying the length of the message. + /// + /// Clear()'s the object before deserializing. + /// + /// Returns the remaining bytes in the buffer as an ArraySegment. + member x.DeserializeLengthDelimited (zcb:ZeroCopyReadBuffer) = + x.Clear() + x.MergeLengthDelimited(zcb) + + /// Deserialize from an ArraySegment. + /// + /// Clear()'s the object before deserializing. + /// + /// The entire ArraySegment will be consumed, so must be of the right + /// length to exactly contain the message. + member x.Deserialize (buf:System.ArraySegment) = + x.Clear() + x.Deserialize(buf) + + /// Deserialize from an ArraySegment whose first value is a varint + /// specifying the length of the message. + /// + /// Clear()'s the object before deserializing. + /// + /// Returns the remaining bytes in the buffer as an ArraySegment. + member x.DeserializeLengthDelimited (buf:System.ArraySegment) = + x.Clear() + x.DeserializeLengthDelimited(buf) + + /// Return number of bytes needed to serialize the object + /// + /// @see SerializedLengthDelimitedLength + member x.SerializedLength = + let ncb = NullWriteBuffer() + ncb |> x.Serialize |> ignore + ncb.Length + + /// Serialize the object by applying all functions on the EncoderRing. + /// + /// Will also serialize all fields stored on the UnknownFields list. + member x.Serialize (zcb:ZeroCopyWriteBuffer) = + zcb + |> x.EncoderRing + |> x.SerializeUnknownFields + + /// Serialize the object by applying all functions on the EncoderRing. + /// + /// Will also serialize all fields stored on the UnknownFields list. + member x.Serialize (buf:System.ArraySegment) = + ZeroCopyWriteBuffer(buf) + |> x.Serialize + |> asArraySegment + + /// Serialize to a new byte array, and return as an ArraySegment. + member x.Serialize() = + Array.zeroCreate (int32 x.SerializedLength) + |> ArraySegment + |> x.Serialize + + /// Return number of bytes needed to serialize both the object and the + /// length of the object as a varint. + member x.SerializedLengthDelimitedLength = + let len = x.SerializedLength + let lenlen = Utility.varIntLenNoDefault (uint64 len) + (uint32 lenlen) + len + + /// Serialize first the length as a varint, followed by the serialized + /// object. + member x.SerializeLengthDelimited (zcb:ZeroCopyWriteBuffer) = + zcb + |> WireFormat.encodeVarint (uint64 x.SerializedLength) + |> x.Serialize + + /// Serialize first the length as a varint, followed by the serialized + /// object. + member x.SerializeLengthDelimited (buf:System.ArraySegment) = + ZeroCopyWriteBuffer(buf) + |> x.SerializeLengthDelimited + |> asArraySegment + + /// Serialize, to a new byte array, the length as a varint followed by + /// the serialized object, and return as an ArraySegment. + member x.SerializeLengthDelimited() = + Array.zeroCreate (int32 x.SerializedLengthDelimitedLength) + |> ArraySegment + |> x.SerializeLengthDelimited + + /// Deserialize a single field, using the DecoderRing. + /// If the field number is not on the DecoderRing, then store the + /// RawField on the UnknownFields list. + member private x.DeserializeField (field:Encoding.RawField) = + let n = field.FieldNum + match x.DecoderRing |> Map.tryFind n with + | Some(deserializeFn) -> deserializeFn field + | None -> m_unknownFields <- field :: m_unknownFields + + /// Serialize all unknown fields + member private x.SerializeUnknownFields (zcb:ZeroCopyWriteBuffer) = + + let inline emplace (src:ArraySegment) (dst:ArraySegment) = + Array.Copy(src.Array, src.Offset, dst.Array, dst.Offset, src.Count) + + for field in x.UnknownFields do + match field with + | RawField.Varint (n,v) -> + zcb |> WireFormat.encodeFieldVarint n v + | RawField.LengthDelimited (n,v) -> + zcb |> WireFormat.encodeFieldLengthDelimited n (v.Count|>uint32) (emplace v) + | RawField.Fixed32 (n,v) -> + zcb |> WireFormat.encodeFieldFixed32 n v + | RawField.Fixed64 (n,v) -> + zcb |> WireFormat.encodeFieldFixed64 n v + |> ignore + zcb diff --git a/Core/WireFormat.fs b/Core/WireFormat.fs new file mode 100644 index 0000000..9feff84 --- /dev/null +++ b/Core/WireFormat.fs @@ -0,0 +1,233 @@ +module Froto.Core.WireFormat + +open System +open Froto.Core.Encoding + +/// Maximum length of a length-delimited field (64Mb) +let MAX_FIELD_LEN = 64u * 1024u * 1024u + + +/// Decode a varint-encoded uint64 (1-9 bytes) +let decodeVarint (src:ZeroCopyReadBuffer) = + + // note: can't remember if using the outer `src` will cause a new + // lambda to be created (on each call) to capture `src`, so be + // safe and pass it in as a parameter. + let getNext (src:ZeroCopyReadBuffer) = + let b = src.ReadByte() + let bMore = (b &&& 0x80uy) <> 0uy + let lower7 = uint64 (b &&& 0x7Fuy) + (bMore,lower7) + + let rec loop acc n (src:ZeroCopyReadBuffer) = + let maxLen = 9 + if n <= maxLen then + let bMore,lower7 = getNext src + let acc = (lower7 <<< (7 * n)) ||| acc + if bMore then + loop acc (n+1) src + else + acc + else + raise <| ProtobufWireFormatException( "Unterminated VarInt" ) + + loop 0UL 0 src + +let rec internal decodeFixedLoop acc len n (src:ZeroCopyReadBuffer) = + if n < len + then + let byt = src.ReadByte() + let acc = (uint64 byt <<< (8*n)) ||| acc + decodeFixedLoop acc len (n+1) src + else + acc + +/// Decode fixed-size uint32 (8-bytes) +let decodeFixed32 src = uint32 (decodeFixedLoop 0UL 4 0 src) + +/// Decode fized-size uint64 (8-bytes) +let decodeFixed64 src = uint64 (decodeFixedLoop 0UL 8 0 src) + +/// Decode fixed-size float (4-bytes) +let decodeSingle src = + let u = decodeFixed32 src + // TODO: eliminate the Array allocation, + // perhaps using CIL (MSIL) to load float from a register + let bytes = BitConverter.GetBytes(u) + if not BitConverter.IsLittleEndian then Array.Reverse bytes + BitConverter.ToSingle(bytes,0) + +/// Decode fized-size double (8-bytes) +let decodeDouble src = + let u = decodeFixed64 src + // TODO: eliminate the Array allocation, + // perhaps using CIL (MSIL) to load float from a register + let bytes = BitConverter.GetBytes(u) + if not BitConverter.IsLittleEndian then Array.Reverse bytes + BitConverter.ToDouble(bytes,0) + +/// Decode length delimited field [0,MAX_FIELD_LEN) bytes. +/// +/// NOTE: Returns an ArraySegement which references a section of the +/// underlying source Array, rather than a copy. Changes to that source +/// Array will therefore impact this function's returned value. +let decodeLengthDelimited src = + let len = decodeVarint src + if len < uint64 MAX_FIELD_LEN then + src.ReadByteSegment(uint32 len) + else + raise <| ProtobufWireFormatException( "Maximum field length exceeded" ) + +/// Encode uint64 as a varint (1-9 bytes) +let encodeVarint (u:uint64) (dest:ZeroCopyWriteBuffer) = + + let rec loop acc (dest:ZeroCopyWriteBuffer) = + let bMore = acc > 0x7FUL + let lower7 = byte ( acc &&& 0x7FUL ) + let b = if not bMore + then lower7 + else lower7 ||| 0x80uy + dest.WriteByte b + if bMore + then loop (acc >>> 7) dest + else () + + loop u dest + dest + +/// Encode uint32 as fixed32 (4 bytes) +let encodeFixed32 (u:uint32) (dest:ZeroCopyWriteBuffer) = + dest.WriteByte (byte u) + dest.WriteByte (byte (u >>> 8)) + dest.WriteByte (byte (u >>> 16)) + dest.WriteByte (byte (u >>> 24)) + dest + +/// Encode uint64 as fixed64 (8 bytes) +let encodeFixed64 (u:uint64) dest = + dest + |> encodeFixed32 (uint32 u) + |> encodeFixed32 (uint32 (u >>> 32)) + +/// Encode float as fixed32 (4 bytes) +let encodeSingle (f:single) = + let bytes = BitConverter.GetBytes(f) + if not BitConverter.IsLittleEndian then Array.Reverse bytes + let u = BitConverter.ToUInt32(bytes,0) + encodeFixed32 u + +/// Encode double as fixed64 (8-bytes) +let encodeDouble (d:double) = + let bytes = BitConverter.GetBytes(d) + if not BitConverter.IsLittleEndian then Array.Reverse bytes + let UL = BitConverter.ToUInt64(bytes,0) + encodeFixed64 UL + +/// Encode [0,MAX_FIELD_LEN) bytes as length delimited field +/// +/// +/// Specifies the length of data to be encoded. This is needed up-front, +/// because the length is written as a varint (variable number of bytes) +/// before the data bytes. +/// +/// +/// Function used to copy or emplace data directly into the output buffer, +/// rather than requiring copies to be passed around. Especially useful to +/// encode UTF-8 strings directly into the output buffer. +/// +/// +/// Destination buffer. +/// Generally, this parameter is omitted when defining a set of serialization +/// functions; several serialization functions are then chained together to +/// produce a single function which serializes the entire Message. +/// +let encodeLengthDelimited (len:uint32) (emplace:ArraySegment->unit) (dest:ZeroCopyWriteBuffer) = + dest |> encodeVarint (uint64 len) |> ignore + dest.WriteByteSegment len emplace + dest + +/// Decode the field number and wire-type from a tag. +/// Note: a tag is a field number with a wire type packed in the lower 3 bits. +let decodeTag src = + let u = decodeVarint src + let fieldNum = u >>> 3 + // theoretic range is [0, UInt64.MaxValue>>>3], + // but descriptor.proto defines it as an int32 + // in the range [1,2^28) + if fieldNum > 0UL && fieldNum <= uint64 RawField.MaxFieldNum then + let fieldNum = int32 fieldNum + let wireType = enum (int32 u &&& 0x07) + (fieldNum, wireType) + else + raise <| ProtobufWireFormatException("Decode failure: field number must be in range [1, 2^28)") + +/// Decode a field into a RawField Discriminated Union. +let decodeField src = + let fieldNum, wireType = decodeTag src + match wireType with + | WireType.Varint -> + Varint (fieldNum, decodeVarint src) + | WireType.Fixed64 -> + Fixed64 (fieldNum, decodeFixed64 src) + | WireType.LengthDelimited -> + LengthDelimited (fieldNum, decodeLengthDelimited src) + | WireType.Fixed32 -> + Fixed32 (fieldNum, decodeFixed32 src) + + | WireType.StartGroup + | WireType.EndGroup + | _ -> raise <| ProtobufWireFormatException(sprintf "Decode failure: unsupported wiretype: %A" wireType) + +/// Encode a Tag (consisting of a field number and wire-type) as a single varint +let encodeTag (fieldNum:int32) (wireType:WireType) = + if fieldNum > 0 && fieldNum <= RawField.MaxFieldNum then + let tag = (fieldNum <<< 3) ||| (int32 wireType) + encodeVarint (uint64 tag) + else + raise <| ProtobufWireFormatException("Encode failure: field numbeer must be in range [1, 2^28)") + +/// Encode a varint-based field +let encodeFieldVarint (fieldNum:int32) (u:uint64) = + encodeTag fieldNum WireType.Varint + >> encodeVarint u + +/// Encode a fixed32-based field +let encodeFieldFixed32 (fieldNum:int32) (u:uint32) = + encodeTag fieldNum WireType.Fixed32 + >> encodeFixed32 u + +/// Encode a fixed64-based field +let encodeFieldFixed64 (fieldNum:int32) (u:uint64) = + encodeTag fieldNum WireType.Fixed64 + >> encodeFixed64 u + +/// Encode a floating point single field +let encodeFieldSingle (fieldNum:int32) (f:single) = + encodeTag fieldNum WireType.Fixed32 + >> encodeSingle f + +/// Encode a floating point double field +let encodeFieldDouble (fieldNum:int32) (d:double) = + encodeTag fieldNum WireType.Fixed64 + >> encodeDouble d + +/// Encode a length delimited field +let encodeFieldLengthDelimited (fieldNum:int32) len (emplace:ArraySegment->unit) = + encodeTag fieldNum WireType.LengthDelimited + >> encodeLengthDelimited len emplace + +/// Encode a bytes field +let encodeFieldBytes (fieldNum:int32) (source:ArraySegment) = + encodeFieldLengthDelimited + (int32 fieldNum) + (uint32 source.Count) + (fun dest -> Array.Copy( source.Array, source.Offset, dest.Array, dest.Offset, source.Count)) + +/// Encode a string field +let encodeFieldString fieldNum (s:string) = + let utf8 = System.Text.Encoding.UTF8 + let len = utf8.GetByteCount(s) |> uint32 + encodeFieldLengthDelimited fieldNum + len + (fun dest -> utf8.GetBytes( s, 0, s.Length, dest.Array, dest.Offset) |> ignore) + diff --git a/Core/ZeroCopyBuffer.fs b/Core/ZeroCopyBuffer.fs new file mode 100644 index 0000000..4c1aaf8 --- /dev/null +++ b/Core/ZeroCopyBuffer.fs @@ -0,0 +1,125 @@ +namespace Froto.Core + +open System + +/// +/// Stream-like object backed by caller-supplied ArraySegment. +/// +/// Provides methods that work directly on the underlying backing buffer, +/// in order to minimize memory copies. +/// +/// +/// Why not use a Stream? +/// This class allows use of an ArraySegment with a non-zero starting offset +/// to house the source (or desitination) for decoding (or encoding); +/// specifically, so that a length-encoded field can be extracted as an +/// ArraySegment, without copying the associated bytes to a new Array. +/// System.IO.MemoryStream can't do that. Plus, the Stream API requires +/// additional error checking. + +type ZeroCopyBufferBase (seg:ArraySegment) = + + member val Array = seg.Array with get + member val Position = uint32 seg.Offset with get,set + member val Limit = uint32 seg.Offset + uint32 seg.Count with get + + new ( backing : byte array ) = + ZeroCopyBufferBase(ArraySegment(backing)) + + member x.IsEof + with get() = x.Position >= x.Limit + +/// Readable ZeroCopyBuffer (see @ZeroCopyBufferBase) +type ZeroCopyReadBuffer (seg:ArraySegment) = + inherit ZeroCopyBufferBase(seg) + + new (o:ZeroCopyBufferBase) = + ZeroCopyReadBuffer( o.Array ) + + new (backing : byte array ) = + ZeroCopyReadBuffer(ArraySegment(backing)) + + // Return portion of buffer still unread as an ArraySegment + member x.Remainder + with get() = ArraySegment( seg.Array, int x.Position, int <| x.Limit - x.Position ) + + /// Read one byte from the backing buffer. + member x.ReadByte () = + if x.Position < x.Limit then + let b = x.Array.[int x.Position] + x.Position <- x.Position + 1u + b + else + raise <| ProtobufWireFormatException("Read past end of protobuf buffer") + + /// Read multiple bytes, returning an ArraySegment which points directly + /// into the backing buffer. + member x.ReadByteSegment (n:uint32) = + if x.Position + n <= x.Limit then + let buf = ArraySegment( x.Array, int x.Position, int n) + x.Position <- x.Position + n + buf + else + raise <| ProtobufWireFormatException("Read past end of protobuf buffer") + +/// Writable ZeroCopyBuffer (see @ZeroCopyBufferBase) +/// +/// TODO: Should this optionally growable? If so, then by how much? +type ZeroCopyWriteBuffer (seg:ArraySegment) = + inherit ZeroCopyBufferBase(seg) + + new (o:ZeroCopyBufferBase) = + ZeroCopyWriteBuffer( o.Array ) + + new (backing : byte array) = + ZeroCopyWriteBuffer(ArraySegment(backing)) + + new size = + ZeroCopyWriteBuffer(Array.zeroCreate size) + + // Return portion of buffer written as an ArraySegment + member x.AsArraySegment + with get() = ArraySegment( seg.Array, seg.Offset, int x.Position - seg.Offset ) + + // Return portion of buffer written as an Array (mainly for testing) + member x.ToArray() = + seg.Array.[ seg.Offset .. int x.Position - 1 ] + + abstract WriteByte : byte -> unit + abstract WriteByteSegment : uint32 -> (ArraySegment->unit) -> unit + + /// Write one byte into the backing buffer. + override x.WriteByte b = + if x.Position < x.Limit then + x.Array.[int x.Position] <- b + x.Position <- x.Position + 1u + else + raise <| ProtobufWireFormatException("Write past end of protobuf buffer") + + /// Write 'len' bytes, via the caller supplied emplace function, + /// directly into the backing buffer. + override x.WriteByteSegment (len:uint32) (emplace:ArraySegment->unit) = + if x.Position + len <= x.Limit then + let buf = ArraySegment( x.Array, int x.Position, int len ) + emplace buf + x.Position <- x.Position + len + else + raise <| ProtobufWireFormatException("Write past end of protobuf buffer") + +/// Null ZeroCopyWriteBuffer. Used to calculate serialized length, without +/// actually writing anything. +// +// Note: This class simplifies the serialization logic, because a single code +// path can be used to both serialize AND calculate size requiements. However, +// this probably does cost a bit of performance. + +type NullWriteBuffer() = + inherit ZeroCopyWriteBuffer(0) + + override x.WriteByte b = + x.Position <- x.Position + 1u + + override x.WriteByteSegment (len:uint32) (_:ArraySegment->unit) = + x.Position <- x.Position + len + + member x.Length = x.Position diff --git a/Core/paket.references b/Core/paket.references new file mode 100644 index 0000000..640cf91 --- /dev/null +++ b/Core/paket.references @@ -0,0 +1 @@ +FSharp.Core \ No newline at end of file diff --git a/Froto.Core.Test/ExampleProtoClass.fs b/Froto.Core.Test/ExampleProtoClass.fs new file mode 100644 index 0000000..f062fcc --- /dev/null +++ b/Froto.Core.Test/ExampleProtoClass.fs @@ -0,0 +1,85 @@ +module SampleProto + +open Froto.Core.Encoding + +type InnerMessage () = + inherit MessageBase() + let ETestDefault = ETest.One // NOTE: Non-zero default is only supported in Proto2 + let m_id = ref 0 + let m_name = ref "" + let m_option = ref false + let m_test = ref ETestDefault + let m_packedFixed32 = ref List.empty + let m_repeatedInt32 = ref List.empty + + member x.ID with get() = !m_id and set(v) = m_id := v + member x.Name with get() = !m_name and set(v) = m_name := v + member x.bOption with get() = !m_option and set(v) = m_option := v + member x.Test with get() = !m_test and set(v) = m_test := v + member x.PackedFixed32 with get() = !m_packedFixed32 and set(v) = m_packedFixed32 := v + member x.RepeatedInt32 with get() = !m_repeatedInt32 and set(v) = m_repeatedInt32 := v + + override x.Clear() = + m_id := 0 + m_name := "" + m_option := false + m_test := ETestDefault + m_packedFixed32 := List.empty + m_repeatedInt32 := List.empty + + override x.DecoderRing = + [ + 1, m_id |> Serializer.hydrateInt32 + 2, m_name |> Serializer.hydrateString + 3, m_option |> Serializer.hydrateBool + 4, m_test |> Serializer.hydrateEnum + 5, m_packedFixed32 |> Serializer.hydratePackedFixed32 + 6, m_repeatedInt32 |> Serializer.hydrateRepeated Serializer.hydrateInt32 + ] + |> Map.ofList + + override x.EncoderRing(zcb) = + let encode = + (!m_id |> Serializer.dehydrateVarint 1) >> + (!m_name |> Serializer.dehydrateString 2) >> + (!m_option |> Serializer.dehydrateBool 3) >> + (!m_test |> Serializer.dehydrateDefaultedVarint ETestDefault 4) >> + (!m_packedFixed32 |> Serializer.dehydratePackedFixed32 5) >> + (!m_repeatedInt32 |> Serializer.dehydrateRepeated Serializer.dehydrateVarint 6) + encode zcb + + static member FromArraySegment (buf:System.ArraySegment) = + let self = InnerMessage() + self.Merge(buf) |> ignore + self + +and ETest = + | Nada = 0 + | One = 1 + | Two = 2 + +type OuterMessage() = + inherit MessageBase() + let m_inner = ref None + + member x.Inner with get() = !m_inner and set(v) = m_inner := v + + override x.Clear() = + m_inner := None + + override x.DecoderRing = + [ + 42, m_inner |> Serializer.hydrateOptionalMessage (InnerMessage.FromArraySegment) + ] + |> Map.ofList + + override x.EncoderRing(zcb) = + let encode = + (!m_inner |> Serializer.dehydrateOptionalMessage 42) + encode zcb + + static member FromArraySegment (buf:System.ArraySegment) = + let self = OuterMessage() + self.Merge(buf) |> ignore + self + diff --git a/Froto.Core.Test/Froto.Core.Test.fsproj b/Froto.Core.Test/Froto.Core.Test.fsproj new file mode 100644 index 0000000..c46c7e3 --- /dev/null +++ b/Froto.Core.Test/Froto.Core.Test.fsproj @@ -0,0 +1,156 @@ + + + + + Debug + AnyCPU + 2.0 + af2d437d-55a1-4f31-86bd-9b9d7e05caea + Library + Froto.Core.Test + Froto.Core.Test + v4.5 + 4.4.0.0 + true + Froto.Core.Test + + + + true + full + false + false + bin\Debug\ + DEBUG;TRACE + 3 + bin\Debug\Froto.Core.Test.XML + + + pdbonly + true + true + bin\Release\ + TRACE + 3 + bin\Release\Froto.Core.Test.XML + + + 11 + + + + + $(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets + + + + + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\FSharp\Microsoft.FSharp.Targets + + + + + + + + + + ..\packages\FsUnit.xUnit\lib\net45\FsUnit.Xunit.dll + True + True + + + ..\packages\FsUnit.xUnit\lib\net45\NHamcrest.dll + True + True + + + + + + + + + ..\packages\xunit.abstractions\lib\net35\xunit.abstractions.dll + True + True + + + + + + + + + ..\packages\xunit.assert\lib\portable-net45+win8+wp8+wpa81\xunit.assert.dll + True + True + + + + + + + + + ..\packages\xunit.extensibility.core\lib\portable-net45+win8+wp8+wpa81\xunit.core.dll + True + True + + + + + + + + + ..\packages\xunit.extensibility.execution\lib\net45\xunit.execution.desktop.dll + True + True + + + + + + + + <__paket__xunit_runner_visualstudio_props>net20\xunit.runner.visualstudio + + + + + + + + + + + + + + + + + + True + + + + + + + + + + Froto.Core + {c37745d0-275d-4dce-9ddc-c4663a462edd} + True + + + \ No newline at end of file diff --git a/Froto.Core.Test/TestMessageSerialization.fs b/Froto.Core.Test/TestMessageSerialization.fs new file mode 100644 index 0000000..6bf9cd8 --- /dev/null +++ b/Froto.Core.Test/TestMessageSerialization.fs @@ -0,0 +1,178 @@ +namespace TestSerializer + +open Xunit +open FsUnit.Xunit + +open System +open Froto.Core +open Froto.Core.Encoding + +[] +module MessageSerialization = + + let toArray (seg:ArraySegment<'a>) = + seg.Array.[ seg.Offset .. (seg.Count-1) ] + + type InnerMessage () = + inherit MessageBase() + let m_id = ref 0 + let m_name = ref "" + + member x.ID with get() = !m_id and set(v) = m_id := v + member x.Name with get() = !m_name and set(v) = m_name := v + + override x.Clear() = + m_id := 0 + m_name := "" + + + override x.RequiredFields = + [ 1; 2 ] + |> Set.ofList + + override x.DecoderRing = + [ 1, m_id |> Serializer.hydrateInt32 + 2, m_name |> Serializer.hydrateString + ] + |> Map.ofList + + override x.EncoderRing zcb = + let encode = + (!m_id |> Serializer.dehydrateVarint 1) >> + (!m_name |> Serializer.dehydrateString 2) + encode zcb + + static member FromArraySegment (buf:ArraySegment) = + let self = InnerMessage() + self.Merge(buf) |> ignore + self + + [] + let ``Deserialize simple message`` () = + let buf = + [| + 0x01uy<<<3 ||| 0uy; // tag: id=1; varint + 99uy; // value 99 + 0x02uy<<<3 ||| 2uy // tag: id=2; length delim + 12uy; // length = 12 + 0x54uy; 0x65uy; 0x73uy; 0x74uy; 0x20uy; 0x6duy; 0x65uy; 0x73uy; 0x73uy; 0x61uy; 0x67uy; 0x65uy + // value "Test message" + |] |> ArraySegment + let msg = InnerMessage.FromArraySegment(buf) + msg.ID |> should equal 99 + msg.Name |> should equal "Test message" + + [] + let ``Missing required field throws exception`` () = + let buf = + [| +// 0x01uy<<<3 ||| 0uy; // tag: id=1; varint +// 99uy; // value 99 + 0x02uy<<<3 ||| 2uy // tag: id=2; length delim + 12uy; // length = 12 + 0x54uy; 0x65uy; 0x73uy; 0x74uy; 0x20uy; 0x6duy; 0x65uy; 0x73uy; 0x73uy; 0x61uy; 0x67uy; 0x65uy + // value "Test message" + |] |> ArraySegment + fun () -> InnerMessage.FromArraySegment(buf) |> ignore + |> should throw typeof + + [] + let ``Serialize simple message`` () = + let msg = InnerMessage() + msg.ID <- 98 + msg.Name <- "ABC0" + msg.Serialize() + |> toArray + |> should equal + [| + 0x01uy<<<3 ||| 0uy; // tag: id=1; varint + 98uy; // value 98 + 0x02uy<<<3 ||| 2uy // tag: id=2; length delim + 4uy; // length = 4 + 0x41uy; 0x42uy; 0x43uy; 0x30uy + // value "ABC0" + |] + + type OuterMessage () = + inherit MessageBase() + let m_id = ref 0 + let m_inner = ref None + let m_hasMore = ref false + + member x.ID with get() = !m_id and set(v) = m_id := v + member x.Inner with get() = !m_inner and set(v) = m_inner := v + member x.HasMore with get() = !m_hasMore and set(v) = m_hasMore := v + + override x.Clear() = + m_id := 0 + m_inner := None + m_hasMore := false + + override x.DecoderRing = + [ 1, m_id |> Serializer.hydrateInt32; + 42, m_inner |> Serializer.hydrateOptionalMessage (InnerMessage.FromArraySegment); + 43, m_hasMore |> Serializer.hydrateBool; + ] + |> Map.ofList + + override x.EncoderRing zcb = + let encode = + (!m_id |> Serializer.dehydrateVarint 1) >> + (!m_inner |> Serializer.dehydrateOptionalMessage 42) >> + (!m_hasMore |> Serializer.dehydrateBool 43) + encode zcb + + static member FromArraySegment (buf:ArraySegment) = + let self = OuterMessage() + self.Merge(buf) |> ignore + self + + [] + let ``Deserialize compound message`` () = + let buf = + [| + 0x01uy<<<3 ||| 0uy; // tag: fldnum=1, varint + 21uy; // value 21 + 0xD0uy ||| 2uy; 0x02uy; // tag: fldnum=42, length delim + 16uy; // length 16 + 0x01uy<<<3 ||| 0uy; // tag: fldnum=1; varint + 99uy; // value 99 + 0x02uy<<<3 ||| 2uy; // tag: fldnum=2; length delim + 12uy; // length = 12 + 0x54uy; 0x65uy; 0x73uy; 0x74uy; 0x20uy; 0x6duy; 0x65uy; 0x73uy; 0x73uy; 0x61uy; 0x67uy; 0x65uy + // value "Test message" + 0xD8uy ||| 0uy; 0x02uy; // tag: fldnum=43, varint + 0x01uy; // value true + |] |> ArraySegment + let msg = OuterMessage.FromArraySegment(buf) + msg.ID |> should equal 21 + msg.Inner.IsSome |> should equal true + msg.Inner.Value.ID |> should equal 99 + msg.Inner.Value.Name |> should equal "Test message" + + [] + let ``Serialize compound message`` () = + let msg = OuterMessage() + msg.Inner <- Some(InnerMessage()) + msg.ID <- 5 + msg.Inner.Value.ID <- 6 + msg.Inner.Value.Name <- "ABC0" + msg.HasMore <- true + msg.Serialize() + |> toArray + |> should equal + [| + 0x01uy<<<3 ||| 0uy; // tag: id=1; varint + 5uy; // value 5 + 0xD0uy ||| 2uy; 0x02uy; // tag: fldnum=42, length delim + 8uy; // length = 8 + 0x01uy<<<3 ||| 0uy; // tag: id=1; varint + 6uy; // value 6 + 0x02uy<<<3 ||| 2uy; // tag: id=2; length delim + 4uy; // length = 4 + 0x41uy; 0x42uy; 0x43uy; 0x30uy; + // value "ABC0" + 0xD8uy ||| 0uy; 0x02uy; // tag: fldnum=43, varint + 0x01uy; // value true + |] + diff --git a/Froto.Core.Test/TestSerializer.fs b/Froto.Core.Test/TestSerializer.fs new file mode 100644 index 0000000..c9874e6 --- /dev/null +++ b/Froto.Core.Test/TestSerializer.fs @@ -0,0 +1,483 @@ +namespace TestSerializer + +(* TODO: + Write tests (and implementation) for: + - Missing required field causes exception + - Optional fields can be detected as ommitted? + - Enum properly has correct default (handle via codegen) +*) + +open Xunit +open FsUnit.Xunit + +open System +open Froto.Core +open Froto.Core.Encoding + +[] +module Utility = + open Froto.Core.Encoding.Utility + + [] + let ``Zig to Zag`` () = + zigZag32 0 |> should equal 0 + zigZag32 -1 |> should equal 1 + zigZag32 1 |> should equal 2 + zigZag32 -2 |> should equal 3 + zigZag32 2 |> should equal 4 + + zigZag64 0L |> should equal 0L + zigZag64 -1L |> should equal 1L + zigZag64 1L |> should equal 2L + zigZag64 -2L |> should equal 3L + zigZag64 2L |> should equal 4L + + let ``Zag to Zig`` () = + zagZig32 0 |> should equal 0 + zagZig32 1 |> should equal -1 + zagZig32 2 |> should equal 1 + zagZig32 3 |> should equal -2 + zagZig32 4 |> should equal 2 + + zagZig64 0L |> should equal 0L + zagZig64 1L |> should equal -1L + zagZig64 2L |> should equal 1L + zagZig64 3L |> should equal -2L + zagZig64 4L |> should equal 2L + + [] + let ``Length when encoded to Varint`` () = + + varIntLen 0UL |> should equal 0 + varIntLen 0x7FUL |> should equal 1 + varIntLen 0x80UL |> should equal 2 + varIntLen 0x3FFFUL |> should equal 2 + varIntLen 0x4000UL |> should equal 3 + + [] + let ``Length of field number when encoded in a tag`` () = + tagLen 0 |> should equal 1 + tagLen 0x0F |> should equal 1 + tagLen 0x10 |> should equal 2 + +[] +module Deserialize = + + open Froto.Core.Encoding.Serializer + + type ETestEnum = + | ZERO = 0 + | ONE = 1 + | TWO = 2 + + [] + let ``hydrate numeric types from Varint`` () = + let vi = RawField.Varint (1,2UL) + + // Int32 + let x = ref 0 + vi |> hydrateInt32 x + !x |> should equal 2 + + // Int64 + let x = ref 0L + vi |> hydrateInt64 x + !x |> should equal 2L + + // UInt32 + let x = ref 0u + vi |> hydrateUInt32 x + !x |> should equal 2u + + // UInt64 + let x = ref 0UL + vi |> hydrateUInt64 x + !x |> should equal 2UL + + // SInt32 + let x = ref 0 + vi |> hydrateSInt32 x + !x |> should equal 1 + + // SInt64 + let x = ref 0L + vi |> hydrateSInt64 x + !x |> should equal 1L + + // Bool + let x = ref false + vi |> hydrateBool x + !x |> should equal true + + // Enum + let x = ref ETestEnum.ZERO + vi |> hydrateEnum x + !x |> should equal ETestEnum.TWO + + [] + let ``Int overflow is truncated`` () = + let vi = RawField.Varint (1, 0x0000000100000001UL) + let x = ref 0 + vi |> hydrateInt32 x + !x |> should equal 1 + + [] + let ``Unknown enum value is preserved`` () = + let vi = RawField.Varint (1, 42UL) + let x = ref ETestEnum.ZERO + vi |> hydrateEnum x + int(!x) |> should equal 42 + + [] + let ``Unexpected wire type throws`` () = + let f32 = RawField.Fixed32(1, 42u) + let x = ref 0 + fun () -> + f32 |> hydrateInt32 x + |> should throw typeof + + [] + let ``Hydrate numeric types from fixed`` () = + let vi = RawField.Fixed32 (1, 42u) + let x = ref 0u + vi |> hydrateFixed32 x + !x |> should equal 42u + + let vi = RawField.Fixed64 (1, 42UL) + let x = ref 0UL + vi |> hydrateFixed64 x + !x |> should equal 42UL + + let vi = RawField.Fixed32 (1, 42u) + let x = ref 0 + vi |> hydrateSFixed32 x + !x |> should equal 42 + + let vi = RawField.Fixed64 (1, 42UL) + let x = ref 0L + vi |> hydrateSFixed64 x + !x |> should equal 42L + + let vi = RawField.Fixed32 (1, 0b01000000u <<< (3*8)) + let x = ref 0.0f + vi |> hydrateSingle x + !x |> should equal 2.0f + + let vi = RawField.Fixed64 (1, 0b01000000UL <<< (7*8) ) + let x = ref 0.0 + vi |> hydrateDouble x + !x |> should equal 2.0 + + [] + let ``Hydrate bytes`` () = + let buf = [| 0uy; 1uy; 2uy; 3uy; 4uy; 5uy; 6uy; 7uy; 8uy |] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let x = ref Array.empty + vi |> hydrateBytes x + !x |> should equal buf + + [] + let ``Hydrate string`` () = + let buf = [| 0x41uy; 0x42uy; 0x43uy; 0x34uy; 0x32uy |] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let x = ref "" + vi |> hydrateString x + !x |> should equal "ABC42" + + [] + let ``Hydrate packed numeric types`` () = + + // Packet Int32 + let buf = [|0uy; 1uy; 2uy; 3uy; 4uy|] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedInt32 xs + !xs |> should equal [ 0; 1; 2; 3; 4 ] + + // Packet UInt32 + let buf = [|0uy; 1uy; 2uy; 3uy; 4uy|] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedUInt32 xs + !xs |> should equal [ 0u; 1u; 2u; 3u; 4u ] + + // Packet SInt32 + let buf = [|0uy; 1uy; 2uy; 3uy; 4uy|] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedSInt32 xs + !xs |> should equal [ 0; -1; 1; -2; 2 ] + + // Packet Int64 + let buf = [|0uy; 1uy; 2uy; 3uy; 4uy|] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedInt64 xs + !xs |> should equal [ 0L; 1L; 2L; 3L; 4L ] + + // Packet UInt64 + let buf = [|0uy; 1uy; 2uy; 3uy; 4uy|] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedUInt64 xs + !xs |> should equal [ 0UL; 1UL; 2UL; 3UL; 4UL ] + + // Packet SInt64 + let buf = [|0uy; 1uy; 2uy; 3uy; 4uy|] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedSInt64 xs + !xs |> should equal [ 0L; -1L; 1L; -2L; 2L ] + + // Packed Bool + let buf = [|0uy; 1uy; 2uy; 0uy; 4uy|] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedBool xs + !xs |> should equal [ false; true; true; false; true ] + + // Packed Enum + let buf = [|0uy; 1uy; 2uy|] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs : ETestEnum list ref = ref List.empty + vi |> hydratePackedEnum xs + !xs |> should equal [ ETestEnum.ZERO; ETestEnum.ONE; ETestEnum.TWO ] + + // Packed Fixed32 + let buf = [|0uy; 0uy; 0uy; 0uy; 0x01uy; 0x02uy; 0x00uy; 0x00uy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy |] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedFixed32 xs + !xs |> should equal [ 0u; 0x00000201u; 0xFFFFFFFFu ] + + // Packed Fixed64 + let buf = [|0uy; 0uy; 0uy; 0uy; 0x01uy; 0x02uy; 0x00uy; 0x00uy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0uy; 0uy; 0uy; 0xFFuy |] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedFixed64 xs + !xs |> should equal [ 0x0000020100000000UL; 0xFF000000FFFFFFFFUL ] + + // Packed SFixed32 + let buf = [|0uy; 0uy; 0uy; 0uy; 0x01uy; 0x02uy; 0x00uy; 0x00uy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy |] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedSFixed32 xs + !xs |> should equal [ 0; 0x00000201; -1 ] + + // Packed SFixed64 + let buf = [|0uy; 0uy; 0uy; 0uy; 0x01uy; 0x02uy; 0x00uy; 0x00uy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy |] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedSFixed64 xs + !xs |> should equal [ 0x0000020100000000L; -1L ] + + // Packed Single + let buf = [|0uy; 0uy; 0uy; 0uy; 0x00uy; 0x00uy; 0x00uy; 0b01000000uy |] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedSingle xs + !xs |> should equal [ 0.0f; 2.0f ] + + // Packed Double + let buf = [| 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; + 0x9Auy; 0x99uy; 0x99uy; 0x99uy; 0x99uy; 0x99uy; 0xB9uy; 0x3Fuy; + 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; |] + let vi = RawField.LengthDelimited (1, ArraySegment(buf)) + let xs = ref List.empty + vi |> hydratePackedDouble xs + !xs |> should equal [ 0.0; 0.10; 0.0 ] + +[] +module Serialize = + open Froto.Core.Encoding.Serializer + + type ZCB = ZeroCopyWriteBuffer + let toArray(zcb:ZCB) = zcb.ToArray() + + let fid = 1 // field ID + + [] + let ``Dehydrate integer varint`` () = + ZCB(2) + |> dehydrateVarint fid 0UL + |> toArray + |> should equal Array.empty + + ZCB(2) + |> dehydrateVarint fid 2UL + |> toArray + |> should equal [| 0x08uy; 2uy |] + + [] + let ``Dehydrate SInt32 varint`` () = + ZCB(2) + |> dehydrateSInt32 fid 0 + |> toArray + |> should equal Array.empty + + ZCB(2) + |> dehydrateSInt32 fid -1 + |> toArray + |> should equal [| 0x08uy; 1uy |] + + ZCB(2) + |> dehydrateSInt32 fid 1 + |> toArray + |> should equal [| 0x08uy; 2uy |] + + [] + let ``Dehydrate SInt64 varint`` () = + ZCB(2) + |> dehydrateSInt64 fid 0L + |> toArray + |> should equal Array.empty + + ZCB(2) + |> dehydrateSInt64 fid -1L + |> toArray + |> should equal [| 0x08uy; 1uy |] + + ZCB(2) + |> dehydrateSInt64 fid 1L + |> toArray + |> should equal [| 0x08uy; 2uy |] + + [] + let ``Dehydrate bool varint`` () = + ZCB(2) + |> dehydrateBool fid false + |> toArray + |> should equal Array.empty + + ZCB(2) + |> dehydrateBool fid true + |> toArray + |> should equal [| 0x08uy; 1uy |] + + [] + let ``Dehydrate Fixed32`` () = + ZCB(5) + |> dehydrateFixed32 fid 5 + |> toArray + |> should equal [| 0x08uy ||| 5uy; 5uy;0uy;0uy;0uy |] + + [] + let ``Dehydrate Fixed64`` () = + ZCB(9) + |> dehydrateFixed64 fid 5 + |> toArray + |> should equal [| 0x08uy ||| 1uy; 5uy;0uy;0uy;0uy; 0uy;0uy;0uy;0uy |] + + [] + let ``Dehydrate Single`` () = + ZCB(5) + |> dehydrateSingle fid 2.0f + |> toArray + |> should equal [| 0x08uy ||| 5uy; 0uy; 0uy; 0b00000000uy; 0b01000000uy |] + + [] + let ``Dehydrate Double`` () = + ZCB(9) + |> dehydrateDouble fid 0.10 + |> toArray + |> should equal [| 0x08uy ||| 1uy; 0x9Auy; 0x99uy; 0x99uy; 0x99uy; 0x99uy; 0x99uy; 0xB9uy; 0x3Fuy |] + + [] + let ``Dehydrate String`` () = + ZCB(6) + |> dehydrateString fid "0ABC" + |> toArray + |> should equal [| 0x08uy ||| 2uy; 4uy; 0x30uy; 0x41uy; 0x42uy; 0x43uy |] + + [] + let ``Dehydrate Bytes`` () = + ZCB(6) + |> dehydrateBytes fid (ArraySegment([| 3uy; 4uy; 5uy; 6uy; |])) + |> toArray + |> should equal [| 0x08uy ||| 2uy; 4uy; 3uy; 4uy; 5uy; 6uy |] + + + let ``Dehydrate with default value`` () = + // Verify non-default value results in an actual value + ZCB(2) + |> dehydrateDefaultedVarint 0UL fid 2UL + |> toArray + |> should equal [| 0x08uy; 2uy |] + + // Now, check that default value result in an empty array (value is elided) + let checkGetsElided f = + ZCB(16) + |> f + |> toArray + |> should equal Array.empty + + checkGetsElided <| dehydrateDefaultedVarint 1UL fid 1UL + checkGetsElided <| dehydrateDefaultedSInt32 2 fid 2 + checkGetsElided <| dehydrateDefaultedSInt64 3L fid 3L + checkGetsElided <| dehydrateDefaultedBool true fid true + checkGetsElided <| dehydrateDefaultedFixed32 4 fid 4 + checkGetsElided <| dehydrateDefaultedFixed64 5L fid 5L + checkGetsElided <| dehydrateDefaultedSingle 0.60f fid 0.60f + checkGetsElided <| dehydrateDefaultedDouble 0.70 fid 0.70 + checkGetsElided <| dehydrateDefaultedString "Hello" fid "Hello" + checkGetsElided <| dehydrateDefaultedBytes (ArraySegment([|8uy;9uy|])) fid (ArraySegment([|8uy;9uy|])) + + [] + let ``Dehydrate Packed Varint`` () = + ZCB(8) + |> dehydratePackedVarint fid [ 0; 1; 128; 129 ] + |> toArray + |> should equal [| 0x08uy ||| 2uy; 6uy; 0uy; 1uy; 0x80uy; 0x01uy; 0x81uy; 0x01uy |] + + [] + let ``Dehydrate Packed Bool`` () = + ZCB(5) + |> dehydratePackedBool fid [ false; true; false ] + |> toArray + |> should equal [| 0x08uy ||| 2uy; 3uy; 0uy; 1uy; 0uy |] + + [] + let ``Dehydrate Packed SInt32`` () = + ZCB(5) + |> dehydratePackedSInt32 fid [ 0; -1; 1 ] + |> toArray + |> should equal [| 0x08uy ||| 2uy; 3uy; 0uy; 1uy; 2uy |] + + [] + let ``Dehydrate Packed SInt64`` () = + ZCB(5) + |> dehydratePackedSInt64 fid [ 0L; -1L; 1L ] + |> toArray + |> should equal [| 0x08uy ||| 2uy; 3uy; 0uy; 1uy; 2uy |] + + [] + let ``Dehydrate Packed Fixed32`` () = + ZCB(10) + |> dehydratePackedFixed32 fid [ 0; -1 ] + |> toArray + |> should equal [| 0x08uy ||| 2uy; 8uy; 0x00uy;0x00uy;0x00uy;0x00uy; 0xFFuy;0xFFuy;0xFFuy;0xFFuy |] + + + [] + let ``Dehydrate Packed Fixed64`` () = + ZCB(18) + |> dehydratePackedFixed64 fid [ 0; -1 ] + |> toArray + |> should equal [| 0x08uy ||| 2uy; 16uy; 0x00uy;0x00uy;0x00uy;0x00uy;0x00uy;0x00uy;0x00uy;0x00uy; 0xFFuy;0xFFuy;0xFFuy;0xFFuy;0xFFuy;0xFFuy;0xFFuy;0xFFuy |] + + [] + let ``Dehydrate Packed Single`` () = + ZCB(10) + |> dehydratePackedSingle fid [ 0.0f; 2.0f ] + |> toArray + |> should equal [| 0x08uy ||| 2uy; 8uy; 0x00uy;0x00uy;0x00uy;0x00uy; 0uy;0uy;0b00000000uy;0b01000000uy |] + + + [] + let ``Dehydrate Packed Double`` () = + ZCB(18) + |> dehydratePackedDouble fid [ 0.0; 0.10 ] + |> toArray + |> should equal [| 0x08uy ||| 2uy; 16uy; 0x00uy;0x00uy;0x00uy;0x00uy;0x00uy;0x00uy;0x00uy;0x00uy; 0x9Auy;0x99uy;0x99uy;0x99uy;0x99uy;0x99uy;0xB9uy;0x3Fuy |] + + diff --git a/Froto.Core.Test/TestWireFormat.fs b/Froto.Core.Test/TestWireFormat.fs new file mode 100644 index 0000000..3bad57a --- /dev/null +++ b/Froto.Core.Test/TestWireFormat.fs @@ -0,0 +1,376 @@ +namespace TestWireFormat + +open Xunit +open FsUnit.Xunit + +open System + +open Froto.Core +open Froto.Core.WireFormat +open Froto.Core.Encoding + +module Helpers = + type System.ArraySegment<'a> + with member x.ToArray() = + x.Array.[ x.Offset .. x.Offset + x.Count - 1] + +[] +module Decode = + open Helpers + + type ZCR = ZeroCopyReadBuffer + + let toArray (a:ArraySegment) = + a.ToArray() + + [] + let ``Can decode a varint`` () = + + [| 0b00000001uy |] + |> ZCR + |> decodeVarint + |> should equal 1UL + + [| 0b10101100uy; 0b00000010uy |] + |> ZCR + |> decodeVarint + |> should equal 300UL + + [] + let ``Decode Varint stops after 64 bits of bytes`` () = + [| 0x81uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x00uy |] + |> ZCR + |> decodeVarint + |> should equal 1UL + + fun () -> + [| 0x81uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x00uy |] + |> ZCR + |> decodeVarint + |> ignore + |> should throw typeof + + [] + let ``Can decode max Varint`` () = + [| 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0x01uy |] + |> ZCR + |> decodeVarint + |> should equal System.UInt64.MaxValue + + [] + let ``Decode varint ignores overflow`` () = + // TODO: Should this really throw an error? + // That would add another IF statment to the inner decode loop of every varint... + [| 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0x7Fuy |] + |> ZCR + |> decodeVarint + |> should equal System.UInt64.MaxValue + + + [] + let ``Varint throws if buffer too small`` () = + fun () -> + [| 0x80uy; 0x80uy; 0x80uy |] + |> ZCR + |> decodeVarint + |> ignore + |> should throw typeof + + [] + let ``Can decode fixed`` () = + [| 0x01uy; 0x20uy; 0x03uy; 0x40uy |] + |> ZCR + |> decodeFixed32 + |> should equal 0x40032001u + + [| 0x01uy; 0x20uy; 0x03uy; 0x40uy; 0x00uy; 0x00uy; 0x00uy; 0x80uy |] + |> ZCR + |> decodeFixed64 + |> should equal 0x8000000040032001UL + + [] + let ``Fixed throws if buffer too small`` () = + fun () -> + [| 0x01uy; 0x20uy; 0x03uy |] + |> ZCR + |> decodeFixed32 + |> ignore + |> should throw typeof + + [] + let ``Single decodes`` () = + [| 0uy; 0uy; 0b00000000uy; 0b01000000uy |] + |> ZCR + |> decodeSingle + |> should equal 2.0f + + [] + let ``Double decodes`` () = + [| 0x9Auy; 0x99uy; 0x99uy; 0x99uy; 0x99uy; 0x99uy; 0xB9uy; 0x3Fuy |] + |> ZCR + |> decodeDouble + |> should equal 0.10 + + [] + let ``Decode length delimited`` () = + + // len=3; should not return last byte + [| 0x03uy; 0x00uy; 0x01uy; 0x02uy; 0x00uy |] + |> ZCR + |> decodeLengthDelimited + |> toArray + |> should equal [| 00uy; 01uy; 02uy |] + +[] +module Encode = + open Helpers + + type ZCW = ZeroCopyWriteBuffer + + let toArray (a:ZCW) = + a.ToArray() + + [] + let ``Encode one-byte varint`` () = + ZCW(2) + |> encodeVarint 0x01UL + |> toArray + |> should equal [| 0x01uy |] + + [] + let ``Encode two-byte varint`` () = + ZCW(2) + |> encodeVarint 0x81UL + |> toArray + |> should equal [| 0x81uy; 0x01uy |] + + [] + let ``Encode max-byte varint`` () = + ZCW(10) + |> encodeVarint 0x8000000000000000UL + |> toArray + |> should equal [| 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x01uy |] + + [] + let ``Encode max varint`` () = + ZCW(10) + |> encodeVarint System.UInt64.MaxValue + |> toArray + |> should equal [| 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0x01uy |] + + [] + let ``Encode Fixed32`` () = + ZCW(4) + |> encodeFixed32 0x80000001u + |> toArray + |> should equal [| 0x01uy; 0x00uy; 0x00uy; 0x80uy |] + + [] + let ``Encode Fixed64`` () = + ZCW(8) + |> encodeFixed64 0x8000000000000001UL + |> toArray + |> should equal [| 0x01uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x00uy; 0x80uy |] + + [] + let ``Encode Single and Double`` () = + ZCW(4) + |> encodeSingle 2.0f + |> ZeroCopyReadBuffer + |> decodeSingle + |> should equal 2.0f + + ZCW(8) + |> encodeDouble 0.10 + |> ZeroCopyReadBuffer + |> decodeDouble + |> should equal 0.10 + + [] + let ``Encode length delimited`` () = + + let src = [| 0x00uy; 0x01uy; 0x02uy; 0x03uy; 0x04uy |] + let len = src.Length + + ZCW(256) + |> encodeLengthDelimited + (uint32 len) + (fun dest-> Array.Copy(src, 0L, dest.Array, int64 dest.Offset, int64 len) ) + |> toArray + |> should equal [| 0x05uy; 0x00uy; 0x01uy; 0x02uy; 0x03uy; 0x04uy |] + +[] +module DecodeField = + open Helpers + + type ZCR = ZeroCopyReadBuffer + + [] + let ``Read tag`` () = + + [| 0x08uy |] + |> ZCR + |> decodeTag + |> should equal (1, WireType.Varint) + + [| 0x11uy |] + |> ZCR + |> decodeTag + |> should equal (2, WireType.Fixed64) + + [| 0xD2uy; 0x02uy |] + |> ZCR + |> decodeTag + |> should equal (42, WireType.LengthDelimited ) + + + [] + let ``Tag validated to range [1,2^28)`` () = + + fun () -> + [| 0x08uy |] + |> ZCR + |> decodeTag + |> ignore + |> should not' (throw typeof) + + fun () -> + [| 0xFFuy; 0xFFuy; 0xFFuy; 0xFFuy; 0x08uy |] + |> ZCR + |> decodeTag + |> ignore + |> should not' (throw typeof) + + fun () -> + [| 0x00uy |] + |> ZCR + |> decodeTag + |> ignore + |> should throw typeof + + fun () -> + [| 0x80uy; 0x80uy; 0x80uy; 0x80uy; 0x10uy |] + |> ZCR + |> decodeTag + |> ignore + |> should throw typeof + + [] + let ``Read varint field`` () = + [| 0x08uy; 2uy |] + |> ZCR + |> decodeField + |> should equal (Varint (1, 2UL)) + + [] + let ``Read fixed64 field`` () = + [| 0x09uy; 0x00uy;0x00uy;0x00uy;0x00uy; 0x00uy;0x01uy;0x02uy;0x03uy |] + |> ZCR + |> decodeField + |> should equal (Fixed64 (1, 0x0302010000000000UL)) + + [] + let ``Read length delimited field`` () = + let field = + [| 0x1Auy; 0x03uy; 0x00uy;0x00uy;0x01uy; 0x00uy;0x00uy;0x01uy;0x02uy;0x03uy |] + |> ZCR + |> decodeField + + match field with + | LengthDelimited (num, seg) -> + seg.ToArray() + |> should equal [| 0uy; 0uy; 1uy |] + | _ -> failwithf "Expected: LengthDelimited; Found: %A" field + + [] + let ``Read fixed32 field`` () = + [| byte ((9<<<3) ||| 5); 0x00uy;0x01uy;0x02uy;0x03uy |] + |> ZCR + |> decodeField + |> should equal (Fixed32 (9, 0x03020100u)) + +[] +module EncodeField = + open Helpers + + type ZCW = ZeroCopyWriteBuffer + + let toArray (a:ZCW) = a.ToArray() + + [] + let ``Write tag`` () = + ZCW(256) + |> encodeTag 2 WireType.Fixed64 + |> toArray + |> should equal [| 0x11uy |] + + [] + let ``Field number validated to range [1, 2^28)`` () = + let buf = ZCW(256) + + fun () -> buf |> encodeTag 1 WireType.Fixed64 |> ignore + |> should not' (throw typeof) + + fun () -> buf |> encodeTag RawField.MaxFieldNum WireType.Fixed64 |> ignore + |> should not' (throw typeof) + + fun () -> buf |> encodeTag 0 WireType.Fixed64 |> ignore + |> should throw typeof + + fun () -> buf |> encodeTag (RawField.MaxFieldNum+1) WireType.Fixed64 |> ignore + |> should throw typeof + + fun () -> buf |> encodeTag -1 WireType.Fixed64 |> ignore + |> should throw typeof + + fun () -> buf |> encodeTag (Int32.MinValue) WireType.Fixed64 |> ignore + |> should throw typeof + + [] + let ``Write varint field`` () = + ZCW(256) + |> encodeFieldVarint 2 42UL + |> toArray + |> should equal [| 0x10uy; 42uy |] + + [] + let ``Write fixed64 field`` () = + ZCW(256) + |> encodeFieldFixed64 2 42UL + |> toArray + |> should equal [| 0x11uy; 42uy; 0uy; 0uy; 0uy; 0uy; 0uy; 0uy; 0uy|] + + [] + let ``Write length delimited field`` () = + let src = [| 0x00uy; 0x01uy; 0x02uy; 0x03uy; 0x04uy |] + let len = src.Length + + ZCW(256) + |> encodeFieldLengthDelimited 2 + (uint32 len) + (fun dest-> Array.Copy(src, 0L, dest.Array, int64 dest.Offset, int64 len) ) + |> toArray + |> should equal [| 0x12uy; 0x05uy; 0x00uy; 0x01uy; 0x02uy; 0x03uy; 0x04uy |] + + let ``Write bytes field`` () = + let src = [| for i in 1..64 -> byte i|] + + ZCW(256) + |> encodeFieldBytes 2 (ArraySegment(src)) + |> toArray + |> should equal (Array.append [| 0x12uy; 64uy |] src) + + let ``Write string field`` () = + let src = "123" + + ZCW(256) + |> encodeFieldString 2 src + |> toArray + |> should equal [| 0x12uy; 3uy; 0x31uy; 0x32uy; 0x33uy |] + + [] + let ``Write fixed32 field`` () = + ZCW(256) + |> encodeFieldFixed32 2 42u + |> toArray + |> should equal [| 0x15uy; 42uy; 0uy; 0uy; 0uy|] diff --git a/Froto.Core.Test/TestZeroCopyBuffer.fs b/Froto.Core.Test/TestZeroCopyBuffer.fs new file mode 100644 index 0000000..f9a524d --- /dev/null +++ b/Froto.Core.Test/TestZeroCopyBuffer.fs @@ -0,0 +1,85 @@ +[] +module TestZeroCopyBuffer + +open Xunit +open FsUnit.Xunit + +open System + +open Froto.Core + +type System.ArraySegment<'a> + with member x.ToArray() = + x.Array.[ x.Offset .. x.Offset + x.Count - 1] + + +[] +let ``Can read bytes`` () = + let zc = ZeroCopyReadBuffer( [| 3uy; 2uy; 1uy |] ) + + let a = zc.ReadByte() + a |> should equal 3uy + + let b = zc.ReadByte() + b |> should equal 2uy + + let c = zc.ReadByte() + c |> should equal 1uy + + fun () -> zc.ReadByte() |> ignore + |> should throw typeof + +[] +let ``Can read range`` () = + let zc = ZeroCopyReadBuffer([| 3uy; 2uy; 1uy |]) + + let r = zc.ReadByteSegment(2u) + r.Count |> should equal 2 + r.Offset |> should equal 0 + r.ToArray() |> should equal [| 3uy; 2uy |] + + let rem = zc.Remainder + rem.Offset |> should equal 2 + rem.Count |> should equal 1 + + let r = zc.ReadByteSegment(1u) + r.Count |> should equal 1 + r.Offset |> should equal 2 + r.ToArray() |> should equal [| 1uy |] + + fun () -> zc.ReadByte() |> ignore + |> should throw typeof + +[] +let ``Can write bytes`` () = + let zc = ZeroCopyWriteBuffer([| 3uy; 2uy; 1uy |]) + + zc.WriteByte(42uy) + zc.WriteByte(43uy) + zc.ToArray() |> should equal [| 42uy; 43uy |] + + let arr = zc.AsArraySegment + arr.Offset |> should equal 0 + arr.Count |> should equal 2 + + zc.WriteByte(44uy) + + zc.ToArray() |> should equal [| 42uy; 43uy; 44uy |] + + fun () -> zc.WriteByte(0uy) |> ignore + |> should throw typeof + +[] +let ``Can write range`` () = + + let xs = [|1uy;2uy;3uy;4uy;5uy|] + let len = xs.Length + let emplace (dest:ArraySegment) = + Array.Copy( xs, 0L, dest.Array, int64 dest.Offset, int64 len) + + let zc = ZeroCopyWriteBuffer(256) + zc.WriteByteSegment (uint32 len) emplace + + zc.ToArray() + |> should equal [| 1uy; 2uy; 3uy; 4uy; 5uy |] + diff --git a/Froto.Core.Test/app.config b/Froto.Core.Test/app.config new file mode 100644 index 0000000..9cfe68d --- /dev/null +++ b/Froto.Core.Test/app.config @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Froto.Core.Test/packages.config b/Froto.Core.Test/packages.config new file mode 100644 index 0000000..87777c5 --- /dev/null +++ b/Froto.Core.Test/packages.config @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/Froto.Core.Test/paket.references b/Froto.Core.Test/paket.references new file mode 100644 index 0000000..0eb1cc9 --- /dev/null +++ b/Froto.Core.Test/paket.references @@ -0,0 +1,4 @@ +FSharp.Core +FsUnit.xUnit +xunit +xunit.runner.visualstudio diff --git a/Froto.sln b/Froto.sln index 68fa464..b2c7836 100644 --- a/Froto.sln +++ b/Froto.sln @@ -44,6 +44,15 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{992FDC44-0 EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Exe", "Exe\Exe.fsproj", "{F5D8D709-919D-4539-8908-91C540C970B3}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Froto.Core.Test", "Froto.Core.Test\Froto.Core.Test.fsproj", "{AF2D437D-55A1-4F31-86BD-9B9D7E05CAEA}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "google", "google", "{E1A81C99-B921-4EAF-9912-2B2306779E22}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "protobuf", "protobuf", "{828E3BC0-8DF8-48B5-BA98-BDB8C108C25A}" + ProjectSection(SolutionItems) = preProject + test\google\protobuf\descriptor.proto = test\google\protobuf\descriptor.proto + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -74,11 +83,17 @@ Global {F5D8D709-919D-4539-8908-91C540C970B3}.Debug|Any CPU.Build.0 = Debug|Any CPU {F5D8D709-919D-4539-8908-91C540C970B3}.Release|Any CPU.ActiveCfg = Release|Any CPU {F5D8D709-919D-4539-8908-91C540C970B3}.Release|Any CPU.Build.0 = Release|Any CPU + {AF2D437D-55A1-4F31-86BD-9B9D7E05CAEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AF2D437D-55A1-4F31-86BD-9B9D7E05CAEA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AF2D437D-55A1-4F31-86BD-9B9D7E05CAEA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AF2D437D-55A1-4F31-86BD-9B9D7E05CAEA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution {992FDC44-075B-4B9D-BFF4-0C34093627E6} = {2291EB39-A4CC-471C-BA12-1E58F298B395} + {E1A81C99-B921-4EAF-9912-2B2306779E22} = {992FDC44-075B-4B9D-BFF4-0C34093627E6} + {828E3BC0-8DF8-48B5-BA98-BDB8C108C25A} = {E1A81C99-B921-4EAF-9912-2B2306779E22} EndGlobalSection EndGlobal diff --git a/ProtoParser.Test/Froto.Parser.Test.fsproj b/ProtoParser.Test/Froto.Parser.Test.fsproj index 1cb01a8..d5e4b41 100644 --- a/ProtoParser.Test/Froto.Parser.Test.fsproj +++ b/ProtoParser.Test/Froto.Parser.Test.fsproj @@ -35,9 +35,6 @@ - - PreserveNewest - diff --git a/ProtoParser.Test/TestModel.fs b/ProtoParser.Test/TestModel.fs index 07ba36a..41393fb 100644 --- a/ProtoParser.Test/TestModel.fs +++ b/ProtoParser.Test/TestModel.fs @@ -20,7 +20,7 @@ let getTestFile file = Path.Combine(solutionPath, Path.Combine("test",file)) [] -let `` can parse SearchRequest proto`` () = +let ``can parse SearchRequest proto`` () = let proto = getTestFile "SearchRequest.proto" |> parseFile 1 |> should equal proto.Sections.Length let messages = proto.Messages diff --git a/ProtoParser.Test/TestParser.fs b/ProtoParser.Test/TestParser.fs index 522a1f4..9e51366 100644 --- a/ProtoParser.Test/TestParser.fs +++ b/ProtoParser.Test/TestParser.fs @@ -1,4 +1,4 @@ -module TestParser +namespace TestParser open Xunit open FsUnit.Xunit @@ -10,6 +10,7 @@ open FParsec open Froto.Parser.Proto open Froto.Parser.Ast +[] module Identifiers = [] let ``Identifier can be parsed`` () = @@ -53,6 +54,7 @@ module Identifiers = fun () -> parseString pGroupName_ws "myGroup" |> ignore |> should throw typeof +[] module Literals = [] @@ -166,6 +168,7 @@ module Literals = fun () -> parseString pStrLit @"'\009'" |> ignore |> should throw typeof +[] module SyntaxStatement = [] @@ -183,6 +186,7 @@ module SyntaxStatement = fun () -> parseString pSyntax @"syntax = 'protox';" |> ignore |> should throw typeof +[] module ImportStatement = [] @@ -200,6 +204,7 @@ module ImportStatement = parseString pImport @"import weak 'test.proto';" |> should equal (TImport ("test.proto", TWeak)) +[] module PackageStatement = [] @@ -207,6 +212,7 @@ module PackageStatement = parseString pPackage @"package abc.def;" |> should equal (TPackage "abc.def") +[] module OptionStatement = [] @@ -259,6 +265,7 @@ module OptionStatement = parseString pOptionStatement @"option (test).field.more = true;" |> should equal (TOption ("test.field.more", TBoolLit true)) +[] module Message = [] @@ -403,6 +410,7 @@ module Message = ]) ])) +[] module Service = [] @@ -423,6 +431,7 @@ module Service = TRpc ("TestMethod", "outer", false, "foo", false, []) ])) +[] module Proto = [] @@ -590,11 +599,21 @@ module Proto = ] ) + open System.IO + + /// gets the path for a test file based on the relative path from the executing assembly + let getTestFile file = + let codeBase = Reflection.Assembly.GetExecutingAssembly().CodeBase + let assemblyPath = DirectoryInfo (Uri codeBase).LocalPath + let solutionPath = (assemblyPath.Parent.Parent.Parent.Parent).FullName + Path.Combine(solutionPath, Path.Combine("test",file)) + [] let ``Parse Google protobuf 'descriptor.proto' without error`` () = - parseFile pProto "data/google/protobuf/descriptor.proto" + parseFile pProto <| getTestFile "google/protobuf/descriptor.proto" |> ignore +[] module Proto3 = [] @@ -697,6 +716,7 @@ module Proto3 = """ |> ignore |> should throw typeof +[] module Proto2 = [] diff --git a/ProtoParser/Froto.Parser.nuspec b/ProtoParser/Froto.Parser.nuspec index 11beaeb..1f04093 100644 --- a/ProtoParser/Froto.Parser.nuspec +++ b/ProtoParser/Froto.Parser.nuspec @@ -3,7 +3,7 @@ Froto.Parser 0.0.0 - Cameron Taggart + Cameron Taggart, James Hugard https://opensource.org/licenses/MIT https://github.com/ctaggart/froto false diff --git a/Roslyn.Test/Froto.Roslyn.Test.fsproj b/Roslyn.Test/Froto.Roslyn.Test.fsproj index 7a1b561..81ff98d 100644 --- a/Roslyn.Test/Froto.Roslyn.Test.fsproj +++ b/Roslyn.Test/Froto.Roslyn.Test.fsproj @@ -58,6 +58,17 @@ True + + + + + ..\packages\FSharp.Core\lib\net40\FSharp.Core.dll + True + True + + + + diff --git a/build.config b/build.config index 52b4c6c..4dd13fe 100644 --- a/build.config +++ b/build.config @@ -1,6 +1,6 @@ - - + + diff --git a/build.fsx b/build.fsx index a7b856c..8f738ae 100644 --- a/build.fsx +++ b/build.fsx @@ -40,6 +40,7 @@ Target "AssemblyInfo" <| fun _ -> Attribute.Version assemblyVersion Attribute.InformationalVersion iv.String ] common |> CreateFSharpAssemblyInfo "ProtoParser/AssemblyInfo.fs" + common |> CreateFSharpAssemblyInfo "Core/AssemblyInfo.fs" common |> CreateFSharpAssemblyInfo "Roslyn/AssemblyInfo.fs" common |> CreateFSharpAssemblyInfo "Exe/AssemblyInfo.fs" @@ -55,6 +56,7 @@ Target "UnitTest" <| fun _ -> Parallel = ParallelMode.All }) [ @"ProtoParser.Test\bin\Release\Froto.Parser.Test.dll" + @"Froto.Core.Test\bin\Release\Froto.Core.Test.dll" @"Roslyn.Test\bin\Release\Froto.Roslyn.Test.dll" ] @@ -65,6 +67,7 @@ Target "SourceLink" <| fun _ -> let url = "https://raw.githubusercontent.com/ctaggart/froto/{0}/%var2%" SourceLink.Index p.Compiles pdbToIndex __SOURCE_DIRECTORY__ url sourceIndex "ProtoParser/Froto.Parser.fsproj" None + sourceIndex "Core/Froto.Core.fsproj" None sourceIndex "Roslyn/Froto.Roslyn.fsproj" None sourceIndex "Exe/Exe.fsproj" None @@ -85,6 +88,20 @@ Target "NuGet" <| fun _ -> }] }) "ProtoParser/Froto.Parser.nuspec" + NuGet (fun p -> + { p with + Version = buildVersion + WorkingDir = "Core/bin/Release" + OutputPath = "bin" + DependenciesByFramework = + [{ + FrameworkVersion = "net45" + Dependencies = + [ + ] + }] + }) "Core/Froto.Core.nuspec" + NuGet (fun p -> { p with Version = buildVersion diff --git a/paket.dependencies b/paket.dependencies index 3e19987..7c9aeb1 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -14,4 +14,4 @@ nuget xunit.runner.visualstudio version_in_path: true nuget FAKE nuget SourceLink.Fake -nuget NuGet.CommandLine +nuget NuGet.CommandLine \ No newline at end of file diff --git a/readme.md b/readme.md index cbeb7ef..996dd83 100644 --- a/readme.md +++ b/readme.md @@ -7,19 +7,43 @@ * https://github.com/mgravell/protobuf-net ## NuGet - * [Froto.Parser](http://www.nuget.org/packages/Froto.Parser) + * [Froto.Parser](http://www.nuget.org/packages/Froto.Parser) - Protobuf Parser + * [Froto.Core](http://www.nuget.org/packages/Froto.Core) - Protobuf F# Serialization Framework and low-level WireFormat library. ## Build Environment Setup for Visual Studio * Install [Paket for Visual Studio](https://github.com/fsprojects/Paket.VisualStudio) from the "Tools/Extensions and Updates..." menu * Solution path cannot contain pound sign (#), such as ".../F#/froto/" [due to a .net limitation](http://stackoverflow.com/questions/9319656/how-to-encode-a-path-that-contains-a-hash) ## Status - * 2016-01-26 Complete rewrite of parser to support full proto2 and proto3 syntax - * 2014-02-28 Dusted off project and moved to GitHub - * 2012-11-02 blog [Parsing a Protocol Buffers .proto File in F#](http://blog.ctaggart.com/2012/11/parsing-protocol-buffers-proto-file-in-f.html) + * v0.2.1 (2016-03-01) Added F# serialization framework to Core, w/full wire format support. + * v0.2.0 (2016-01-26) Complete rewrite of parser to support full proto2 and proto3 syntax + * v0.1.0 (2014-02-28) v0.1.0 Dusted off project and moved to GitHub + * v0.0.5 (2012-11-02) blog [Parsing a Protocol Buffers .proto File in F#](http://blog.ctaggart.com/2012/11/parsing-protocol-buffers-proto-file-in-f.html) ## Updating from Froto 0.1.0 +### Core + * `Froto.Core` was reworked to provide serialization and deserialization of + all supported wire types and to minimize buffer copying. + + * Code depending on `Froto.Core.IO` will need to be rewritten to use the + following modules and types: + - `Froto.Core.WireFormat` + - `Froto.Core.Encoding.RawField` + - `Froto.Core.ZeroCopyBuffer` and subclasses + + * Alternatively, the functions in the `Froto.Core.Encoding.Serializer` + module can provide a slightly higher level of abstraction. + + * Or, see next. + + * Added a framework for easily constructing serializable class types. + + * See `Froto.Core.Encoding.MessageBase` for an abstract base class which + provides the serialization framework, and see + `Froto.Core.Test/ExampleProtoClass.fs` for example usage. + +### Parser * The parser now generates an AST based on Discriminated Unions, rather than objects. `Froto.Parser.Ast` (and the underlying parser) now support the full proto2 and proto3 languages. diff --git a/release_notes.md b/release_notes.md index 594c651..64fe228 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,8 +1,12 @@ +### 0.2.1 _ 2016-03 + * [#19](https://github.com/ctaggart/froto/issues/19) Improve support for serialization/deserialization + ### 0.2.0 _ 2016-02 under construction + * [#6](https://github.com/ctaggart/froto/issues/6) Add support for proto3 language * [#8](https://github.com/ctaggart/froto/issues/8) Strict/complete proto2 compliance - * [#9](https://github.com/ctaggart/froto/issues/9) updated build & dependencies - * switched from Apache 2 to MIT license + * [#9](https://github.com/ctaggart/froto/issues/9) Updated build & dependencies + * switched from Apache 2 to MIT license ### 0.1.0 _ 2014 * moved project to GitHub https://github.com/ctaggart/froto diff --git a/ProtoParser.Test/data/google/protobuf/descriptor.proto b/test/google/protobuf/descriptor.proto similarity index 100% rename from ProtoParser.Test/data/google/protobuf/descriptor.proto rename to test/google/protobuf/descriptor.proto