From f9cc66e2785ed2325283387f33d7e80c5495cfe8 Mon Sep 17 00:00:00 2001 From: Matt Soucy Date: Tue, 5 Jul 2016 22:12:35 -0400 Subject: [PATCH 1/5] Add deserialization for packed structures --- .gitignore | 1 + import/dproto/attributes.d | 17 +++++++++++++---- import/dproto/unittests.d | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index a4c1861..bbc7fbc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *.sw[po] __test__library__ dub.selections.json +dproto_dprotoc diff --git a/import/dproto/attributes.d b/import/dproto/attributes.d index f54e6c2..5322453 100644 --- a/import/dproto/attributes.d +++ b/import/dproto/attributes.d @@ -86,7 +86,8 @@ template ProtoAccessors() alias __fieldData = getAnnotation!(__field, ProtoField); if(__msgdata.msgNum == __fieldData.fieldNumber) { enum wt = __fieldData.wireType; - __field.putProtoVal!wt(__r); + alias __isPacked = hasValueAnnotation!(__field, Packed); + __field.putProtoVal!(wt, __isPacked)(__r); __matched = true; } } @@ -162,13 +163,21 @@ void serializeField(alias field, R)(ref R r) const serializer!fieldData(rawField, r); } -void putProtoVal(string wireType, T, R)(ref T t, auto ref R r) +void putProtoVal(string wireType, alias __isPacked, T, R)(ref T t, auto ref R r) if(isProtoInputRange!R) { static if(is(T : U[], U) && !(is(T : string) || is(T : const(ubyte)[]))) { U u; - u.putSingleProtoVal!wireType(r); - t ~= u; + static if(__isPacked) { + auto nelems = r.readVarint(); + for(auto n = 0; n < nelems; n++) { + u.putSingleProtoVal!wireType(r); + t ~= u; + } + } else { + u.putSingleProtoVal!wireType(r); + t ~= u; + } } else { t.putSingleProtoVal!wireType(r); } diff --git a/import/dproto/unittests.d b/import/dproto/unittests.d index 1827cd3..50060f7 100644 --- a/import/dproto/unittests.d +++ b/import/dproto/unittests.d @@ -771,3 +771,19 @@ unittest } };`)), "Malformed proto structure accepted"); } + +unittest +{ + mixin ProtocolBufferFromString!` + message Foo { + repeated uint32 arr = 1 [packed=true]; + } +`; + + Foo foo; + foo.arr = [1]; + + auto serialized_foo = foo.serialize(); + + auto foo2 = Foo(serialized_foo); // FAILS +} From 46db4d73410b896520781b08f409903d8a962b5d Mon Sep 17 00:00:00 2001 From: Matt Soucy Date: Tue, 5 Jul 2016 22:34:10 -0400 Subject: [PATCH 2/5] Pass around more data --- import/dproto/attributes.d | 8 ++++---- import/dproto/unittests.d | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/import/dproto/attributes.d b/import/dproto/attributes.d index 5322453..c091437 100644 --- a/import/dproto/attributes.d +++ b/import/dproto/attributes.d @@ -84,10 +84,9 @@ template ProtoAccessors() foreach(__member; ProtoFields!this) { alias __field = Identity!(__traits(getMember, this, __member)); alias __fieldData = getAnnotation!(__field, ProtoField); + alias __isPacked = hasValueAnnotation!(__field, Packed); if(__msgdata.msgNum == __fieldData.fieldNumber) { - enum wt = __fieldData.wireType; - alias __isPacked = hasValueAnnotation!(__field, Packed); - __field.putProtoVal!(wt, __isPacked)(__r); + __field.putProtoVal!(__fieldData, __isPacked)(__r); __matched = true; } } @@ -163,9 +162,10 @@ void serializeField(alias field, R)(ref R r) const serializer!fieldData(rawField, r); } -void putProtoVal(string wireType, alias __isPacked, T, R)(ref T t, auto ref R r) +void putProtoVal(alias __fieldData, alias __isPacked, T, R)(ref T t, auto ref R r) if(isProtoInputRange!R) { + enum wireType = __fieldData.wireType; static if(is(T : U[], U) && !(is(T : string) || is(T : const(ubyte)[]))) { U u; static if(__isPacked) { diff --git a/import/dproto/unittests.d b/import/dproto/unittests.d index 50060f7..78934c4 100644 --- a/import/dproto/unittests.d +++ b/import/dproto/unittests.d @@ -785,5 +785,5 @@ unittest auto serialized_foo = foo.serialize(); - auto foo2 = Foo(serialized_foo); // FAILS + auto foo2 = Foo(serialized_foo); } From a53af043c3619af6c18f2f43d5b9f405fd415501 Mon Sep 17 00:00:00 2001 From: Matt Soucy Date: Tue, 5 Jul 2016 23:02:41 -0400 Subject: [PATCH 3/5] Pass through more information directly --- import/dproto/attributes.d | 41 ++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/import/dproto/attributes.d b/import/dproto/attributes.d index c091437..1920731 100644 --- a/import/dproto/attributes.d +++ b/import/dproto/attributes.d @@ -86,7 +86,7 @@ template ProtoAccessors() alias __fieldData = getAnnotation!(__field, ProtoField); alias __isPacked = hasValueAnnotation!(__field, Packed); if(__msgdata.msgNum == __fieldData.fieldNumber) { - __field.putProtoVal!(__fieldData, __isPacked)(__r); + __r.putProtoVal!(__field)(); __matched = true; } } @@ -162,24 +162,35 @@ void serializeField(alias field, R)(ref R r) const serializer!fieldData(rawField, r); } -void putProtoVal(alias __fieldData, alias __isPacked, T, R)(ref T t, auto ref R r) - if(isProtoInputRange!R) +void putProtoVal(alias __field, R)(auto ref R r) + if (isProtoInputRange!R) { - enum wireType = __fieldData.wireType; - static if(is(T : U[], U) && !(is(T : string) || is(T : const(ubyte)[]))) { - U u; - static if(__isPacked) { + import std.range.primitives : ElementType; + import std.traits : isSomeString, isDynamicArray; + + alias T = typeof(__field); + enum wireType = getAnnotation!(__field, ProtoField).wireType; + enum isBinString(T) = Identity!(is(T : const(ubyte)[])); + static if (isDynamicArray!T && !(isSomeString!T || isBinString!T)) + { + ElementType!T u; + static if (hasValueAnnotation!(__field, Packed)) + { auto nelems = r.readVarint(); - for(auto n = 0; n < nelems; n++) { - u.putSingleProtoVal!wireType(r); - t ~= u; - } - } else { + } + else + { + auto nelems = 1; + } + for (auto n = 0; n < nelems; n++) + { u.putSingleProtoVal!wireType(r); - t ~= u; + __field ~= u; } - } else { - t.putSingleProtoVal!wireType(r); + } + else + { + __field.putSingleProtoVal!wireType(r); } } From 75845beac4c09e7e22db2e5a8e1622e348932888 Mon Sep 17 00:00:00 2001 From: Matt Soucy Date: Tue, 5 Jul 2016 23:04:18 -0400 Subject: [PATCH 4/5] Backward compatible --- import/dproto/attributes.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/dproto/attributes.d b/import/dproto/attributes.d index 1920731..c55c56d 100644 --- a/import/dproto/attributes.d +++ b/import/dproto/attributes.d @@ -165,7 +165,7 @@ void serializeField(alias field, R)(ref R r) const void putProtoVal(alias __field, R)(auto ref R r) if (isProtoInputRange!R) { - import std.range.primitives : ElementType; + import std.range : ElementType; import std.traits : isSomeString, isDynamicArray; alias T = typeof(__field); From e6800f95f27e184166272c9b094ddafb586342ae Mon Sep 17 00:00:00 2001 From: Matt Soucy Date: Sat, 9 Jul 2016 14:24:27 -0400 Subject: [PATCH 5/5] Unpack packed data regardless of option being set --- import/dproto/attributes.d | 14 +++++--------- import/dproto/unittests.d | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/import/dproto/attributes.d b/import/dproto/attributes.d index c55c56d..bb59d32 100644 --- a/import/dproto/attributes.d +++ b/import/dproto/attributes.d @@ -84,9 +84,8 @@ template ProtoAccessors() foreach(__member; ProtoFields!this) { alias __field = Identity!(__traits(getMember, this, __member)); alias __fieldData = getAnnotation!(__field, ProtoField); - alias __isPacked = hasValueAnnotation!(__field, Packed); if(__msgdata.msgNum == __fieldData.fieldNumber) { - __r.putProtoVal!(__field)(); + __r.putProtoVal!(__field)(__msgdata); __matched = true; } } @@ -162,7 +161,7 @@ void serializeField(alias field, R)(ref R r) const serializer!fieldData(rawField, r); } -void putProtoVal(alias __field, R)(auto ref R r) +void putProtoVal(alias __field, R)(auto ref R r, ulong __msgdata) if (isProtoInputRange!R) { import std.range : ElementType; @@ -174,13 +173,10 @@ void putProtoVal(alias __field, R)(auto ref R r) static if (isDynamicArray!T && !(isSomeString!T || isBinString!T)) { ElementType!T u; - static if (hasValueAnnotation!(__field, Packed)) + ulong nelems = 1; + if (wireType.msgType != PACKED_MSG_TYPE && __msgdata.wireType == PACKED_MSG_TYPE) { - auto nelems = r.readVarint(); - } - else - { - auto nelems = 1; + nelems = r.readVarint(); } for (auto n = 0; n < nelems; n++) { diff --git a/import/dproto/unittests.d b/import/dproto/unittests.d index 78934c4..36fd5d5 100644 --- a/import/dproto/unittests.d +++ b/import/dproto/unittests.d @@ -787,3 +787,25 @@ unittest auto foo2 = Foo(serialized_foo); } + +unittest +{ + import std.algorithm; + mixin ProtocolBufferFromString!` + message FooA { + repeated uint32 arr = 1 [packed=true]; + } + message FooB { + repeated uint32 arr = 1; + } +`; + + FooA foo; + foo.arr = [1,3,5,7,2,4,6,8]; + + auto serialized_foo = foo.serialize(); + auto foo2 = FooB(serialized_foo); + assert(equal(foo.arr, foo2.arr)); + auto foo3 = FooA(foo2.serialize()); + assert(equal(foo2.arr, foo3.arr)); +}