Skip to content

Commit

Permalink
codec: optimize decoding for ZeroCopy mode and use mode for benchmarks
Browse files Browse the repository at this point in the history 
NewDecoderBytes with ZeroCopy=true will return []byte and string that point into the input []byte.

This reduces allocation and improves performance, in both codecgen and normal mode,
showing ~ 50% reduction in allocation.

To support this, we track decByteState after during each call to DecodeBytes or DecodeStringAsByte.
Folks that care can check the value right after the call.

We also optimized json decoding, so we can return a view into the input []byte if possible,
instead of always doing a copy.

Helper functions were added to make it easy to get a string value from a []byte with optimization e.g.
zerocopy, interning, etc.

We now run benchmarks by default with ZeroCopy=true.
  • Loading branch information
@ugorji
ugorji committed Jan 5, 2021
1 parent 75391f7 commit ec0371e
Show file tree
Hide file tree
Showing 24 changed files with 621 additions and 438 deletions.
2 changes: 1 addition & 1 deletion codec/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ some caveats. See Encode documentation.

```go
const CborStreamBytes byte = 0x5f ...
const GenVersion = 20
const GenVersion = 21
var SelfExt = &extFailWrapper{}
var GoRpc goRpc
var MsgpackSpecRpc msgpackSpecRpc
Expand Down
6 changes: 4 additions & 2 deletions codec/bench/bench.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ _usage() {
printf "\t-sgx run test suite for codec; if g, use generated files; if x, do external also\n"
printf "\t-jqp run test suite for [json, json-quick, json-profile]\n"
printf "\t-z run tests for bench.out.txt\n"
printf "\t-f [pprof file] run pprof\n"
}

_main() {
Expand All @@ -211,10 +212,10 @@ _main() {
local args=()
local do_x="0"
local do_g="0"
while getopts "dcbsjqptxklgz" flag
while getopts "dcbsjqptxklgzf" flag
do
case "$flag" in
d|c|b|s|j|q|p|t|x|k|l|g|z) args+=( "$flag" ) ;;
d|c|b|s|j|q|p|t|x|k|l|g|z|f) args+=( "$flag" ) ;;
*) _usage; return 1 ;;
esac
done
Expand All @@ -237,6 +238,7 @@ _main() {
[[ " ${args[*]} " == *"p"* ]] && _suite_very_quick_json_only_profile "$@" | _suite_trim_output
[[ " ${args[*]} " == *"t"* ]] && _suite_tests "$@" | _suite_trim_output | _suite_tests_strip_file_line
[[ " ${args[*]} " == *"z"* ]] && _bench_dot_out_dot_txt
[[ " ${args[*]} " == *"f"* ]] && ${gocmd} tool pprof bench.test ${1:-mem.out}

true
# shift $((OPTIND-1))
Expand Down
7 changes: 7 additions & 0 deletions codec/bench/bench_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,13 @@ func benchPreInit() {
// if bytesLen < approxSize {
// bytesLen = approxSize
// }

// use zerocopy for the benchmarks, for best performance
testJsonH.ZeroCopy = true
testCborH.ZeroCopy = true
testMsgpackH.ZeroCopy = true
testSimpleH.ZeroCopy = true
testBincH.ZeroCopy = true
}

func benchReinit() {
Expand Down
3 changes: 0 additions & 3 deletions codec/bench/shared_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,9 +170,6 @@ func testHEDGet(h Handle) *testHED {
func testReinit() {
testOnce = sync.Once{}
testHEDs = nil
// for _, v := range testHandles {
// v.getBasicHandle().clearInited() // so it is reinitialized next time around
// }
}

func testInitAll() {
Expand Down
54 changes: 28 additions & 26 deletions codec/binc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -702,24 +702,22 @@ func (d *bincDecDriver) decLenNumber() (v uint64) {
return
}

func (d *bincDecDriver) decStringBytes(bs []byte, zerocopy bool) (bs2 []byte) {
// func (d *bincDecDriver) decStringBytes(bs []byte, zerocopy bool) (bs2 []byte) {
func (d *bincDecDriver) DecodeStringAsBytes() (bs2 []byte) {
d.d.decByteState = decByteStateNone
if d.advanceNil() {
return
}
var slen = -1
switch d.vd {
case bincVdString, bincVdByteArray:
slen = d.decLen()
if zerocopy {
if d.d.bytes {
bs2 = d.d.decRd.rb.readx(uint(slen))
} else if len(bs) == 0 {
bs2 = decByteSlice(d.d.r(), slen, d.d.h.MaxInitLen, d.d.b[:])
} else {
bs2 = decByteSlice(d.d.r(), slen, d.d.h.MaxInitLen, bs)
}
if d.d.bytes {
d.d.decByteState = decByteStateZerocopy
bs2 = d.d.decRd.rb.readx(uint(slen))
} else {
bs2 = decByteSlice(d.d.r(), slen, d.d.h.MaxInitLen, bs)
d.d.decByteState = decByteStateReuseBuf
bs2 = decByteSlice(d.d.r(), slen, d.d.h.MaxInitLen, d.d.b[:])
}
case bincVdSymbol:
// zerocopy doesn't apply for symbols,
Expand Down Expand Up @@ -760,20 +758,21 @@ func (d *bincDecDriver) decStringBytes(bs []byte, zerocopy bool) (bs2 []byte) {
return
}

func (d *bincDecDriver) DecodeStringAsBytes() (s []byte) {
return d.decStringBytes(d.d.b[:], true)
}

func (d *bincDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
func (d *bincDecDriver) DecodeBytes(bs []byte) (bsOut []byte) {
d.d.decByteState = decByteStateNone
if d.advanceNil() {
return
}
if d.vd == bincVdArray {
if zerocopy && len(bs) == 0 {
if bs == nil {
bs = d.d.b[:]
d.d.decByteState = decByteStateReuseBuf
}
slen := d.ReadArrayStart()
bs = usableByteSlice(bs, slen)
var changed bool
if bs, changed = usableByteSlice(bs, slen); changed {
d.d.decByteState = decByteStateNone
}
for i := 0; i < slen; i++ {
bs[i] = uint8(chkOvf.UintV(d.DecodeUint64(), 8))
}
Expand All @@ -786,11 +785,13 @@ func (d *bincDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
d.d.errorf("bytes - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs))
}
d.bdRead = false
if d.d.bytes && (zerocopy || d.h.ZeroCopy) {
if d.d.bytes && d.h.ZeroCopy {
d.d.decByteState = decByteStateZerocopy
return d.d.decRd.rb.readx(uint(clen))
}
if zerocopy && len(bs) == 0 {
if bs == nil {
bs = d.d.b[:]
d.d.decByteState = decByteStateReuseBuf
}
return decByteSlice(d.d.r(), clen, d.d.h.MaxInitLen, bs)
}
Expand All @@ -802,20 +803,20 @@ func (d *bincDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) {
if d.advanceNil() {
return
}
realxtag1, xbs := d.decodeExtV(ext != nil, uint8(xtag))
xbs, realxtag1, zerocopy := d.decodeExtV(ext != nil, uint8(xtag))
realxtag := uint64(realxtag1)
if ext == nil {
re := rv.(*RawExt)
re.Tag = realxtag
re.Data = detachZeroCopyBytes(d.d.bytes, re.Data, xbs)
re.setData(xbs, zerocopy)
} else if ext == SelfExt {
d.d.sideDecode(rv, xbs)
} else {
ext.ReadExt(rv, xbs)
}
}

func (d *bincDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs []byte) {
func (d *bincDecDriver) decodeExtV(verifyTag bool, tag byte) (xbs []byte, xtag byte, zerocopy bool) {
if d.vd == bincVdCustomExt {
l := d.decLen()
xtag = d.d.decRd.readn1()
Expand All @@ -824,11 +825,12 @@ func (d *bincDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs []b
}
if d.d.bytes {
xbs = d.d.decRd.rb.readx(uint(l))
zerocopy = true
} else {
xbs = decByteSlice(d.d.r(), l, d.d.h.MaxInitLen, d.d.b[:])
}
} else if d.vd == bincVdByteArray {
xbs = d.DecodeBytes(nil, true)
xbs = d.DecodeBytes(nil)
} else {
d.d.errorf("ext expects extensions or byte array - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs))
}
Expand Down Expand Up @@ -890,12 +892,12 @@ func (d *bincDecDriver) DecodeNaked() {
n.f = d.decFloat()
case bincVdString:
n.v = valueTypeString
n.s = d.d.string(d.DecodeStringAsBytes())
n.s = d.d.stringZC(d.DecodeStringAsBytes())
case bincVdByteArray:
d.d.fauxUnionReadRawBytes()
d.d.fauxUnionReadRawBytes(false)
case bincVdSymbol:
n.v = valueTypeSymbol
n.s = d.d.string(d.DecodeStringAsBytes())
n.s = d.d.stringZC(d.DecodeStringAsBytes())
case bincVdTimestamp:
n.v = valueTypeTime
tt, err := bincDecodeTime(d.d.decRd.readx(uint(d.vs)))
Expand Down
35 changes: 19 additions & 16 deletions codec/cbor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -572,7 +572,8 @@ func (d *cborDecDriver) ReadArrayStart() (length int) {
return d.decLen()
}

func (d *cborDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
func (d *cborDecDriver) DecodeBytes(bs []byte) (bsOut []byte) {
d.d.decByteState = decByteStateNone
if d.advanceNil() {
return
}
Expand All @@ -582,20 +583,16 @@ func (d *cborDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
if d.bd == cborBdIndefiniteBytes || d.bd == cborBdIndefiniteString {
d.bdRead = false
if bs == nil {
if zerocopy {
return d.decAppendIndefiniteBytes(d.d.b[:0])
}
return d.decAppendIndefiniteBytes(zeroByteSlice)
d.d.decByteState = decByteStateReuseBuf
return d.decAppendIndefiniteBytes(d.d.b[:0])
}
return d.decAppendIndefiniteBytes(bs[:0])
}
if d.bd == cborBdIndefiniteArray {
d.bdRead = false
if zerocopy && len(bs) == 0 {
bs = d.d.b[:]
}
if bs == nil {
bs = []byte{}
d.d.decByteState = decByteStateReuseBuf
bs = d.d.b[:0]
} else {
bs = bs[:0]
}
Expand All @@ -606,29 +603,35 @@ func (d *cborDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
}
if d.bd>>5 == cborMajorArray {
d.bdRead = false
if zerocopy && len(bs) == 0 {
if bs == nil {
d.d.decByteState = decByteStateReuseBuf
bs = d.d.b[:]
}
slen := d.decLen()
bs = usableByteSlice(bs, slen)
var changed bool
if bs, changed = usableByteSlice(bs, slen); changed {
d.d.decByteState = decByteStateNone
}
for i := 0; i < len(bs); i++ {
bs[i] = uint8(chkOvf.UintV(d.DecodeUint64(), 8))
}
return bs
}
clen := d.decLen()
d.bdRead = false
if d.d.bytes && (zerocopy || d.h.ZeroCopy) {
if d.d.bytes && d.h.ZeroCopy {
d.d.decByteState = decByteStateZerocopy
return d.d.decRd.rb.readx(uint(clen))
}
if zerocopy && len(bs) == 0 {
if bs == nil {
d.d.decByteState = decByteStateReuseBuf
bs = d.d.b[:]
}
return decByteSlice(d.d.r(), clen, d.h.MaxInitLen, bs)
}

func (d *cborDecDriver) DecodeStringAsBytes() (s []byte) {
return d.DecodeBytes(d.d.b[:], true)
return d.DecodeBytes(nil)
}

func (d *cborDecDriver) DecodeTime() (t time.Time) {
Expand Down Expand Up @@ -704,10 +707,10 @@ func (d *cborDecDriver) DecodeNaked() {
n.v = valueTypeInt
n.i = d.DecodeInt64()
case cborMajorBytes:
d.d.fauxUnionReadRawBytes()
d.d.fauxUnionReadRawBytes(false)
case cborMajorString:
n.v = valueTypeString
n.s = d.d.string(d.DecodeStringAsBytes())
n.s = d.d.stringZC(d.DecodeStringAsBytes())
case cborMajorArray:
n.v = valueTypeArray
decodeFurther = true
Expand Down
2 changes: 1 addition & 1 deletion codec/codec_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1530,7 +1530,7 @@ func doTestMapEncodeForCanonical(t *testing.T, h Handle) {
if !bytes.Equal(b1t, b2t) {
t.Logf("Unequal bytes of length: %v vs %v", len(b1t), len(b2t))
if testVerbose {
t.Logf("Unequal bytes: %v VS %v", b1t, b2t)
t.Logf("Unequal bytes: \n\t%v \n\tVS \n\t%v", b1t, b2t)
}
t.FailNow()
}
Expand Down
Loading

0 comments on commit ec0371e

Please sign in to comment.