Replace decoded payload key with encoded key buf

Reduce data held in RAM by dozens of GB, reduce allocs,
and improve speed of: ledger update, checkpoint serialization,
and rebuilding Mtrie at startup by:
- keeping mtrie leaf node's payload key encoded in memory
- using encoded key directly while serializing checkpoint/WAL/TrieProof
- using encoded key directly while deserializing checkpoint/WAL/TrieProof

Benchmark is only for TrieUpdate.  Other improvements are not
benchmarked yet.

name          old time/op    new time/op    delta
TrieUpdate-4     439ms ± 2%     409ms ± 1%   -6.94%  (p=0.000 n=18+20)

name          old alloc/op   new alloc/op   delta
TrieUpdate-4    73.5MB ± 0%    34.1MB ± 0%  -53.60%  (p=0.000 n=20+20)

name          old allocs/op  new allocs/op  delta
TrieUpdate-4      187k ± 0%      147k ± 0%  -21.44%  (p=0.000 n=20+20)

ledger/complete/ledger_benchmark_test.go

@@ -136,7 +136,7 @@ func benchmarkStorage(steps int, b *testing.B) {
 func BenchmarkTrieUpdate(b *testing.B) {
 	// key updates per iteration
 	numInsPerStep := 10000
-	keyNumberOfParts := 10
+	keyNumberOfParts := 3
 	keyPartMinByteSize := 1
 	keyPartMaxByteSize := 100
 	valueMaxByteSize := 32

ledger/trie.go

@@ -207,18 +207,52 @@ func ToPath(pathBytes []byte) (Path, error) {
 	return path, nil
 }
 
+// encKey represents an encoded ledger key.
+type encKey []byte
+
+// Size returns the byte size of the encoded key.
+func (k encKey) Size() int {
+	return len(k)
+}
+
+// String returns the string representation of the encoded key.
+func (k encKey) String() string {
+	return hex.EncodeToString(k)
+}
+
+// Equals compares this encoded key to another encoded key.
+// A nil encoded key is equivalent to an empty encoded key.
+func (k encKey) Equals(other encKey) bool {
+	return bytes.Equal(k, other)
+}
+
+// DeepCopy returns a deep copy of the encoded key.
+func (k encKey) DeepCopy() encKey {
+	newK := make([]byte, len(k))
+	copy(newK, k)
+	return newK
+}
+
 // Payload is the smallest immutable storable unit in ledger
 type Payload struct {
-	key   Key
-	value Value
+	// encKey is key encoded using PayloadVersion.
+	// Version and type data are not encoded to save 3 bytes.
+	encKey encKey
+	value  Value
 }
 
 // Key returns payload key.
+// CAUTION: do not modify returned key because it shares underlying data with payload key.
 func (p *Payload) Key() (Key, error) {
-	return p.key, nil
+	k, err := decodeKey(p.encKey, true, PayloadVersion)
+	if err != nil {
+		return Key{}, err
+	}
+	return *k, nil
 }
 
 // Value returns payload value.
+// CAUTION: do not modify returned value because it shares underlying data with payload value.
 func (p *Payload) Value() Value {
 	return p.value
 }
@@ -228,7 +262,7 @@ func (p *Payload) Size() int {
 	if p == nil {
 		return 0
 	}
-	return p.key.Size() + p.value.Size()
+	return p.encKey.Size() + p.value.Size()
 }
 
 // IsEmpty returns true if payload is nil or value is empty
@@ -239,27 +273,19 @@ func (p *Payload) IsEmpty() bool {
 // TODO fix me
 func (p *Payload) String() string {
 	// TODO improve this key, values
-	return p.key.String() + " " + p.value.String()
-}
-
-// KeyString returns key's string representation.
-func (p *Payload) KeyString() string {
-	return p.key.String()
+	return p.encKey.String() + " " + p.value.String()
 }
 
 // Equals compares this payload to another payload
 // A nil payload is equivalent to an empty payload.
 func (p *Payload) Equals(other *Payload) bool {
-	if p == nil || (len(p.key.KeyParts) == 0 && len(p.value) == 0) {
-		return other == nil || (len(other.key.KeyParts) == 0 && len(other.value) == 0)
+	if p == nil || (p.encKey.Size() == 0 && p.value.Size() == 0) {
+		return other == nil || (other.encKey.Size() == 0 && other.value.Size() == 0)
 	}
 	if other == nil {
 		return false
 	}
-	if p.key.Equals(&other.key) && p.value.Equals(other.value) {
-		return true
-	}
-	return false
+	return p.encKey.Equals(other.encKey) && p.value.Equals(other.value)
 }
 
 // ValueEquals compares this payload value to another payload value.
@@ -287,14 +313,15 @@ func (p *Payload) DeepCopy() *Payload {
 	if p == nil {
 		return nil
 	}
-	k := p.key.DeepCopy()
+	k := p.encKey.DeepCopy()
 	v := p.value.DeepCopy()
-	return &Payload{key: k, value: v}
+	return &Payload{encKey: k, value: v}
 }
 
 // NewPayload returns a new payload
 func NewPayload(key Key, value Value) *Payload {
-	return &Payload{key: key, value: value}
+	ek := encodeKey(&key, PayloadVersion)
+	return &Payload{encKey: ek, value: value}
 }
 
 // EmptyPayload returns an empty payload

ledger/trie_encoder.go

@@ -15,6 +15,9 @@ import (
 // the newer version of data. New code handling new data version
 // should be updated to also support backward compatibility if needed.
 const (
+	// CAUTION: if payload key encoding is changed, convertEncodedPayloadKey()
+	// must be modified to convert encoded payload key from one version to
+	// another version.
 	PayloadVersion        = uint16(1)
 	TrieUpdateVersion     = uint16(0) // Use payload version 0 encoding
 	TrieProofVersion      = uint16(0) // Use payload version 0 encoding
@@ -335,7 +338,6 @@ func EncodePayload(p *Payload) []byte {
 
 	// append encoded payload content
 	buffer = append(buffer, encodePayload(p, PayloadVersion)...)
-
 	return buffer
 }
 
@@ -358,17 +360,24 @@ func encodePayload(p *Payload, version uint16) []byte {
 	return encodeAndAppendPayload(buffer, p, version)
 }
 
+// convertEncodedPayloadKey returns encoded payload key in toVersion
+// converted from encoded payload key in fromVersion.
+func convertEncodedPayloadKey(key []byte, fromVersion uint16, toVersion uint16) []byte {
+	// No conversion is needed for now because
+	// payload key encoding version 0 is the same as version 1.
+	return key
+}
+
 func encodeAndAppendPayload(buffer []byte, p *Payload, version uint16) []byte {
 
-	// Error isn't checked here because encoded key will be used directly
-	// in later commit and no error will be returned.
-	k, _ := p.Key()
+	// convert payload encoded key from PayloadVersion to version.
+	encKey := convertEncodedPayloadKey(p.encKey, PayloadVersion, version)
 
 	// encode encoded key size
-	buffer = utils.AppendUint32(buffer, uint32(encodedKeyLength(&k, version)))
+	buffer = utils.AppendUint32(buffer, uint32(len(encKey)))
 
-	// encode key
-	buffer = encodeAndAppendKey(buffer, &k, version)
+	// append encoded key
+	buffer = append(buffer, encKey...)
 
 	// encode encoded value size
 	encodedValueLen := encodedValueLength(p.Value(), version)
@@ -452,16 +461,13 @@ func decodePayload(inp []byte, zeroCopy bool, version uint16) (*Payload, error)
 	}
 
 	// read encoded key
-	encKey, rest, err := utils.ReadSlice(rest, int(encKeySize))
+	ek, rest, err := utils.ReadSlice(rest, int(encKeySize))
 	if err != nil {
 		return nil, fmt.Errorf("error decoding payload: %w", err)
 	}
 
-	// decode the key
-	key, err := decodeKey(encKey, zeroCopy, version)
-	if err != nil {
-		return nil, fmt.Errorf("error decoding payload: %w", err)
-	}
+	// convert payload encoded key from version to PayloadVersion.
+	encKey := convertEncodedPayloadKey(ek, version, PayloadVersion)
 
 	// read encoded value size
 	var encValueSize int
@@ -487,12 +493,14 @@ func decodePayload(inp []byte, zeroCopy bool, version uint16) (*Payload, error)
 	}
 
 	if zeroCopy {
-		return NewPayload(*key, encValue), nil
+		return &Payload{encKey, encValue}, nil
 	}
 
+	k := make([]byte, len(encKey))
+	copy(k, encKey)
 	v := make([]byte, len(encValue))
 	copy(v, encValue)
-	return NewPayload(*key, v), nil
+	return &Payload{k, v}, nil
 }
 
 // EncodeTrieUpdate encodes a trie update struct