Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bug Fix: Saving big payloads by cache CouchDB #1909

Merged
merged 3 commits into from
Sep 24, 2020
Merged

Bug Fix: Saving big payloads by cache CouchDB #1909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2bc4d95
Bug Fix: Saving big payloads by cache CouchDB
przemo22 Sep 22, 2020
fb10369
Update fastcache and use new API from fastcache
przemo22 Sep 23, 2020
da178bc
Cache support only small payloads
przemo22 Sep 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions core/ledger/kvledger/txmgmt/statedb/statecouchdb/cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,12 @@ func (c *cache) getState(chainID, namespace, key string) (*CacheValue, error) {
}

cacheKey := constructCacheKey(chainID, namespace, key)
valBytes := cache.Get(nil, cacheKey)
if valBytes == nil {

if !cache.Has(cacheKey) {
return nil, nil
}

cacheValue := &CacheValue{}
valBytes := cache.Get(nil, cacheKey)
if err := proto.Unmarshal(valBytes, cacheValue); err != nil {
return nil, err
}
Expand All @@ -85,9 +85,15 @@ func (c *cache) putState(chainID, namespace, key string, cacheValue *CacheValue)

cacheKey := constructCacheKey(chainID, namespace, key)
valBytes, err := proto.Marshal(cacheValue)

if err != nil {
return err
}

if cache.Has(cacheKey) {
cache.Del(cacheKey)
}

cache.Set(cacheKey, valBytes)
return nil
}
Expand Down Expand Up @@ -126,11 +132,12 @@ func (c *cache) UpdateStates(chainID string, updates cacheUpdates) error {
cache.Del(cacheKey)
continue
}
if oldVal := cache.Get(nil, cacheKey); oldVal != nil {
if cache.Has(cacheKey) {
newValBytes, err := proto.Marshal(newVal)
if err != nil {
return err
}
cache.Del(cacheKey)
cache.Set(cacheKey, newValBytes)
}
}
Expand Down
62 changes: 62 additions & 0 deletions core/ledger/kvledger/txmgmt/statedb/statecouchdb/cache_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ SPDX-License-Identifier: Apache-2.0
package statecouchdb

import (
"math/rand"
"testing"

"github.com/VictoriaMetrics/fastcache"
Expand Down Expand Up @@ -57,6 +58,67 @@ func TestGetPutState(t *testing.T) {
require.True(t, proto.Equal(expectedValue1, v))
}

func TestGetPutStateWithBigPayloadIfKeyDoesNotExist(t *testing.T) {
cache := newCache(32, sysNamespaces)

expectedValue := &CacheValue{Value: []byte("value")}
require.NoError(t, cache.putState("ch1", "ns1", "k1", expectedValue))
v, err := cache.getState("ch1", "ns1", "k1")
require.NoError(t, err)
require.True(t, proto.Equal(expectedValue, v))

// test PutState with BigPayload
token := make([]byte, (64*1024)+1)
rand.Read(token)
expectedValue1 := &CacheValue{Value: token}
require.NoError(t, cache.putState("ch1", "ns1", "k1", expectedValue1))

Comment on lines +74 to +75
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar to this, you could have added a test for the function cache.UpdateStates, as that is not covered in this test.

v, err = cache.getState("ch1", "ns1", "k1")
require.NoError(t, err)
// actually bigPayloads are not saved in cache, should return nil/nothing
require.Nil(t, v)
}

func TestUpdateStatesWithSingleSmallAndSingleBigPayloads(t *testing.T) {
cache := newCache(32, sysNamespaces)

expectedValue1 := &CacheValue{Value: []byte("value1")}
require.NoError(t, cache.putState("ch1", "ns1", "k1", expectedValue1))

expectedValue2 := &CacheValue{Value: []byte("value2")}
require.NoError(t, cache.putState("ch1", "ns1", "k2", expectedValue2))

v1, err := cache.getState("ch1", "ns1", "k1")
require.NoError(t, err)
require.True(t, proto.Equal(expectedValue1, v1))

v2, err := cache.getState("ch1", "ns1", "k2")
require.NoError(t, err)
require.True(t, proto.Equal(expectedValue2, v2))

token := make([]byte, (64*1024)+1)
rand.Read(token)

expectedValue3 := &CacheValue{Value: []byte("value3")}
expectedValue4 := &CacheValue{Value: token}

updates := cacheUpdates{
"ns1": cacheKVs{
"k1": expectedValue3,
"k2": expectedValue4,
},
}
require.NoError(t, cache.UpdateStates("ch1", updates))

v3, err := cache.getState("ch1", "ns1", "k1")
require.NoError(t, err)
require.True(t, proto.Equal(expectedValue3, v3))

v4, err := cache.getState("ch1", "ns1", "k2")
require.NoError(t, err)
require.Nil(t, v4)
}

func TestUpdateStates(t *testing.T) {
cache := newCache(32, sysNamespaces)

Expand Down
3 changes: 2 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ require (
github.com/Microsoft/hcsshim v0.8.6 // indirect
github.com/Shopify/sarama v1.20.1
github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect
github.com/VictoriaMetrics/fastcache v1.4.6
github.com/VictoriaMetrics/fastcache v1.5.7
github.com/VividCortex/gohistogram v1.0.0 // indirect
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect
github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect
Expand All @@ -28,6 +28,7 @@ require (
github.com/fsouza/go-dockerclient v1.4.1
github.com/go-kit/kit v0.8.0
github.com/golang/protobuf v1.3.3
github.com/golang/snappy v0.0.2 // indirect
github.com/google/go-cmp v0.5.0 // indirect
github.com/gorilla/handlers v1.4.0
github.com/gorilla/mux v1.7.2
Expand Down
19 changes: 8 additions & 11 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,30 +13,28 @@ github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiU
github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Shopify/sarama v1.20.1 h1:Bb0h3I++r4eX333Y0uZV2vwUXepJbt6ig05TUU1qt9I=
github.com/Shopify/sarama v1.20.1/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/VictoriaMetrics/fastcache v1.4.6 h1:cNywEjbbl77KfS20Nim0ukHMcBq9DGwOR5Pwazv2mw0=
github.com/VictoriaMetrics/fastcache v1.4.6/go.mod h1:FZV1r1HyPy1UHCTJ4vuuYJ1oDYM5SRPwOj9wlxbhog4=
github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw=
github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE=
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/allegro/bigcache v1.1.1-0.20190116153254-84a0ff3f153c h1:JnxT952szw65TwPg7iz1iYVqEoikvX8mSkSgIhNr0Vo=
github.com/allegro/bigcache v1.1.1-0.20190116153254-84a0ff3f153c/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
Expand Down Expand Up @@ -103,9 +101,10 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
Expand Down Expand Up @@ -243,8 +242,6 @@ github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.3.1 h1:GPTpEAuNr98px18yNQ66JllNil98wfRZ/5Ukny8FeQA=
github.com/spf13/afero v1.3.1/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
Expand Down
20 changes: 0 additions & 20 deletions vendor/github.com/VictoriaMetrics/fastcache/.travis.yml
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion vendor/github.com/VictoriaMetrics/fastcache/README.md
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion vendor/github.com/VictoriaMetrics/fastcache/bigcache.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

51 changes: 41 additions & 10 deletions vendor/github.com/VictoriaMetrics/fastcache/fastcache.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading