Merge branch 'v2/develop' into iproudhon/v2/develop

go.mod

@@ -13,6 +13,7 @@ require (
  github.com/confio/ics23/go v0.6.3
  github.com/cosmos/go-bip39 v1.0.0
  github.com/cosmos/ledger-cosmos-go v0.11.1
+ github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de
  github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b
  github.com/enigmampc/btcutil v1.0.3-0.20200723161021-e2fb6adb2a25
  github.com/go-kit/kit v0.10.0

server/config/config.go

@@ -74,6 +74,9 @@ type BaseConfig struct {
  // IAVL cache size; bytes size unit
  IAVLCacheSize int `mapstructure:"iavl-cache-size"`
 
+ // Bech32CacheSize is the maximum bytes size of bech32 cache (Default : 1GB)
+ Bech32CacheSize int `mapstructure:"bech32-cache-size"`
+
  // When true, Prometheus metrics are served under /metrics on prometheus_listen_addr in config.toml.
  // It works when tendermint's prometheus option (config.toml) is set to true.
  Prometheus bool `mapstructure:"prometheus"`
@@ -179,6 +182,7 @@ func DefaultConfig() *Config {
  InterBlockCache: true,
  InterBlockCacheSize: cache.DefaultCommitKVStoreCacheSize,
  IAVLCacheSize: iavl.DefaultIAVLCacheSize,
+ Bech32CacheSize: sdk.DefaultBech32CacheSize,
  Pruning: storetypes.PruningOptionDefault,
  PruningKeepRecent: "0",
  PruningKeepEvery: "0",

server/config/toml.go

@@ -69,6 +69,9 @@ inter-block-cache-size = {{ .BaseConfig.InterBlockCacheSize }}
 # IAVLCacheSize is the maximum bytes size of iavl node cache
 iavl-cache-size = {{ .BaseConfig.IAVLCacheSize }}
 
+# Bech32CacheSize is the maximum bytes size of bech32 cache (Default : 1GB)
+bech32-cache-size = {{ .BaseConfig.Bech32CacheSize }}
+
 # IndexEvents defines the set of events in the form {eventType}.{attributeKey},
 # which informs Tendermint what to index. If empty, all events will be indexed.
 #

server/start.go

@@ -43,6 +43,7 @@ const (
  FlagInterBlockCache = "inter-block-cache"
  FlagInterBlockCacheSize = "inter-block-cache-size"
  FlagIAVLCacheSize = "iavl-cache-size"
+ FlagBech32CacheSize = "bech32-cache-size"
  FlagUnsafeSkipUpgrades = "unsafe-skip-upgrades"
  FlagTrace = "trace"
  FlagInvCheckPeriod = "inv-check-period"

simapp/simd/cmd/root.go

@@ -165,6 +165,11 @@ func (a appCreator) newApp(logger log.Logger, db tmdb.DB, traceStore io.Writer,
  cast.ToInt(appOpts.Get(server.FlagInterBlockCacheSize)), ibCacheMetricsProvider)
  }
 
+ bech32CacheSize := cast.ToInt(appOpts.Get(server.FlagBech32CacheSize))
+ if bech32CacheSize > 0 {
+ sdk.SetBech32Cache(int64(bech32CacheSize))
+ }
+
  skipUpgradeHeights := make(map[int64]bool)
  for _, h := range cast.ToIntSlice(appOpts.Get(server.FlagUnsafeSkipUpgrades)) {
  skipUpgradeHeights[int64(h)] = true

types/address.go

@@ -8,6 +8,7 @@ import (
  "fmt"
  "strings"
 
+ "github.com/dgraph-io/ristretto"
  yaml "gopkg.in/yaml.v2"
 
  "github.com/line/lbm-sdk/v2/codec/legacy"
@@ -92,6 +93,63 @@ var _ yaml.Marshaler = ConsAddress{}
 // account
 // ----------------------------------------------------------------------------
 
+// TODO We should add a layer to choose whether to access the cache or to run actual conversion
+// bech32 encoding and decoding takes a lot of time, so memoize it
+var bech32Cache Bech32Cache
+
+type Bech32Cache struct {
+ bech32ToAddrCache *ristretto.Cache
+ addrToBech32Cache *ristretto.Cache
+}
+
+func SetBech32Cache(size int64) {
+ var err error
+ config := &ristretto.Config{
+ NumCounters: 1e7, // number of keys to track frequency of (10M).
+ MaxCost: size,
+ BufferItems: 64, // number of keys per Get buffer.
+ }
+ bech32Cache.bech32ToAddrCache, err = ristretto.NewCache(config)
+ if err != nil {
+ panic(err)
+ }
+ bech32Cache.addrToBech32Cache, err = ristretto.NewCache(config)
+ if err != nil {
+ panic(err)
+ }
+}
+
+func (cache *Bech32Cache) GetAddr(bech32Addr string) ([]byte, bool) {
+ if cache.bech32ToAddrCache != nil {
+ rawAddr, ok := cache.bech32ToAddrCache.Get(bech32Addr)
+ if ok {
+ return rawAddr.([]byte), ok
+ }
+ }
+ return nil, false
+}
+
+func (cache *Bech32Cache) GetBech32(rawAddr []byte) (string, bool) {
+ if cache.addrToBech32Cache != nil {
+ bech32Addr, ok := cache.addrToBech32Cache.Get(string(rawAddr))
+ if ok {
+ return bech32Addr.(string), ok
+ }
+ }
+ return "", false
+}
+
+func (cache *Bech32Cache) Set(bech32Addr string, rawAddr []byte) {
+ if cache.bech32ToAddrCache != nil {
+ cache.bech32ToAddrCache.Set(bech32Addr, rawAddr, int64(len(rawAddr)))
+ }
+ if cache.addrToBech32Cache != nil {
+ cache.addrToBech32Cache.Set(string(rawAddr), bech32Addr, int64(len(bech32Addr)))
+ }
+}
+
+const DefaultBech32CacheSize = 1 << 30 // maximum size of cache (1GB).
+
 // AccAddress a wrapper around bytes meant to represent an account address.
 // When marshaled to a string or JSON, it uses Bech32.
 type AccAddress []byte
@@ -117,14 +175,19 @@ func VerifyAddressFormat(bz []byte) error {
 }
 
 // AccAddressFromBech32 creates an AccAddress from a Bech32 string.
-func AccAddressFromBech32(address string) (addr AccAddress, err error) {
- if len(strings.TrimSpace(address)) == 0 {
+func AccAddressFromBech32(bech32Addr string) (AccAddress, error) {
+ addr, ok := bech32Cache.GetAddr(bech32Addr)
+ if ok {
+ return addr, nil
+ }
+
+ if len(strings.TrimSpace(bech32Addr)) == 0 {
  return AccAddress{}, errors.New("empty address string is not allowed")
  }
 
  bech32PrefixAccAddr := GetConfig().GetBech32AccountAddrPrefix()
 
- bz, err := GetFromBech32(address, bech32PrefixAccAddr)
+ bz, err := GetFromBech32(bech32Addr, bech32PrefixAccAddr)
  if err != nil {
  return nil, err
  }
@@ -133,8 +196,8 @@ func AccAddressFromBech32(address string) (addr AccAddress, err error) {
  if err != nil {
  return nil, err
  }
-
- return AccAddress(bz), nil
+ bech32Cache.Set(bech32Addr, bz)
+ return bz, nil
 }
 
 // Returns boolean for whether two AccAddresses are Equal
@@ -229,6 +292,11 @@ func (aa AccAddress) Bytes() []byte {
 
 // String implements the Stringer interface.
 func (aa AccAddress) String() string {
+ bech32Addr, ok := bech32Cache.GetBech32(aa)
+ if ok {
+ return bech32Addr
+ }
+
  if aa.Empty() {
  return ""
  }
@@ -239,7 +307,7 @@ func (aa AccAddress) String() string {
  if err != nil {
  panic(err)
  }
-
+ bech32Cache.Set(bech32Addr, aa)
  return bech32Addr
 }
 

types/address_test.go

@@ -520,3 +520,35 @@ func (s *addressTestSuite) TestGetFromBech32() {
  s.Require().Error(err)
  s.Require().Equal("invalid Bech32 prefix; expected x, got cosmos", err.Error())
 }
+
+func (s *addressTestSuite) TestBech32Cache() {
+ pubBz := make([]byte, ed25519.PubKeySize)
+ pub := &ed25519.PubKey{Key: pubBz}
+
+ s.T().Log("access bech32ToAddrCache before access addrToBech32Cache")
+ {
+ rand.Read(pub.Key)
+ addr := types.AccAddress(pub.Address())
+ bech32Addr := addr.String()
+ types.SetBech32Cache(types.DefaultBech32CacheSize)
+
+ rawAddr, err := types.AccAddressFromBech32(bech32Addr)
+ s.Require().Nil(err)
+ require.Equal(s.T(), addr, rawAddr)
+
+ require.Equal(s.T(), bech32Addr, addr.String())
+ }
+ s.T().Log("access addrToBech32Cache before access bech32ToAddrCache")
+ {
+ rand.Read(pub.Key)
+ addr := types.AccAddress(pub.Address())
+ bech32Addr := addr.String()
+ types.SetBech32Cache(types.DefaultBech32CacheSize)
+
+ require.Equal(s.T(), bech32Addr, addr.String())
+
+ rawAddr, err := types.AccAddressFromBech32(bech32Addr)
+ s.Require().Nil(err)
+ require.Equal(s.T(), addr, rawAddr)
+ }
+}