feat: modify cache sizes to be configurable

baseapp/options.go

@@ -51,6 +51,11 @@ func SetTrace(trace bool) func(*BaseApp) {
  return func(app *BaseApp) { app.setTrace(trace) }
 }
 
+// SetIAVLCacheSize sets the maximum number of entries in the all iavl node caches
+func SetIAVLCacheSize(cacheSize int) func(*BaseApp) {
+ return func(app *BaseApp) { app.cms.SetIAVLCacheSize(cacheSize) }
+}
+
 // SetMetrics sets prometheus metrics
 func SetMetrics(bappMetrics *Metrics, storeMetrics *storetypes.Metrics, iavlMetricsProvider iavl.MetricsProvider) func(*BaseApp) {
  return func(app *BaseApp) {

go.mod

@@ -46,6 +46,6 @@ require (
 
 replace (
  github.com/keybase/go-keychain => github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4
- github.com/tendermint/iavl v0.14.3 => github.com/line/iavl v0.14.4-0.20210205104021-924a85cfa292
+ github.com/tendermint/iavl v0.14.3 => github.com/line/iavl v0.14.4-0.20210208092835-82115f00616e
  github.com/tendermint/tendermint v0.33.9 => github.com/line/tendermint v0.33.9-0.1.0-rc2
 )

go.sum

@@ -267,6 +267,8 @@ github.com/line/iavl v0.14.3-0.1.0-rc2 h1:3bZ7NYHTgK4NK/wEmXTQqpJY4yDwriwq8PDYGB
 github.com/line/iavl v0.14.3-0.1.0-rc2/go.mod h1:eG6hI8RbMxL1nR+nJBykXD//gKjUpKCAT2tvi9V93sA=
 github.com/line/iavl v0.14.4-0.20210205104021-924a85cfa292 h1:C+ZVBfPd6Jt4ZbCXQ/M7vlWf7rO96E6mFz+2YrNJ29c=
 github.com/line/iavl v0.14.4-0.20210205104021-924a85cfa292/go.mod h1:eG6hI8RbMxL1nR+nJBykXD//gKjUpKCAT2tvi9V93sA=
+github.com/line/iavl v0.14.4-0.20210208092835-82115f00616e h1:xQixQJ3eBRCpr+6SSaDtvRJP1GqAT43n8bF6VgWfSOM=
+github.com/line/iavl v0.14.4-0.20210208092835-82115f00616e/go.mod h1:eG6hI8RbMxL1nR+nJBykXD//gKjUpKCAT2tvi9V93sA=
 github.com/line/tendermint v0.33.9-0.1.0-rc2 h1:G2plZ9h1xitXP+nYtGlR8SRrHbBuim8KeYxP3JDBoGY=
 github.com/line/tendermint v0.33.9-0.1.0-rc2/go.mod h1:0yUs9eIuuDq07nQql9BmI30FtYGcEC60Tu5JzB5IezM=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=

server/config/config.go

@@ -37,6 +37,11 @@ type BaseConfig struct {
  // Note: Commitment of state will be attempted on the corresponding block.
  HaltTime uint64 `mapstructure:"halt-time"`
 
+ // IAVLCacheSize is the maximum number of entries in the all iavl node caches.
+ IAVLCacheSize int `mapstructure:"iavl-cache-size"`
+
+ // InterBlockCacheSize is the maximum number of entries in the inter-block cache.
+ InterBlockCacheSize int `mapstructure:"inter-block-cache-size"`
  // InterBlockCache enables inter-block caching.
  InterBlockCache bool `mapstructure:"inter-block-cache"`
 
@@ -81,12 +86,14 @@ func (c *Config) GetMinGasPrices() sdk.DecCoins {
 func DefaultConfig() *Config {
  return &Config{
  BaseConfig: BaseConfig{
- MinGasPrices: defaultMinGasPrices,
- InterBlockCache: true,
- Pruning: storetypes.PruningOptionDefault,
- PruningKeepRecent: "0",
- PruningKeepEvery: "0",
- PruningInterval: "0",
+ MinGasPrices: defaultMinGasPrices,
+ InterBlockCache: true,
+ InterBlockCacheSize: 1000,
+ IAVLCacheSize: 10000,
+ Pruning: storetypes.PruningOptionDefault,
+ PruningKeepRecent: "0",
+ PruningKeepEvery: "0",
+ PruningInterval: "0",
  },
  }
 }

server/config/toml.go

@@ -44,6 +44,11 @@ halt-time = {{ .BaseConfig.HaltTime }}
 
 # InterBlockCache enables inter-block caching.
 inter-block-cache = {{ .BaseConfig.InterBlockCache }}
+# InterBlockCacheSize is the maximum number of entries in the inter-block cache.
+inter-block-cache-size = {{ .BaseConfig.InterBlockCache }}
+
+# IAVLCacheSize is the maximum number of entries in the all iavl node caches
+iavl-cache-size = {{ .BaseConfig.InterBlockCache }}
 
 # When true, Prometheus metrics are served under /metrics on prometheus_listen_addr in config.toml.
 # It works when tendermint's prometheus option (config.toml) is set to true.

server/mock/store.go

@@ -100,6 +100,10 @@ func (ms multiStore) SetInterBlockCache(_ sdk.MultiStorePersistentCache) {
  panic("not implemented")
 }
 
+func (ms multiStore) SetIAVLCacheSize(cacheSize int) {
+ panic("not implemented")
+}
+
 func (ms multiStore) SetMetrics(metrics *store.Metrics, iavlMetricsProvider iavl.MetricsProvider) {
  panic("not implemented")
 }

server/start.go

@@ -22,17 +22,19 @@ import (
 
 // Tendermint full-node start flags
 const (
- flagWithTendermint = "with-tendermint"
- flagAddress = "address"
- flagTraceStore = "trace-store"
- flagCPUProfile = "cpu-profile"
- FlagMinGasPrices = "minimum-gas-prices"
- FlagHaltHeight = "halt-height"
- FlagHaltTime = "halt-time"
- FlagInterBlockCache = "inter-block-cache"
- FlagUnsafeSkipUpgrades = "unsafe-skip-upgrades"
- FlagTrace = "trace"
- FlagPrometheus = "prometheus"
+ flagWithTendermint = "with-tendermint"
+ flagAddress = "address"
+ flagTraceStore = "trace-store"
+ flagCPUProfile = "cpu-profile"
+ FlagMinGasPrices = "minimum-gas-prices"
+ FlagHaltHeight = "halt-height"
+ FlagHaltTime = "halt-time"
+ FlagInterBlockCache = "inter-block-cache"
+ FlagInterBlockCacheSize = "inter-block-cache-size"
+ FlagIAVLCacheSize = "iavl-cache-size"
+ FlagUnsafeSkipUpgrades = "unsafe-skip-upgrades"
+ FlagTrace = "trace"
+ FlagPrometheus = "prometheus"
 
  FlagPruning = "pruning"
  FlagPruningKeepRecent = "pruning-keep-recent"
@@ -98,13 +100,16 @@ which accepts a path for the resulting pprof file.
  cmd.Flags().Uint64(FlagHaltHeight, 0, "Block height at which to gracefully halt the chain and shutdown the node")
  cmd.Flags().Uint64(FlagHaltTime, 0, "Minimum block time (in Unix seconds) at which to gracefully halt the chain and shutdown the node")
  cmd.Flags().Bool(FlagInterBlockCache, true, "Enable inter-block caching")
+ cmd.Flags().Int(FlagInterBlockCacheSize, 1000, "The maximum number of entries in the inter-block cache")
+ cmd.Flags().Int(FlagIAVLCacheSize, 10000, "The maximum number of entries in the all iavl node caches")
  cmd.Flags().String(flagCPUProfile, "", "Enable CPU profiling and write to the provided file")
 
  cmd.Flags().String(FlagPruning, storetypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)")
  cmd.Flags().Uint64(FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')")
  cmd.Flags().Uint64(FlagPruningKeepEvery, 0, "Offset heights to keep on disk after 'keep-every' (ignored if pruning is not 'custom')")
  cmd.Flags().Uint64(FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')")
  viper.BindPFlag(FlagTrace, cmd.Flags().Lookup(FlagTrace))
+ cmd.Flags().Bool(FlagPrometheus, false, "Enable prometheus metrics")
  viper.BindPFlag(FlagPruning, cmd.Flags().Lookup(FlagPruning))
  viper.BindPFlag(FlagPruningKeepRecent, cmd.Flags().Lookup(FlagPruningKeepRecent))
  viper.BindPFlag(FlagPruningKeepEvery, cmd.Flags().Lookup(FlagPruningKeepEvery))

simapp/sim_bench_test.go

@@ -5,6 +5,7 @@ import (
  "os"
  "testing"
 
+ "github.com/cosmos/cosmos-sdk/baseapp"
  abci "github.com/tendermint/tendermint/abci/types"
 
  "github.com/cosmos/cosmos-sdk/x/simulation"
@@ -26,7 +27,8 @@ func BenchmarkFullAppSimulation(b *testing.B) {
  }
  }()
 
- app := NewSimApp(logger, db, nil, true, map[int64]bool{}, FlagPeriodValue, interBlockCacheOpt())
+ app := NewSimApp(logger, db, nil, true, map[int64]bool{}, FlagPeriodValue, interBlockCacheOpt(1000),
+ baseapp.SetIAVLCacheSize(10000))
 
  // run randomized simulation
  _, simParams, simErr := simulation.SimulateFromSeed(
@@ -65,7 +67,8 @@ func BenchmarkInvariants(b *testing.B) {
  }
  }()
 
- app := NewSimApp(logger, db, nil, true, map[int64]bool{}, FlagPeriodValue, interBlockCacheOpt())
+ app := NewSimApp(logger, db, nil, true, map[int64]bool{}, FlagPeriodValue, interBlockCacheOpt(1000),
+ baseapp.SetIAVLCacheSize(10000))
 
  // run randomized simulation
  _, simParams, simErr := simulation.SimulateFromSeed(

simapp/sim_test.go

@@ -47,8 +47,8 @@ func fauxMerkleModeOpt(bapp *baseapp.BaseApp) {
 
 // interBlockCacheOpt returns a BaseApp option function that sets the persistent
 // inter-block write-through cache.
-func interBlockCacheOpt() func(*baseapp.BaseApp) {
- return baseapp.SetInterBlockCache(store.NewCommitKVStoreCacheManager(cache.NopMetricsProvider()))
+func interBlockCacheOpt(cacheSize int) func(*baseapp.BaseApp) {
+ return baseapp.SetInterBlockCache(store.NewCommitKVStoreCacheManager(cacheSize, cache.NopMetricsProvider()))
 }
 
 func TestFullAppSimulation(t *testing.T) {
@@ -266,7 +266,8 @@ func TestAppStateDeterminism(t *testing.T) {
 
  db := dbm.NewMemDB()
 
- app := NewSimApp(logger, db, nil, true, map[int64]bool{}, FlagPeriodValue, interBlockCacheOpt())
+ app := NewSimApp(logger, db, nil, true, map[int64]bool{}, FlagPeriodValue, interBlockCacheOpt(1000),
+ baseapp.SetIAVLCacheSize(10000))
 
  fmt.Printf(
  "running non-determinism simulation; seed %d: %d/%d, attempt: %d/%d\n",

store/cache/cache.go

@@ -12,10 +12,6 @@ import (
 var (
  _ types.CommitKVStore = (*CommitKVStoreCache)(nil)
  _ types.MultiStorePersistentCache = (*CommitKVStoreCacheManager)(nil)
-
- // DefaultCommitKVStoreCacheSize defines the persistent ARC cache size for a
- // CommitKVStoreCache.
- DefaultCommitKVStoreCacheSize uint = 1000
 )
 
 type (
@@ -36,17 +32,18 @@ type (
  // in an inter-block (persistent) manner and typically provided by a
  // CommitMultiStore.
  CommitKVStoreCacheManager struct {
- cacheSize uint
+ cacheSize int
  caches map[string]types.CommitKVStore
  metricsProvider func(storeName string) *Metrics
  }
 )
 
-func NewCommitKVStoreCache(store types.CommitKVStore, size uint, metrics *Metrics) *CommitKVStoreCache {
- cache, err := lru.NewARC(int(size))
+func NewCommitKVStoreCache(store types.CommitKVStore, size int, metrics *Metrics) *CommitKVStoreCache {
+ cache, err := lru.NewARC(size)
  if err != nil {
  panic(fmt.Errorf("failed to create KVStore cache: %s", err))
  }
+ metrics.InterBlockCacheSize.Set(float64(size))
 
  return &CommitKVStoreCache{
  CommitKVStore: store,
@@ -55,7 +52,7 @@ func NewCommitKVStoreCache(store types.CommitKVStore, size uint, metrics *Metric
  }
 }
 
-func NewCommitKVStoreCacheManager(size uint, metricsProvider MetricsProvider) *CommitKVStoreCacheManager {
+func NewCommitKVStoreCacheManager(size int, metricsProvider MetricsProvider) *CommitKVStoreCacheManager {
  return &CommitKVStoreCacheManager{
  cacheSize: size,
  caches: make(map[string]types.CommitKVStore),

store/cache/cache_test.go

@@ -1,10 +1,9 @@
-package cache_test
+package cache
 
 import (
  "fmt"
  "testing"
 
- "github.com/cosmos/cosmos-sdk/store/cache"
  iavlstore "github.com/cosmos/cosmos-sdk/store/iavl"
  "github.com/cosmos/cosmos-sdk/store/types"
 
@@ -15,7 +14,10 @@ import (
 
 func TestGetOrSetStoreCache(t *testing.T) {
  db := dbm.NewMemDB()
- mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize, cache.NopMetricsProvider())
+
+ cacheSize := 1000
+ mngr := NewCommitKVStoreCacheManager(cacheSize, NopMetricsProvider())
+ require.Equal(t, cacheSize, mngr.cacheSize)
 
  sKey := types.NewKVStoreKey("test")
  tree, err := iavl.NewMutableTree(db, 100)
@@ -29,7 +31,9 @@ func TestGetOrSetStoreCache(t *testing.T) {
 
 func TestUnwrap(t *testing.T) {
  db := dbm.NewMemDB()
- mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize, cache.NopMetricsProvider())
+ cacheSize := 1000
+ mngr := NewCommitKVStoreCacheManager(cacheSize, NopMetricsProvider())
+ require.Equal(t, cacheSize, mngr.cacheSize)
 
  sKey := types.NewKVStoreKey("test")
  tree, err := iavl.NewMutableTree(db, 100)
@@ -43,15 +47,17 @@ func TestUnwrap(t *testing.T) {
 
 func TestStoreCache(t *testing.T) {
  db := dbm.NewMemDB()
- mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize, cache.NopMetricsProvider())
+ cacheSize := 1000
+ mngr := NewCommitKVStoreCacheManager(cacheSize, NopMetricsProvider())
+ require.Equal(t, cacheSize, mngr.cacheSize)
 
  sKey := types.NewKVStoreKey("test")
  tree, err := iavl.NewMutableTree(db, 100)
  require.NoError(t, err)
  store := iavlstore.UnsafeNewStore(tree)
  kvStore := mngr.GetStoreCache(sKey, store)
 
- for i := uint(0); i < cache.DefaultCommitKVStoreCacheSize*2; i++ {
+ for i := 0; i < cacheSize*2; i++ {
  key := []byte(fmt.Sprintf("key_%d", i))
  value := []byte(fmt.Sprintf("value_%d", i))
 

store/cache/metrics.go

@@ -15,6 +15,7 @@ const (
 
 // Metrics contains metrics exposed by this package.
 type Metrics struct {
+ InterBlockCacheSize metrics.Gauge
  InterBlockCacheHits metrics.Counter
  InterBlockCacheMisses metrics.Counter
 }
@@ -28,6 +29,12 @@ func PrometheusMetrics(namespace string, storeName string, labelsAndValues ...st
  labels = append(labels, labelsAndValues[i])
  }
  return &Metrics{
+ InterBlockCacheSize: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+ Namespace: namespace,
+ Subsystem: MetricsSubsystem,
+ Name: storeName + "_inter_block_cache_size",
+ Help: "The maximum number of entries in the inter-block cache",
+ }, labels).With(labelsAndValues...),
  InterBlockCacheHits: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
  Namespace: namespace,
  Subsystem: MetricsSubsystem,
@@ -46,6 +53,7 @@ func PrometheusMetrics(namespace string, storeName string, labelsAndValues ...st
 // NopMetrics returns no-op Metrics.
 func NopMetrics() *Metrics {
  return &Metrics{
+ InterBlockCacheSize: discard.NewGauge(),
  InterBlockCacheHits: discard.NewCounter(),
  InterBlockCacheMisses: discard.NewCounter(),
  }

store/iavl/store.go

@@ -16,10 +16,6 @@ import (
  sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 )
 
-const (
- defaultIAVLCacheSize = 10000
-)
-
 var (
  _ types.KVStore = (*Store)(nil)
  _ types.CommitStore = (*Store)(nil)
@@ -35,16 +31,17 @@ type Store struct {
 // LoadStore returns an IAVL Store as a CommitKVStore. Internally, it will load the
 // store's version (id) from the provided DB. An error is returned if the version
 // fails to load.
-func LoadStore(db dbm.DB, id types.CommitID, lazyLoading bool, metric *iavl.Metrics) (types.CommitKVStore, error) {
- return LoadStoreWithInitialVersion(db, id, lazyLoading, 0, metric)
+func LoadStore(db dbm.DB, id types.CommitID, cacheSize int, lazyLoading bool, metric *iavl.Metrics) (types.CommitKVStore, error) {
+ return LoadStoreWithInitialVersion(db, id, cacheSize, lazyLoading, 0, metric)
 }
 
 // LoadStore returns an IAVL Store as a CommitKVStore setting its initialVersion
 // to the one given. Internally, it will load the store's version (id) from the
 // provided DB. An error is returned if the version fails to load.
-func LoadStoreWithInitialVersion(db dbm.DB, id types.CommitID, lazyLoading bool, initialVersion uint64,
+func LoadStoreWithInitialVersion(db dbm.DB, id types.CommitID, cacheSize int, lazyLoading bool, initialVersion uint64,
  metric *iavl.Metrics) (types.CommitKVStore, error) {
- tree, err := iavl.NewMutableTreeWithOpts(db, defaultIAVLCacheSize, &iavl.Options{InitialVersion: initialVersion, Metrics: metric})
+ tree, err := iavl.NewMutableTreeWithOpts(db, cacheSize, &iavl.Options{InitialVersion: initialVersion,
+ Metrics: metric})
  if err != nil {
  return nil, err
  }

store/iavl/store_test.go

@@ -89,18 +89,19 @@ func TestLoadStore(t *testing.T) {
  require.NoError(t, err)
  require.Equal(t, string(hcStore.Get([]byte("hello"))), "ciao")
 
+ cacheSize := 10000
  // Querying a new store at some previous non-pruned height H
- newHStore, err := LoadStore(db, cIDH, false, iavl.NopMetrics())
+ newHStore, err := LoadStore(db, cIDH, cacheSize, false, iavl.NopMetrics())
  require.NoError(t, err)
  require.Equal(t, string(newHStore.Get([]byte("hello"))), "hallo")
 
  // Querying a new store at some previous pruned height Hp
- newHpStore, err := LoadStore(db, cIDHp, false, iavl.NopMetrics())
+ newHpStore, err := LoadStore(db, cIDHp, cacheSize, false, iavl.NopMetrics())
  require.NoError(t, err)
  require.Equal(t, string(newHpStore.Get([]byte("hello"))), "hola")
 
  // Querying a new store at current height H
- newHcStore, err := LoadStore(db, cIDHc, false, iavl.NopMetrics())
+ newHcStore, err := LoadStore(db, cIDHc, cacheSize, false, iavl.NopMetrics())
  require.NoError(t, err)
  require.Equal(t, string(newHcStore.Get([]byte("hello"))), "ciao")
 }

store/rootmulti/proof_test.go

@@ -15,7 +15,7 @@ import (
 func TestVerifyIAVLStoreQueryProof(t *testing.T) {
  // Create main tree for testing.
  db := dbm.NewMemDB()
- iStore, err := iavl.LoadStore(db, types.CommitID{}, false, tiavl.NopMetrics())
+ iStore, err := iavl.LoadStore(db, types.CommitID{}, 10000, false, tiavl.NopMetrics())
  store := iStore.(*iavl.Store)
  require.Nil(t, err)
  store.Set([]byte("MYKEY"), []byte("MYVALUE"))