Merge branch 'develop' into joon/959-pos-init

client/context/helpers.go

@@ -141,13 +141,22 @@ func (ctx CoreContext) SignAndBuild(name, passphrase string, msgs []sdk.Msg, cdc
 	sequence := ctx.Sequence
 	memo := ctx.Memo
 
+	fee := sdk.Coin{}
+	if ctx.Fee != "" {
+		parsedFee, err := sdk.ParseCoin(ctx.Fee)
+		if err != nil {
+			return nil, err
+		}
+		fee = parsedFee
+	}
+
 	signMsg := auth.StdSignMsg{
 		ChainID:       chainID,
 		AccountNumber: accnum,
 		Sequence:      sequence,
 		Msgs:          msgs,
 		Memo:          memo,
-		Fee:           auth.NewStdFee(ctx.Gas, sdk.Coin{}), // TODO run simulate to estimate gas?
+		Fee:           auth.NewStdFee(ctx.Gas, fee), // TODO run simulate to estimate gas?
 	}
 
 	keybase, err := keys.GetKeyBase()

client/context/types.go

@@ -11,6 +11,7 @@ type CoreContext struct {
 	ChainID         string
 	Height          int64
 	Gas             int64
+	Fee             string
 	TrustNode       bool
 	NodeURI         string
 	FromAddressName string
@@ -41,6 +42,12 @@ func (c CoreContext) WithGas(gas int64) CoreContext {
 	return c
 }
 
+// WithFee - return a copy of the context with an updated fee
+func (c CoreContext) WithFee(fee string) CoreContext {
+	c.Fee = fee
+	return c
+}
+
 // WithTrustNode - return a copy of the context with an updated TrustNode flag
 func (c CoreContext) WithTrustNode(trustNode bool) CoreContext {
 	c.TrustNode = trustNode

client/context/viper.go

@@ -31,6 +31,7 @@ func NewCoreContextFromViper() CoreContext {
 		ChainID:         chainID,
 		Height:          viper.GetInt64(client.FlagHeight),
 		Gas:             viper.GetInt64(client.FlagGas),
+		Fee:             viper.GetString(client.FlagFee),
 		TrustNode:       viper.GetBool(client.FlagTrustNode),
 		FromAddressName: viper.GetString(client.FlagName),
 		NodeURI:         nodeURI,

cmd/gaia/app/app.go

@@ -5,10 +5,10 @@ import (
 	"os"
 
 	abci "github.com/tendermint/tendermint/abci/types"
-	tmtypes "github.com/tendermint/tendermint/types"
 	cmn "github.com/tendermint/tendermint/libs/common"
 	dbm "github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
+	tmtypes "github.com/tendermint/tendermint/types"
 
 	bam "github.com/cosmos/cosmos-sdk/baseapp"
 	sdk "github.com/cosmos/cosmos-sdk/types"
@@ -37,12 +37,13 @@ type GaiaApp struct {
 	cdc *wire.Codec
 
 	// keys to access the substores
-	keyMain     *sdk.KVStoreKey
-	keyAccount  *sdk.KVStoreKey
-	keyIBC      *sdk.KVStoreKey
-	keyStake    *sdk.KVStoreKey
-	keySlashing *sdk.KVStoreKey
-	keyGov      *sdk.KVStoreKey
+	keyMain          *sdk.KVStoreKey
+	keyAccount       *sdk.KVStoreKey
+	keyIBC           *sdk.KVStoreKey
+	keyStake         *sdk.KVStoreKey
+	keySlashing      *sdk.KVStoreKey
+	keyGov           *sdk.KVStoreKey
+	keyFeeCollection *sdk.KVStoreKey
 
 	// Manage getting and setting accounts
 	accountMapper       auth.AccountMapper
@@ -59,14 +60,15 @@ func NewGaiaApp(logger log.Logger, db dbm.DB) *GaiaApp {
 
 	// create your application object
 	var app = &GaiaApp{
-		BaseApp:     bam.NewBaseApp(appName, cdc, logger, db),
-		cdc:         cdc,
-		keyMain:     sdk.NewKVStoreKey("main"),
-		keyAccount:  sdk.NewKVStoreKey("acc"),
-		keyIBC:      sdk.NewKVStoreKey("ibc"),
-		keyStake:    sdk.NewKVStoreKey("stake"),
-		keySlashing: sdk.NewKVStoreKey("slashing"),
-		keyGov:      sdk.NewKVStoreKey("gov"),
+		BaseApp:          bam.NewBaseApp(appName, cdc, logger, db),
+		cdc:              cdc,
+		keyMain:          sdk.NewKVStoreKey("main"),
+		keyAccount:       sdk.NewKVStoreKey("acc"),
+		keyIBC:           sdk.NewKVStoreKey("ibc"),
+		keyStake:         sdk.NewKVStoreKey("stake"),
+		keySlashing:      sdk.NewKVStoreKey("slashing"),
+		keyGov:           sdk.NewKVStoreKey("gov"),
+		keyFeeCollection: sdk.NewKVStoreKey("fee"),
 	}
 
 	// define the accountMapper
@@ -82,6 +84,7 @@ func NewGaiaApp(logger log.Logger, db dbm.DB) *GaiaApp {
 	app.stakeKeeper = stake.NewKeeper(app.cdc, app.keyStake, app.coinKeeper, app.RegisterCodespace(stake.DefaultCodespace))
 	app.slashingKeeper = slashing.NewKeeper(app.cdc, app.keySlashing, app.stakeKeeper, app.RegisterCodespace(slashing.DefaultCodespace))
 	app.govKeeper = gov.NewKeeper(app.cdc, app.keyGov, app.coinKeeper, app.stakeKeeper, app.RegisterCodespace(gov.DefaultCodespace))
+	app.feeCollectionKeeper = auth.NewFeeCollectionKeeper(app.cdc, app.keyFeeCollection)
 
 	// register message routes
 	app.Router().
@@ -96,7 +99,7 @@ func NewGaiaApp(logger log.Logger, db dbm.DB) *GaiaApp {
 	app.SetBeginBlocker(app.BeginBlocker)
 	app.SetEndBlocker(app.EndBlocker)
 	app.SetAnteHandler(auth.NewAnteHandler(app.accountMapper, app.feeCollectionKeeper))
-	app.MountStoresIAVL(app.keyMain, app.keyAccount, app.keyIBC, app.keyStake, app.keySlashing, app.keyGov)
+	app.MountStoresIAVL(app.keyMain, app.keyAccount, app.keyIBC, app.keyStake, app.keySlashing, app.keyGov, app.keyFeeCollection)
 	err := app.LoadLatestVersion(app.keyMain)
 	if err != nil {
 		cmn.Exit(err.Error())

store/iavlstore.go

@@ -15,7 +15,8 @@ import (
 
 const (
 	defaultIAVLCacheSize  = 10000
-	defaultIAVLNumHistory = 1<<53 - 1 // DEPRECATED
+	defaultIAVLNumRecent  = 100
+	defaultIAVLStoreEvery = 10000
 )
 
 // load the iavl store
@@ -25,7 +26,7 @@ func LoadIAVLStore(db dbm.DB, id CommitID) (CommitStore, error) {
 	if err != nil {
 		return nil, err
 	}
-	store := newIAVLStore(tree, defaultIAVLNumHistory)
+	store := newIAVLStore(tree, defaultIAVLNumRecent, defaultIAVLStoreEvery)
 	return store, nil
 }
 
@@ -42,17 +43,25 @@ type iavlStore struct {
 	tree *iavl.VersionedTree
 
 	// How many old versions we hold onto.
-	// A value of 0 means keep all history.
-	numHistory int64
+	// A value of 0 means keep no recent states
+	numRecent int64
+
+	// Distance between state-sync waypoint states to be stored
+	// See https://github.com/tendermint/tendermint/issues/828
+	// A value of 1 means store every state
+	// A value of 0 means store no waypoints (node cannot assist in state-sync)
+	// By default this value should be set the same across all nodes,
+	// so that nodes can know the waypoints their peers store
+	// TODO if set to non-default, signal to peers that the node is not suitable as a state sync source
+	storeEvery int64
 }
 
 // CONTRACT: tree should be fully loaded.
-// TODO: use more numHistory's, so the below nolint can be removed
-// nolint: unparam
-func newIAVLStore(tree *iavl.VersionedTree, numHistory int64) *iavlStore {
+func newIAVLStore(tree *iavl.VersionedTree, numRecent int64, storeEvery int64) *iavlStore {
 	st := &iavlStore{
 		tree:       tree,
-		numHistory: numHistory,
+		numRecent:  numRecent,
+		storeEvery: storeEvery,
 	}
 	return st
 }
@@ -67,13 +76,15 @@ func (st *iavlStore) Commit() CommitID {
 		panic(err)
 	}
 
-	// Release an old version of history
-	if st.numHistory > 0 && (st.numHistory < st.tree.Version64()) {
-		toRelease := version - st.numHistory
-		err := st.tree.DeleteVersion(toRelease)
-		if err != nil {
-			// TODO: Handle with #870
-			panic(err)
+	// Release an old version of history, if not a sync waypoint
+	previous := version - 1
+	if st.numRecent < previous {
+		toRelease := previous - st.numRecent
+		if st.storeEvery == 0 || toRelease%st.storeEvery != 0 {
+			err := st.tree.DeleteVersion(toRelease)
+			if err != nil {
+				panic(err)
+			}
 		}
 	}
 
@@ -91,6 +102,11 @@ func (st *iavlStore) LastCommitID() CommitID {
 	}
 }
 
+// VersionExists returns whether or not a given version is stored
+func (st *iavlStore) VersionExists(version int64) bool {
+	return st.tree.VersionExists(version)
+}
+
 // Implements Store.
 func (st *iavlStore) GetStoreType() StoreType {
 	return sdk.StoreTypeIAVL

store/iavlstore_test.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -15,7 +16,8 @@ import (
 
 var (
 	cacheSize        = 100
-	numHistory int64 = 5
+	numRecent  int64 = 5
+	storeEvery int64 = 3
 )
 
 var (
@@ -45,7 +47,7 @@ func newTree(t *testing.T, db dbm.DB) (*iavl.VersionedTree, CommitID) {
 func TestIAVLStoreGetSetHasDelete(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree, _ := newTree(t, db)
-	iavlStore := newIAVLStore(tree, numHistory)
+	iavlStore := newIAVLStore(tree, numRecent, storeEvery)
 
 	key := "hello"
 
@@ -70,7 +72,7 @@ func TestIAVLStoreGetSetHasDelete(t *testing.T) {
 func TestIAVLIterator(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree, _ := newTree(t, db)
-	iavlStore := newIAVLStore(tree, numHistory)
+	iavlStore := newIAVLStore(tree, numRecent, storeEvery)
 	iter := iavlStore.Iterator([]byte("aloha"), []byte("hellz"))
 	expected := []string{"aloha", "hello"}
 	var i int
@@ -143,7 +145,7 @@ func TestIAVLIterator(t *testing.T) {
 func TestIAVLSubspaceIterator(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree, _ := newTree(t, db)
-	iavlStore := newIAVLStore(tree, numHistory)
+	iavlStore := newIAVLStore(tree, numRecent, storeEvery)
 
 	iavlStore.Set([]byte("test1"), []byte("test1"))
 	iavlStore.Set([]byte("test2"), []byte("test2"))
@@ -202,7 +204,7 @@ func TestIAVLSubspaceIterator(t *testing.T) {
 func TestIAVLReverseSubspaceIterator(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree, _ := newTree(t, db)
-	iavlStore := newIAVLStore(tree, numHistory)
+	iavlStore := newIAVLStore(tree, numRecent, storeEvery)
 
 	iavlStore.Set([]byte("test1"), []byte("test1"))
 	iavlStore.Set([]byte("test2"), []byte("test2"))
@@ -258,10 +260,89 @@ func TestIAVLReverseSubspaceIterator(t *testing.T) {
 	require.Equal(t, len(expected), i)
 }
 
+func nextVersion(iavl *iavlStore) {
+	key := []byte(fmt.Sprintf("Key for tree: %d", iavl.LastCommitID().Version))
+	value := []byte(fmt.Sprintf("Value for tree: %d", iavl.LastCommitID().Version))
+	iavl.Set(key, value)
+	iavl.Commit()
+}
+func TestIAVLDefaultPruning(t *testing.T) {
+	//Expected stored / deleted version numbers for:
+	//numRecent = 5, storeEvery = 3
+	var states = []struct {
+		stored  []int64
+		deleted []int64
+	}{
+		{[]int64{}, []int64{}},
+		{[]int64{1}, []int64{}},
+		{[]int64{1, 2}, []int64{}},
+		{[]int64{1, 2, 3}, []int64{}},
+		{[]int64{1, 2, 3, 4}, []int64{}},
+		{[]int64{1, 2, 3, 4, 5}, []int64{}},
+		{[]int64{1, 2, 3, 4, 5, 6}, []int64{}},
+		{[]int64{2, 3, 4, 5, 6, 7}, []int64{1}},
+		{[]int64{3, 4, 5, 6, 7, 8}, []int64{1, 2}},
+		{[]int64{3, 4, 5, 6, 7, 8, 9}, []int64{1, 2}},
+		{[]int64{3, 5, 6, 7, 8, 9, 10}, []int64{1, 2, 4}},
+		{[]int64{3, 6, 7, 8, 9, 10, 11}, []int64{1, 2, 4, 5}},
+		{[]int64{3, 6, 7, 8, 9, 10, 11, 12}, []int64{1, 2, 4, 5}},
+		{[]int64{3, 6, 8, 9, 10, 11, 12, 13}, []int64{1, 2, 4, 5, 7}},
+		{[]int64{3, 6, 9, 10, 11, 12, 13, 14}, []int64{1, 2, 4, 5, 7, 8}},
+		{[]int64{3, 6, 9, 10, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8}},
+	}
+	db := dbm.NewMemDB()
+	tree := iavl.NewVersionedTree(db, cacheSize)
+	iavlStore := newIAVLStore(tree, numRecent, storeEvery)
+	for step, state := range states {
+		for _, ver := range state.stored {
+			require.True(t, iavlStore.VersionExists(ver),
+				"Missing version %d with latest version %d. Should save last %d and every %d",
+				ver, step, numRecent, storeEvery)
+		}
+		for _, ver := range state.deleted {
+			require.False(t, iavlStore.VersionExists(ver),
+				"Unpruned version %d with latest version %d. Should prune all but last %d and every %d",
+				ver, step, numRecent, storeEvery)
+		}
+		nextVersion(iavlStore)
+	}
+}
+func TestIAVLNoPrune(t *testing.T) {
+	db := dbm.NewMemDB()
+	tree := iavl.NewVersionedTree(db, cacheSize)
+	iavlStore := newIAVLStore(tree, numRecent, int64(1))
+	nextVersion(iavlStore)
+	for i := 1; i < 100; i++ {
+		for j := 1; j <= i; j++ {
+			require.True(t, iavlStore.VersionExists(int64(j)),
+				"Missing version %d with latest version %d. Should be storing all versions",
+				j, i)
+		}
+		nextVersion(iavlStore)
+	}
+}
+func TestIAVLPruneEverything(t *testing.T) {
+	db := dbm.NewMemDB()
+	tree := iavl.NewVersionedTree(db, cacheSize)
+	iavlStore := newIAVLStore(tree, int64(0), int64(0))
+	nextVersion(iavlStore)
+	for i := 1; i < 100; i++ {
+		for j := 1; j < i; j++ {
+			require.False(t, iavlStore.VersionExists(int64(j)),
+				"Unpruned version %d with latest version %d. Should prune all old versions",
+				j, i)
+		}
+		require.True(t, iavlStore.VersionExists(int64(i)),
+			"Missing current version on step %d, should not prune current state tree",
+			i)
+		nextVersion(iavlStore)
+	}
+}
+
 func TestIAVLStoreQuery(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree := iavl.NewVersionedTree(db, cacheSize)
-	iavlStore := newIAVLStore(tree, numHistory)
+	iavlStore := newIAVLStore(tree, numRecent, storeEvery)
 
 	k1, v1 := []byte("key1"), []byte("val1")
 	k2, v2 := []byte("key2"), []byte("val2")

store/prefixstore_test.go

@@ -66,7 +66,7 @@ func testPrefixStore(t *testing.T, baseStore KVStore, prefix []byte) {
 func TestIAVLStorePrefix(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree := iavl.NewVersionedTree(db, cacheSize)
-	iavlStore := newIAVLStore(tree, numHistory)
+	iavlStore := newIAVLStore(tree, numRecent, storeEvery)
 
 	testPrefixStore(t, iavlStore, []byte("test"))
 }