R4R: Switch gov proposal-queues to use iterators

PENDING.md

@@ -40,6 +40,7 @@ IMPROVEMENTS
 * Gaia CLI  (`gaiacli`)
 
 * Gaia
+ - #2637 [x/gov] Switched inactive and active proposal queues to an iterator based queue
 
 * SDK
  - \#2573 [x/distribution] add accum invariance

client/lcd/lcd_test.go

@@ -628,8 +628,8 @@ func TestSubmitProposal(t *testing.T) {
 	require.Equal(t, uint32(0), resultTx.CheckTx.Code)
 	require.Equal(t, uint32(0), resultTx.DeliverTx.Code)
 
-	var proposalID int64
-	cdc.UnmarshalBinaryBare(resultTx.DeliverTx.GetData(), &proposalID)
+	var proposalID uint64
+	cdc.MustUnmarshalBinaryLengthPrefixed(resultTx.DeliverTx.GetData(), &proposalID)
 
 	// query proposal
 	proposal := getProposal(t, port, proposalID)
@@ -650,8 +650,8 @@ func TestDeposit(t *testing.T) {
 	require.Equal(t, uint32(0), resultTx.CheckTx.Code)
 	require.Equal(t, uint32(0), resultTx.DeliverTx.Code)
 
-	var proposalID int64
-	cdc.UnmarshalBinaryBare(resultTx.DeliverTx.GetData(), &proposalID)
+	var proposalID uint64
+	cdc.MustUnmarshalBinaryLengthPrefixed(resultTx.DeliverTx.GetData(), &proposalID)
 
 	// query proposal
 	proposal := getProposal(t, port, proposalID)
@@ -684,8 +684,8 @@ func TestVote(t *testing.T) {
 	require.Equal(t, uint32(0), resultTx.CheckTx.Code)
 	require.Equal(t, uint32(0), resultTx.DeliverTx.Code)
 
-	var proposalID int64
-	cdc.UnmarshalBinaryBare(resultTx.DeliverTx.GetData(), &proposalID)
+	var proposalID uint64
+	cdc.MustUnmarshalBinaryLengthPrefixed(resultTx.DeliverTx.GetData(), &proposalID)
 
 	// query proposal
 	proposal := getProposal(t, port, proposalID)
@@ -732,18 +732,18 @@ func TestProposalsQuery(t *testing.T) {
 
 	// Addr1 proposes (and deposits) proposals #1 and #2
 	resultTx := doSubmitProposal(t, port, seeds[0], names[0], passwords[0], addrs[0], 5)
-	var proposalID1 int64
-	cdc.UnmarshalBinaryBare(resultTx.DeliverTx.GetData(), &proposalID1)
+	var proposalID1 uint64
+	cdc.MustUnmarshalBinaryLengthPrefixed(resultTx.DeliverTx.GetData(), &proposalID1)
 	tests.WaitForHeight(resultTx.Height+1, port)
 	resultTx = doSubmitProposal(t, port, seeds[0], names[0], passwords[0], addrs[0], 5)
-	var proposalID2 int64
-	cdc.UnmarshalBinaryBare(resultTx.DeliverTx.GetData(), &proposalID2)
+	var proposalID2 uint64
+	cdc.MustUnmarshalBinaryLengthPrefixed(resultTx.DeliverTx.GetData(), &proposalID2)
 	tests.WaitForHeight(resultTx.Height+1, port)
 
 	// Addr2 proposes (and deposits) proposals #3
 	resultTx = doSubmitProposal(t, port, seeds[1], names[1], passwords[1], addrs[1], 5)
-	var proposalID3 int64
-	cdc.UnmarshalBinaryBare(resultTx.DeliverTx.GetData(), &proposalID3)
+	var proposalID3 uint64
+	cdc.MustUnmarshalBinaryLengthPrefixed(resultTx.DeliverTx.GetData(), &proposalID3)
 	tests.WaitForHeight(resultTx.Height+1, port)
 
 	// Addr2 deposits on proposals #2 & #3
@@ -1230,7 +1230,7 @@ func getValidatorRedelegations(t *testing.T, port string, validatorAddr sdk.ValA
 
 // ============= Governance Module ================
 
-func getProposal(t *testing.T, port string, proposalID int64) gov.Proposal {
+func getProposal(t *testing.T, port string, proposalID uint64) gov.Proposal {
 	res, body := Request(t, port, "GET", fmt.Sprintf("/gov/proposals/%d", proposalID), nil)
 	require.Equal(t, http.StatusOK, res.StatusCode, body)
 	var proposal gov.Proposal
@@ -1239,7 +1239,7 @@ func getProposal(t *testing.T, port string, proposalID int64) gov.Proposal {
 	return proposal
 }
 
-func getDeposits(t *testing.T, port string, proposalID int64) []gov.Deposit {
+func getDeposits(t *testing.T, port string, proposalID uint64) []gov.Deposit {
 	res, body := Request(t, port, "GET", fmt.Sprintf("/gov/proposals/%d/deposits", proposalID), nil)
 	require.Equal(t, http.StatusOK, res.StatusCode, body)
 	var deposits []gov.Deposit
@@ -1248,7 +1248,7 @@ func getDeposits(t *testing.T, port string, proposalID int64) []gov.Deposit {
 	return deposits
 }
 
-func getDeposit(t *testing.T, port string, proposalID int64, depositerAddr sdk.AccAddress) gov.Deposit {
+func getDeposit(t *testing.T, port string, proposalID uint64, depositerAddr sdk.AccAddress) gov.Deposit {
 	res, body := Request(t, port, "GET", fmt.Sprintf("/gov/proposals/%d/deposits/%s", proposalID, depositerAddr), nil)
 	require.Equal(t, http.StatusOK, res.StatusCode, body)
 	var deposit gov.Deposit
@@ -1257,7 +1257,7 @@ func getDeposit(t *testing.T, port string, proposalID int64, depositerAddr sdk.A
 	return deposit
 }
 
-func getVote(t *testing.T, port string, proposalID int64, voterAddr sdk.AccAddress) gov.Vote {
+func getVote(t *testing.T, port string, proposalID uint64, voterAddr sdk.AccAddress) gov.Vote {
 	res, body := Request(t, port, "GET", fmt.Sprintf("/gov/proposals/%d/votes/%s", proposalID, voterAddr), nil)
 	require.Equal(t, http.StatusOK, res.StatusCode, body)
 	var vote gov.Vote
@@ -1266,7 +1266,7 @@ func getVote(t *testing.T, port string, proposalID int64, voterAddr sdk.AccAddre
 	return vote
 }
 
-func getVotes(t *testing.T, port string, proposalID int64) []gov.Vote {
+func getVotes(t *testing.T, port string, proposalID uint64) []gov.Vote {
 	res, body := Request(t, port, "GET", fmt.Sprintf("/gov/proposals/%d/votes", proposalID), nil)
 	require.Equal(t, http.StatusOK, res.StatusCode, body)
 	var votes []gov.Vote
@@ -1358,7 +1358,7 @@ func doSubmitProposal(t *testing.T, port, seed, name, password string, proposerA
 	return results
 }
 
-func doDeposit(t *testing.T, port, seed, name, password string, proposerAddr sdk.AccAddress, proposalID int64, amount int64) (resultTx ctypes.ResultBroadcastTxCommit) {
+func doDeposit(t *testing.T, port, seed, name, password string, proposerAddr sdk.AccAddress, proposalID uint64, amount int64) (resultTx ctypes.ResultBroadcastTxCommit) {
 
 	acc := getAccount(t, port, proposerAddr)
 	accnum := acc.GetAccountNumber()
@@ -1388,7 +1388,7 @@ func doDeposit(t *testing.T, port, seed, name, password string, proposerAddr sdk
 	return results
 }
 
-func doVote(t *testing.T, port, seed, name, password string, proposerAddr sdk.AccAddress, proposalID int64) (resultTx ctypes.ResultBroadcastTxCommit) {
+func doVote(t *testing.T, port, seed, name, password string, proposerAddr sdk.AccAddress, proposalID uint64) (resultTx ctypes.ResultBroadcastTxCommit) {
 	// get the account to get the sequence
 	acc := getAccount(t, port, proposerAddr)
 	accnum := acc.GetAccountNumber()

client/utils/rest.go

@@ -65,6 +65,20 @@ func ParseInt64OrReturnBadRequest(w http.ResponseWriter, s string) (n int64, ok
 	return n, true
 }
 
+// ParseUint64OrReturnBadRequest converts s to a uint64 value.
+func ParseUint64OrReturnBadRequest(w http.ResponseWriter, s string) (n uint64, ok bool) {
+	var err error
+
+	n, err = strconv.ParseUint(s, 10, 64)
+	if err != nil {
+		err := fmt.Errorf("'%s' is not a valid uint64", s)
+		WriteErrorResponse(w, http.StatusBadRequest, err.Error())
+		return n, false
+	}
+
+	return n, true
+}
+
 // ParseFloat64OrReturnBadRequest converts s to a float64 value. It returns a
 // default value, defaultIfEmpty, if the string is empty.
 func ParseFloat64OrReturnBadRequest(w http.ResponseWriter, s string, defaultIfEmpty float64) (n float64, ok bool) {

cmd/gaia/app/sim_test.go

@@ -70,7 +70,7 @@ func appStateFn(r *rand.Rand, accs []simulation.Account) json.RawMessage {
 
 	// Random genesis states
 	govGenesis := gov.GenesisState{
-		StartingProposalID: int64(r.Intn(100)),
+		StartingProposalID: uint64(r.Intn(100)),
 		DepositProcedure: gov.DepositProcedure{
 			MinDeposit:       sdk.Coins{sdk.NewInt64Coin("steak", int64(r.Intn(1e3)))},
 			MaxDepositPeriod: time.Duration(r.Intn(2*172800)) * time.Second,

cmd/gaia/cli_test/cli_test.go

@@ -5,13 +5,14 @@ package clitest
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/tendermint/tendermint/types"
 	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
 	"testing"
 
+	"github.com/tendermint/tendermint/types"
+
 	"github.com/stretchr/testify/require"
 
 	abci "github.com/tendermint/tendermint/abci/types"
@@ -348,7 +349,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) {
 	require.Equal(t, int64(45), fooAcc.GetCoins().AmountOf("steak").Int64())
 
 	proposal1 := executeGetProposal(t, fmt.Sprintf("gaiacli query proposal --proposal-id=1 --output=json %v", flags))
-	require.Equal(t, int64(1), proposal1.GetProposalID())
+	require.Equal(t, uint64(1), proposal1.GetProposalID())
 	require.Equal(t, gov.StatusDepositPeriod, proposal1.GetStatus())
 
 	proposalsQuery, _ = tests.ExecuteT(t, fmt.Sprintf("gaiacli query proposals %v", flags), "")
@@ -391,7 +392,7 @@ func TestGaiaCLISubmitProposal(t *testing.T) {
 	fooAcc = executeGetAccount(t, fmt.Sprintf("gaiacli query account %s %v", fooAddr, flags))
 	require.Equal(t, int64(35), fooAcc.GetCoins().AmountOf("steak").Int64())
 	proposal1 = executeGetProposal(t, fmt.Sprintf("gaiacli query proposal --proposal-id=1 --output=json %v", flags))
-	require.Equal(t, int64(1), proposal1.GetProposalID())
+	require.Equal(t, uint64(1), proposal1.GetProposalID())
 	require.Equal(t, gov.StatusVotingPeriod, proposal1.GetStatus())
 
 	voteStr := fmt.Sprintf("gaiacli tx vote %v", flags)

docs/spec/governance/state.md

@@ -96,11 +96,13 @@ type Proposal struct {
   Type                  ProposalType        //  Type of proposal. Initial set {PlainTextProposal, SoftwareUpgradeProposal}
   TotalDeposit          sdk.Coins           //  Current deposit on this proposal. Initial value is set at InitialDeposit
   Deposits              []Deposit           //  List of deposits on the proposal
-  SubmitTime           time.Time               //  Time of the block where TxGovSubmitProposal was included
-  Submitter             sdk.Address      //  Address of the submitter
+  SubmitTime            time.Time           //  Time of the block where TxGovSubmitProposal was included
+  DepositEndTime        time.Time           //  Time that the DepositPeriod of a proposal would expire
+  Submitter             sdk.Address         //  Address of the submitter
 
   VotingStartTime      time.Time               //  Time of the block where MinDeposit was reached. time.Time{} if MinDeposit is not reached
-  CurrentStatus         ProposalStatus      //  Current status of the proposal
+  VotingEndTime        time.Time               //  Time of the block that the VotingPeriod for a proposal will end.
+  CurrentStatus         ProposalStatus         //  Current status of the proposal
 
   YesVotes              sdk.Dec
   NoVotes               sdk.Dec
@@ -134,47 +136,29 @@ For pseudocode purposes, here are the two function we will use to read or write
 
 **Store:**
 * `ProposalProcessingQueue`: A queue `queue[proposalID]` containing all the 
-  `ProposalIDs` of proposals that reached `MinDeposit`. Each round, the oldest 
-  element of `ProposalProcessingQueue` is checked during `BeginBlock` to see if
-  `CurrentTime == VotingStartTime + activeProcedure.VotingPeriod`. If it is, 
-  then the application tallies the votes, compute the votes of each validator and checks if every validator in the valdiator set have voted
-  and, if not, applies `GovernancePenalty`. If the proposal is accepted, deposits are refunded.
-  After that proposal is ejected from `ProposalProcessingQueue` and the next element of the queue is evaluated. 
+  `ProposalIDs` of proposals that reached `MinDeposit`. Each `EndBlock`, all the proposals
+  that have reached the end of their voting period are processed.
+  To process a finished proposal, the application tallies the votes, compute the votes of
+  each validator and checks if every validator in the valdiator set have voted.
+  If the proposal is accepted, deposits are refunded.
 
 And the pseudocode for the `ProposalProcessingQueue`:
 
 ```go
   in EndBlock do 
-
-    checkProposal()  // First call of the recursive function 
-
-
-  // Recursive function. First call in BeginBlock
-  func checkProposal()  
-    proposalID = ProposalProcessingQueue.Peek()
-    if (proposalID == nil)
-      return
 
-    proposal = load(Governance, <proposalID|'proposal'>) // proposal is a const key
     votingProcedure = load(GlobalParams, 'VotingProcedure')
+
+    for finishedProposalID in GetAllFinishedProposalIDs(block.Time)
+      proposal = load(Governance, <proposalID|'proposal'>) // proposal is a const key
 
-    if (CurrentTime == proposal.VotingStartTime + votingProcedure.VotingPeriod && proposal.CurrentStatus == ProposalStatusActive)
-
-    // End of voting period, tally
-
-      ProposalProcessingQueue.pop()
-      validators = 
-
-
-      Keeper.getAllValidators()
+      validators = Keeper.getAllValidators()
       tmpValMap := map(sdk.Address)ValidatorGovInfo
 
       // Initiate mapping at 0. Validators that remain at 0 at the end of tally will be punished
       for each validator in validators
         tmpValMap(validator).Minus = 0
 
-
-
       // Tally
       voterIterator = rangeQuery(Governance, <proposalID|'addresses'>) //return all the addresses that voted on the proposal
       for each (voterAddress, vote) in voterIterator
@@ -212,5 +196,4 @@ And the pseudocode for the `ProposalProcessingQueue`:
         proposal.CurrentStatus = ProposalStatusRejected
 
       store(Governance, <proposalID|'proposal'>, proposal)
-      checkProposal()        
 ```

types/utils.go

@@ -1,6 +1,7 @@
 package types
 
 import (
+	"encoding/binary"
 	"encoding/json"
 	"time"
 
@@ -36,6 +37,13 @@ func MustSortJSON(toSortJSON []byte) []byte {
 	return js
 }
 
+// Uint64ToBigEndian - marshals uint64 to a bigendian byte slice so it can be sorted
+func Uint64ToBigEndian(i uint64) []byte {
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, i)
+	return b
+}
+
 // Slight modification of the RFC3339Nano but it right pads all zeros and drops the time zone info
 const SortableTimeFormat = "2006-01-02T15:04:05.000000000"
 

x/gov/client/cli/tx.go

@@ -170,7 +170,7 @@ func GetCmdDeposit(cdc *codec.Codec) *cobra.Command {
 				return err
 			}
 
-			proposalID := viper.GetInt64(flagProposalID)
+			proposalID := uint64(viper.GetInt64(flagProposalID))
 
 			amount, err := sdk.ParseCoins(viper.GetString(flagDeposit))
 			if err != nil {
@@ -215,7 +215,7 @@ func GetCmdVote(cdc *codec.Codec) *cobra.Command {
 				return err
 			}
 
-			proposalID := viper.GetInt64(flagProposalID)
+			proposalID := uint64(viper.GetInt64(flagProposalID))
 			option := viper.GetString(flagOption)
 
 			byteVoteOption, err := gov.VoteOptionFromString(client.NormalizeVoteOption(option))
@@ -256,7 +256,7 @@ func GetCmdQueryProposal(queryRoute string, cdc *codec.Codec) *cobra.Command {
 		Short: "Query details of a single proposal",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cliCtx := context.NewCLIContext().WithCodec(cdc)
-			proposalID := viper.GetInt64(flagProposalID)
+			proposalID := uint64(viper.GetInt64(flagProposalID))
 
 			params := gov.QueryProposalParams{
 				ProposalID: proposalID,
@@ -291,7 +291,7 @@ func GetCmdQueryProposals(queryRoute string, cdc *codec.Codec) *cobra.Command {
 			bechDepositerAddr := viper.GetString(flagDepositer)
 			bechVoterAddr := viper.GetString(flagVoter)
 			strProposalStatus := viper.GetString(flagStatus)
-			latestProposalsIDs := viper.GetInt64(flagLatestProposalIDs)
+			latestProposalsIDs := uint64(viper.GetInt64(flagLatestProposalIDs))
 
 			params := gov.QueryProposalsParams{
 				NumLatestProposals: latestProposalsIDs,
@@ -368,7 +368,7 @@ func GetCmdQueryVote(queryRoute string, cdc *codec.Codec) *cobra.Command {
 		Short: "Query details of a single vote",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cliCtx := context.NewCLIContext().WithCodec(cdc)
-			proposalID := viper.GetInt64(flagProposalID)
+			proposalID := uint64(viper.GetInt64(flagProposalID))
 
 			voterAddr, err := sdk.AccAddressFromBech32(viper.GetString(flagVoter))
 			if err != nil {
@@ -407,7 +407,7 @@ func GetCmdQueryVotes(queryRoute string, cdc *codec.Codec) *cobra.Command {
 		Short: "Query votes on a proposal",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cliCtx := context.NewCLIContext().WithCodec(cdc)
-			proposalID := viper.GetInt64(flagProposalID)
+			proposalID := uint64(viper.GetInt64(flagProposalID))
 
 			params := gov.QueryVotesParams{
 				ProposalID: proposalID,
@@ -440,7 +440,7 @@ func GetCmdQueryDeposit(queryRoute string, cdc *codec.Codec) *cobra.Command {
 		Short: "Query details of a deposit",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cliCtx := context.NewCLIContext().WithCodec(cdc)
-			proposalID := viper.GetInt64(flagProposalID)
+			proposalID := uint64(viper.GetInt64(flagProposalID))
 
 			depositerAddr, err := sdk.AccAddressFromBech32(viper.GetString(flagDepositer))
 			if err != nil {
@@ -479,7 +479,7 @@ func GetCmdQueryDeposits(queryRoute string, cdc *codec.Codec) *cobra.Command {
 		Short: "Query deposits on a proposal",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cliCtx := context.NewCLIContext().WithCodec(cdc)
-			proposalID := viper.GetInt64(flagProposalID)
+			proposalID := uint64(viper.GetInt64(flagProposalID))
 
 			params := gov.QueryDepositsParams{
 				ProposalID: proposalID,
@@ -511,7 +511,7 @@ func GetCmdQueryTally(queryRoute string, cdc *codec.Codec) *cobra.Command {
 		Short: "Get the tally of a proposal vote",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cliCtx := context.NewCLIContext().WithCodec(cdc)
-			proposalID := viper.GetInt64(flagProposalID)
+			proposalID := uint64(viper.GetInt64(flagProposalID))
 
 			params := gov.QueryTallyParams{
 				ProposalID: proposalID,