Merge remote-tracking branch 'upstream/master' into golangci-lint-clean

.circleci/config.yml

@@ -70,7 +70,7 @@ workflows:
           name: << matrix.platform >>_build
           matrix: &matrix-default
             parameters:
-              platform: ["amd64", "arm64", "mac_amd64"]
+              platform: ["amd64", "arm64"]
           filters: &filters-default
             branches:
               ignore:
@@ -155,7 +155,7 @@ workflows:
           name: << matrix.platform >>_<< matrix.job_type >>_verification
           matrix:
             parameters:
-              platform: ["amd64", "arm64", "mac_amd64"]
+              platform: ["amd64", "arm64"]
               job_type: ["test", "integration", "e2e_expect"]
           requires:
             - << matrix.platform >>_<< matrix.job_type >>

Makefile

@@ -86,7 +86,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \
 UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ ))
 ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... ))
 
-MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/compactcert ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./compactcert ./data/account ./daemon/algod/api/spec/v2
+MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./stateproof ./data/account ./daemon/algod/api/spec/v2
 
 default: build
 

agreement/service_test.go

@@ -1968,7 +1968,7 @@ func TestAgreementSlowPayloadsPostDeadline(t *testing.T) {
 		activityMonitor.waitForQuiet()
 		zeroes = expectNoNewPeriod(clocks, zeroes)
 
-		triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor)
+		triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor)
 		zeroes = expectNewPeriod(clocks, zeroes)
 	}
 

catchup/catchpointService.go

@@ -22,6 +22,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/algorand/go-algorand/stateproof"
+
 	"github.com/algorand/go-deadlock"
 
 	"github.com/algorand/go-algorand/config"
@@ -464,6 +466,21 @@ func (cs *CatchpointCatchupService) processStageLastestBlockDownload() (err erro
 	return nil
 }
 
+// lookbackForStateproofsSupport calculates the lookback (from topblock round) needed to be downloaded
+// in order to support state proofs verification.
+func lookbackForStateproofsSupport(topBlock *bookkeeping.Block) uint64 {
+	proto := config.Consensus[topBlock.CurrentProtocol]
+	if proto.StateProofInterval == 0 {
+		return 0
+	}
+	lowestStateProofRound := stateproof.GetOldestExpectedStateProof(&topBlock.BlockHeader)
+	// in order to be able to confirm/build lowestStateProofRound we would need to reconstruct
+	// the corresponding voterForRound which is (lowestStateProofRound - stateproofInterval - VotersLookback)
+	lowestStateProofRound = lowestStateProofRound.SubSaturate(basics.Round(proto.StateProofInterval))
+	lowestStateProofRound = lowestStateProofRound.SubSaturate(basics.Round(proto.StateProofVotersLookback))
+	return uint64(topBlock.Round().SubSaturate(lowestStateProofRound))
+}
+
 // processStageBlocksDownload is the fourth catchpoint catchup stage. It downloads all the reminder of the blocks, verifying each one of them against it's predecessor.
 func (cs *CatchpointCatchupService) processStageBlocksDownload() (err error) {
 	topBlock, err := cs.ledgerAccessor.EnsureFirstBlock(cs.ctx)
@@ -478,6 +495,12 @@ func (cs *CatchpointCatchupService) processStageBlocksDownload() (err error) {
 	if lookback < proto.MaxBalLookback {
 		lookback = proto.MaxBalLookback
 	}
+
+	lookbackForStateProofSupport := lookbackForStateproofsSupport(&topBlock)
+	if lookback < lookbackForStateProofSupport {
+		lookback = lookbackForStateProofSupport
+	}
+
 	// in case the effective lookback is going before our rounds count, trim it there.
 	// ( a catchpoint is generated starting round MaxBalLookback, and this is a possible in any round in the range of MaxBalLookback..MaxTxnLife)
 	if lookback >= uint64(topBlock.Round()) {
@@ -567,9 +590,12 @@ func (cs *CatchpointCatchupService) processStageBlocksDownload() (err error) {
 			return cs.abort(fmt.Errorf("processStageBlocksDownload: downloaded block content does not match downloaded block header"))
 		}
 
-		cs.updateBlockRetrievalStatistics(0, 1)
-		peerRank := cs.blocksDownloadPeerSelector.peerDownloadDurationToRank(psp, blockDownloadDuration)
-		cs.blocksDownloadPeerSelector.rankPeer(psp, peerRank)
+		if psp != nil {
+			// the block might have been retrieved from the local ledger, nothing to rank
+			cs.updateBlockRetrievalStatistics(0, 1)
+			peerRank := cs.blocksDownloadPeerSelector.peerDownloadDurationToRank(psp, blockDownloadDuration)
+			cs.blocksDownloadPeerSelector.rankPeer(psp, peerRank)
+		}
 
 		// all good, persist and move on.
 		err = cs.ledgerAccessor.StoreBlock(cs.ctx, blk)

catchup/service_test.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/algorand/go-deadlock"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/algorand/go-algorand/agreement"
@@ -957,7 +958,7 @@ func TestServiceStartStop(t *testing.T) {
 	s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, ledger, &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil)
 	s.Start()
 	s.Stop()
-	_, ok := (<-s.done)
+	_, ok := <-s.done
 	require.False(t, ok)
 }
 
@@ -973,3 +974,42 @@ func TestSynchronizingTime(t *testing.T) {
 	atomic.StoreInt64(&s.syncStartNS, 1000000)
 	require.NotEqual(t, time.Duration(0), s.SynchronizingTime())
 }
+
+func TestDownloadBlocksToSupportStateProofs(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	// make sure we download enough blocks to verify state proof 512
+	topBlk := bookkeeping.Block{}
+	topBlk.BlockHeader.Round = 1500
+	topBlk.BlockHeader.CurrentProtocol = protocol.ConsensusCurrentVersion
+	trackingData := bookkeeping.StateProofTrackingData{StateProofNextRound: 512}
+	topBlk.BlockHeader.StateProofTracking = make(map[protocol.StateProofType]bookkeeping.StateProofTrackingData)
+	topBlk.BlockHeader.StateProofTracking[protocol.StateProofBasic] = trackingData
+
+	lookback := lookbackForStateproofsSupport(&topBlk)
+	oldestRound := topBlk.BlockHeader.Round.SubSaturate(basics.Round(lookback))
+	assert.Equal(t, uint64(oldestRound), 512-config.Consensus[protocol.ConsensusFuture].StateProofInterval-config.Consensus[protocol.ConsensusFuture].StateProofVotersLookback)
+
+	// the network has made progress and now it is on round 8000. in this case we would not download blocks to cover 512.
+	// instead, we will download blocks to confirm only the recovery period lookback.
+	topBlk = bookkeeping.Block{}
+	topBlk.BlockHeader.Round = 8000
+	topBlk.BlockHeader.CurrentProtocol = protocol.ConsensusCurrentVersion
+	trackingData = bookkeeping.StateProofTrackingData{StateProofNextRound: 512}
+	topBlk.BlockHeader.StateProofTracking = make(map[protocol.StateProofType]bookkeeping.StateProofTrackingData)
+	topBlk.BlockHeader.StateProofTracking[protocol.StateProofBasic] = trackingData
+
+	lookback = lookbackForStateproofsSupport(&topBlk)
+	oldestRound = topBlk.BlockHeader.Round.SubSaturate(basics.Round(lookback))
+
+	lowestRoundToRetain := 8000 - (8000 % config.Consensus[protocol.ConsensusCurrentVersion].StateProofInterval) -
+		config.Consensus[protocol.ConsensusCurrentVersion].StateProofInterval*(config.Consensus[protocol.ConsensusCurrentVersion].StateProofMaxRecoveryIntervals+1) - config.Consensus[protocol.ConsensusFuture].StateProofVotersLookback
+
+	assert.Equal(t, uint64(oldestRound), lowestRoundToRetain)
+
+	topBlk = bookkeeping.Block{}
+	topBlk.BlockHeader.Round = 8000
+	topBlk.BlockHeader.CurrentProtocol = protocol.ConsensusV32
+	lookback = lookbackForStateproofsSupport(&topBlk)
+	assert.Equal(t, uint64(0), lookback)
+}

cmd/algokey/part.go

@@ -168,8 +168,9 @@ func printPartkey(partkey account.Participation) {
 	fmt.Printf("Parent address:    %s\n", partkey.Parent.String())
 	fmt.Printf("VRF public key:    %s\n", base64.StdEncoding.EncodeToString(partkey.VRF.PK[:]))
 	fmt.Printf("Voting public key: %s\n", base64.StdEncoding.EncodeToString(partkey.Voting.OneTimeSignatureVerifier[:]))
-	if partkey.StateProofSecrets != nil && !partkey.StateProofSecrets.GetVerifier().IsEmpty() {
-		fmt.Printf("State proof key:   %s\n", base64.StdEncoding.EncodeToString(partkey.StateProofSecrets.GetVerifier()[:]))
+	if partkey.StateProofSecrets != nil && !partkey.StateProofSecrets.GetVerifier().MsgIsZero() {
+		fmt.Printf("State proof key:   %s\n", base64.StdEncoding.EncodeToString(partkey.StateProofSecrets.GetVerifier().Commitment[:]))
+		fmt.Printf("State proof key lifetime:   %d\n", partkey.StateProofSecrets.GetVerifier().KeyLifetime)
 	}
 	fmt.Printf("First round:       %d\n", partkey.FirstValid)
 	fmt.Printf("Last round:        %d\n", partkey.LastValid)

cmd/goal/account.go

@@ -819,7 +819,7 @@ var changeOnlineCmd = &cobra.Command{
 			}
 		}
 
-		firstTxRound, lastTxRound, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		firstTxRound, lastTxRound, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf(err.Error())
 		}
@@ -1430,7 +1430,7 @@ var markNonparticipatingCmd = &cobra.Command{
 
 		dataDir := ensureSingleDataDir()
 		client := ensureFullClient(dataDir)
-		firstTxRound, lastTxRound, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		firstTxRound, lastTxRound, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf(errorConstructingTX, err)
 		}

cmd/goal/application.go

@@ -461,7 +461,7 @@ var createAppCmd = &cobra.Command{
 		tx.Lease = parseLease(cmd)
 
 		// Fill in rounds, fee, etc.
-		fv, lv, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		fv, lv, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -536,7 +536,7 @@ var updateAppCmd = &cobra.Command{
 		tx.Lease = parseLease(cmd)
 
 		// Fill in rounds, fee, etc.
-		fv, lv, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		fv, lv, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -606,7 +606,7 @@ var optInAppCmd = &cobra.Command{
 		tx.Lease = parseLease(cmd)
 
 		// Fill in rounds, fee, etc.
-		fv, lv, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		fv, lv, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -676,7 +676,7 @@ var closeOutAppCmd = &cobra.Command{
 		tx.Lease = parseLease(cmd)
 
 		// Fill in rounds, fee, etc.
-		fv, lv, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		fv, lv, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -746,7 +746,7 @@ var clearAppCmd = &cobra.Command{
 		tx.Lease = parseLease(cmd)
 
 		// Fill in rounds, fee, etc.
-		fv, lv, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		fv, lv, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -816,7 +816,7 @@ var callAppCmd = &cobra.Command{
 		tx.Lease = parseLease(cmd)
 
 		// Fill in rounds, fee, etc.
-		fv, lv, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		fv, lv, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -886,7 +886,7 @@ var deleteAppCmd = &cobra.Command{
 		tx.Lease = parseLease(cmd)
 
 		// Fill in rounds, fee, etc.
-		fv, lv, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		fv, lv, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -1307,7 +1307,7 @@ var methodAppCmd = &cobra.Command{
 		appCallTxn.Lease = parseLease(cmd)
 
 		// Fill in rounds, fee, etc.
-		fv, lv, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		fv, lv, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}

cmd/goal/asset.go

@@ -283,7 +283,7 @@ var createAssetCmd = &cobra.Command{
 		tx.Note = parseNoteField(cmd)
 		tx.Lease = parseLease(cmd)
 
-		fv, lv, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		fv, lv, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -362,7 +362,7 @@ var destroyAssetCmd = &cobra.Command{
 		tx.Note = parseNoteField(cmd)
 		tx.Lease = parseLease(cmd)
 
-		firstValid, lastValid, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		firstValid, lastValid, _, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -455,7 +455,7 @@ var configAssetCmd = &cobra.Command{
 		tx.Note = parseNoteField(cmd)
 		tx.Lease = parseLease(cmd)
 
-		firstValid, lastValid, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		firstValid, lastValid, _, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -540,7 +540,7 @@ var sendAssetCmd = &cobra.Command{
 		tx.Note = parseNoteField(cmd)
 		tx.Lease = parseLease(cmd)
 
-		firstValid, lastValid, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		firstValid, lastValid, _, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -611,7 +611,7 @@ var freezeAssetCmd = &cobra.Command{
 		tx.Note = parseNoteField(cmd)
 		tx.Lease = parseLease(cmd)
 
-		firstValid, lastValid, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		firstValid, lastValid, _, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}
@@ -698,7 +698,7 @@ var optinAssetCmd = &cobra.Command{
 		tx.Note = parseNoteField(cmd)
 		tx.Lease = parseLease(cmd)
 
-		firstValid, lastValid, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		firstValid, lastValid, _, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}

cmd/goal/clerk.go

@@ -384,7 +384,7 @@ var sendCmd = &cobra.Command{
 			}
 		}
 		client := ensureFullClient(dataDir)
-		firstValid, lastValid, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		firstValid, lastValid, _, err = client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf(err.Error())
 		}
@@ -424,7 +424,7 @@ var sendCmd = &cobra.Command{
 					CurrentProtocol: proto,
 				},
 			}
-			groupCtx, err := verify.PrepareGroupContext([]transactions.SignedTxn{uncheckedTxn}, blockHeader)
+			groupCtx, err := verify.PrepareGroupContext([]transactions.SignedTxn{uncheckedTxn}, blockHeader, nil)
 			if err == nil {
 				err = verify.LogicSigSanityCheck(&uncheckedTxn, 0, groupCtx)
 			}
@@ -825,7 +825,7 @@ var signCmd = &cobra.Command{
 			}
 			var groupCtx *verify.GroupContext
 			if lsig.Logic != nil {
-				groupCtx, err = verify.PrepareGroupContext(txnGroup, contextHdr)
+				groupCtx, err = verify.PrepareGroupContext(txnGroup, contextHdr, nil)
 				if err != nil {
 					// this error has to be unsupported protocol
 					reportErrorf("%s: %v", txFilename, err)
@@ -1162,6 +1162,7 @@ var dryrunCmd = &cobra.Command{
 				reportErrorf("program size too large: %d > %d", len(txn.Lsig.Logic), params.LogicSigMaxSize)
 			}
 			ep := logic.NewEvalParams(txgroup, &params, nil)
+			ep.SigLedger = logic.NoHeaderLedger{}
 			err := logic.CheckSignature(i, ep)
 			if err != nil {
 				reportErrorf("program failed Check: %s", err)

cmd/goal/interact.go

@@ -596,7 +596,7 @@ var appExecuteCmd = &cobra.Command{
 		tx.Lease = parseLease(cmd)
 
 		// Fill in rounds, fee, etc.
-		fv, lv, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
+		fv, lv, _, err := client.ComputeValidityRounds(firstValid, lastValid, numValidRounds)
 		if err != nil {
 			reportErrorf("Cannot determine last valid round: %s", err)
 		}

cmd/loadgenerator/main.go

@@ -137,7 +137,7 @@ func main() {
 		}
 	} else if len(algodDir) > 0 {
 		// get test cluster local unlocked wallet
-		privateKeys := findRootKeys(algodDir)
+		privateKeys = findRootKeys(algodDir)
 		if len(privateKeys) == 0 {
 			fmt.Fprintf(os.Stderr, "%s: found no root keys\n", algodDir)
 			os.Exit(1)

cmd/tealdbg/debugger_test.go

@@ -103,6 +103,7 @@ func TestDebuggerSimple(t *testing.T) {
 
 	ep := logic.NewEvalParams(make([]transactions.SignedTxnWithAD, 1), &proto, nil)
 	ep.Debugger = debugger
+	ep.SigLedger = logic.NoHeaderLedger{}
 
 	source := `int 0
 int 1

cmd/tealdbg/local.go

@@ -545,6 +545,7 @@ func (r *LocalRunner) RunAll() error {
 	start := time.Now()
 
 	ep := logic.NewEvalParams(txngroup, &r.proto, &transactions.SpecialAddresses{})
+	ep.SigLedger = logic.NoHeaderLedger{}
 	configureDebugger(ep)
 
 	var last error