Merge branch 'master' into use-avm-abi

.circleci/config.yml

@@ -70,7 +70,7 @@ workflows:
           name: << matrix.platform >>_build
           matrix: &matrix-default
             parameters:
-              platform: ["amd64", "arm64", "mac_amd64"]
+              platform: ["amd64", "arm64"]
           filters: &filters-default
             branches:
               ignore:
@@ -155,7 +155,7 @@ workflows:
           name: << matrix.platform >>_<< matrix.job_type >>_verification
           matrix:
             parameters:
-              platform: ["amd64", "arm64", "mac_amd64"]
+              platform: ["amd64", "arm64"]
               job_type: ["test", "integration", "e2e_expect"]
           requires:
             - << matrix.platform >>_<< matrix.job_type >>

agreement/service_test.go

@@ -1968,7 +1968,7 @@ func TestAgreementSlowPayloadsPostDeadline(t *testing.T) {
 		activityMonitor.waitForQuiet()
 		zeroes = expectNoNewPeriod(clocks, zeroes)
 
-		triggerGlobalTimeout(FilterTimeout(0, version), clocks, activityMonitor)
+		triggerGlobalTimeout(FilterTimeout(1, version), clocks, activityMonitor)
 		zeroes = expectNewPeriod(clocks, zeroes)
 	}
 

catchup/catchpointService.go

@@ -19,10 +19,11 @@ package catchup
 import (
 	"context"
 	"fmt"
-	"github.com/algorand/go-algorand/stateproof"
 	"sync"
 	"time"
 
+	"github.com/algorand/go-algorand/stateproof"
+
 	"github.com/algorand/go-deadlock"
 
 	"github.com/algorand/go-algorand/config"
@@ -473,8 +474,10 @@ func lookbackForStateproofsSupport(topBlock *bookkeeping.Block) uint64 {
 		return 0
 	}
 	lowestStateProofRound := stateproof.GetOldestExpectedStateProof(&topBlock.BlockHeader)
-	// in order to be able to confirm lowestStateProofRound we need to have round number: (lowestStateProofRound - stateproofInterval)
+	// in order to be able to confirm/build lowestStateProofRound we would need to reconstruct
+	// the corresponding voterForRound which is (lowestStateProofRound - stateproofInterval - VotersLookback)
 	lowestStateProofRound = lowestStateProofRound.SubSaturate(basics.Round(proto.StateProofInterval))
+	lowestStateProofRound = lowestStateProofRound.SubSaturate(basics.Round(proto.StateProofVotersLookback))
 	return uint64(topBlock.Round().SubSaturate(lowestStateProofRound))
 }
 
@@ -497,6 +500,7 @@ func (cs *CatchpointCatchupService) processStageBlocksDownload() (err error) {
 	if lookback < lookbackForStateProofSupport {
 		lookback = lookbackForStateProofSupport
 	}
+
 	// in case the effective lookback is going before our rounds count, trim it there.
 	// ( a catchpoint is generated starting round MaxBalLookback, and this is a possible in any round in the range of MaxBalLookback..MaxTxnLife)
 	if lookback >= uint64(topBlock.Round()) {
@@ -586,9 +590,12 @@ func (cs *CatchpointCatchupService) processStageBlocksDownload() (err error) {
 			return cs.abort(fmt.Errorf("processStageBlocksDownload: downloaded block content does not match downloaded block header"))
 		}
 
-		cs.updateBlockRetrievalStatistics(0, 1)
-		peerRank := cs.blocksDownloadPeerSelector.peerDownloadDurationToRank(psp, blockDownloadDuration)
-		cs.blocksDownloadPeerSelector.rankPeer(psp, peerRank)
+		if psp != nil {
+			// the block might have been retrieved from the local ledger, nothing to rank
+			cs.updateBlockRetrievalStatistics(0, 1)
+			peerRank := cs.blocksDownloadPeerSelector.peerDownloadDurationToRank(psp, blockDownloadDuration)
+			cs.blocksDownloadPeerSelector.rankPeer(psp, peerRank)
+		}
 
 		// all good, persist and move on.
 		err = cs.ledgerAccessor.StoreBlock(cs.ctx, blk)

catchup/service_test.go

@@ -977,31 +977,34 @@ func TestSynchronizingTime(t *testing.T) {
 
 func TestDownloadBlocksToSupportStateProofs(t *testing.T) {
 	partitiontest.PartitionTest(t)
+
 	// make sure we download enough blocks to verify state proof 512
 	topBlk := bookkeeping.Block{}
 	topBlk.BlockHeader.Round = 1500
-	topBlk.BlockHeader.CurrentProtocol = protocol.ConsensusFuture
+	topBlk.BlockHeader.CurrentProtocol = protocol.ConsensusCurrentVersion
 	trackingData := bookkeeping.StateProofTrackingData{StateProofNextRound: 512}
 	topBlk.BlockHeader.StateProofTracking = make(map[protocol.StateProofType]bookkeeping.StateProofTrackingData)
 	topBlk.BlockHeader.StateProofTracking[protocol.StateProofBasic] = trackingData
 
 	lookback := lookbackForStateproofsSupport(&topBlk)
 	oldestRound := topBlk.BlockHeader.Round.SubSaturate(basics.Round(lookback))
-	assert.Equal(t, uint64(oldestRound), 512-config.Consensus[protocol.ConsensusFuture].StateProofInterval)
+	assert.Equal(t, uint64(oldestRound), 512-config.Consensus[protocol.ConsensusFuture].StateProofInterval-config.Consensus[protocol.ConsensusFuture].StateProofVotersLookback)
 
 	// the network has made progress and now it is on round 8000. in this case we would not download blocks to cover 512.
 	// instead, we will download blocks to confirm only the recovery period lookback.
 	topBlk = bookkeeping.Block{}
 	topBlk.BlockHeader.Round = 8000
-	topBlk.BlockHeader.CurrentProtocol = protocol.ConsensusFuture
+	topBlk.BlockHeader.CurrentProtocol = protocol.ConsensusCurrentVersion
 	trackingData = bookkeeping.StateProofTrackingData{StateProofNextRound: 512}
 	topBlk.BlockHeader.StateProofTracking = make(map[protocol.StateProofType]bookkeeping.StateProofTrackingData)
 	topBlk.BlockHeader.StateProofTracking[protocol.StateProofBasic] = trackingData
 
 	lookback = lookbackForStateproofsSupport(&topBlk)
 	oldestRound = topBlk.BlockHeader.Round.SubSaturate(basics.Round(lookback))
 
-	lowestRoundToRetain := 8000 - (8000 % 256) - (config.Consensus[protocol.ConsensusFuture].StateProofInterval * (config.Consensus[protocol.ConsensusFuture].StateProofMaxRecoveryIntervals + 1))
+	lowestRoundToRetain := 8000 - (8000 % config.Consensus[protocol.ConsensusCurrentVersion].StateProofInterval) -
+		config.Consensus[protocol.ConsensusCurrentVersion].StateProofInterval*(config.Consensus[protocol.ConsensusCurrentVersion].StateProofMaxRecoveryIntervals+1) - config.Consensus[protocol.ConsensusFuture].StateProofVotersLookback
+
 	assert.Equal(t, uint64(oldestRound), lowestRoundToRetain)
 
 	topBlk = bookkeeping.Block{}

cmd/loadgenerator/main.go

@@ -137,7 +137,7 @@ func main() {
 		}
 	} else if len(algodDir) > 0 {
 		// get test cluster local unlocked wallet
-		privateKeys := findRootKeys(algodDir)
+		privateKeys = findRootKeys(algodDir)
 		if len(privateKeys) == 0 {
 			fmt.Fprintf(os.Stderr, "%s: found no root keys\n", algodDir)
 			os.Exit(1)

config/consensus.go

@@ -1150,37 +1150,59 @@ func initConsensusProtocols() {
 	// v31 can be upgraded to v32, with an update delay of 7 days ( see calculation above )
 	v31.ApprovedUpgrades[protocol.ConsensusV32] = 140000
 
-	// ConsensusFuture is used to test features that are implemented
-	// but not yet released in a production protocol version.
-	vFuture := v32
-	vFuture.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{}
+	v33 := v32
+	v33.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{}
 
 	// Make the accounts snapshot for round X at X-CatchpointLookback
-	vFuture.CatchpointLookback = 320
+	// order to guarantee all nodes produce catchpoint at the same round.
+	v33.CatchpointLookback = 320
 
 	// Require MaxTxnLife + X blocks and headers preserved by a node
-	vFuture.DeeperBlockHeaderHistory = 1
+	v33.DeeperBlockHeaderHistory = 1
+
+	v33.MaxTxnBytesPerBlock = 5 * 1024 * 1024
+
+	Consensus[protocol.ConsensusV33] = v33
+
+	// v32 can be upgraded to v33, with an update delay of 7 days ( see calculation above )
+	v32.ApprovedUpgrades[protocol.ConsensusV33] = 140000
+
+	v34 := v33
+	v34.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{}
 
 	// Enable state proofs.
-	vFuture.StateProofInterval = 256
-	vFuture.StateProofTopVoters = 1024
-	vFuture.StateProofVotersLookback = 16
-	vFuture.StateProofWeightThreshold = (1 << 32) * 30 / 100
-	vFuture.StateProofStrengthTarget = 256
-	vFuture.StateProofMaxRecoveryIntervals = 10
+	v34.StateProofInterval = 256
+	v34.StateProofTopVoters = 1024
+	v34.StateProofVotersLookback = 16
+	v34.StateProofWeightThreshold = (1 << 32) * 30 / 100
+	v34.StateProofStrengthTarget = 256
+	v34.StateProofMaxRecoveryIntervals = 10
 
-	vFuture.LogicSigVersion = 7 // When moving this to a release, put a new higher LogicSigVersion here
-	vFuture.MinInnerApplVersion = 4
+	v34.LogicSigVersion = 7
+	v34.MinInnerApplVersion = 4
 
-	vFuture.UnifyInnerTxIDs = true
+	v34.UnifyInnerTxIDs = true
 
-	vFuture.EnableSHA256TxnCommitmentHeader = true
-	vFuture.EnableOnlineAccountCatchpoints = true
+	v34.EnableSHA256TxnCommitmentHeader = true
+	v34.EnableOnlineAccountCatchpoints = true
 
-	vFuture.UnfundedSenders = true
+	v34.UnfundedSenders = true
+
+	v34.AgreementFilterTimeoutPeriod0 = 3400 * time.Millisecond
+
+	Consensus[protocol.ConsensusV34] = v34
+
+	// v33 can be upgraded to v34, with an update delay of 12h:
+	// 10046 = (12 * 60 * 60 / 4.3)
+	// for the sake of future manual calculations, we'll round that down a bit :
+	v33.ApprovedUpgrades[protocol.ConsensusV34] = 10000
+
+	// ConsensusFuture is used to test features that are implemented
+	// but not yet released in a production protocol version.
+	vFuture := v34
+	vFuture.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{}
 
-	vFuture.AgreementFilterTimeoutPeriod0 = 3400 * time.Millisecond
-	vFuture.MaxTxnBytesPerBlock = 5 * 1024 * 1024
+	vFuture.LogicSigVersion = 8 // When moving this to a release, put a new higher LogicSigVersion here
 
 	Consensus[protocol.ConsensusFuture] = vFuture
 }

crypto/merklesignature/kats_test.go

@@ -60,7 +60,7 @@ func generateMssKat(startRound, atRound, numOfKeys uint64, messageToSign []byte)
 		return mssKat{}, fmt.Errorf("error: Signature round cann't be smaller then start round")
 	}
 
-	interval := config.Consensus[protocol.ConsensusFuture].StateProofInterval
+	interval := config.Consensus[protocol.ConsensusCurrentVersion].StateProofInterval
 	stateProofSecrets, err := New(startRound, startRound+(interval*numOfKeys)-1, interval)
 	if err != nil {
 		return mssKat{}, fmt.Errorf("error: %w", err)

daemon/algod/api/algod.oas2.json

@@ -531,8 +531,8 @@
         "schemes": [
           "http"
         ],
-        "summary": "Get a Merkle proof for a transaction in a block.",
-        "operationId": "GetProof",
+        "summary": "Get a proof for a transaction in a block.",
+        "operationId": "GetTransactionProof",
         "parameters": [
           {
             "type": "integer",
@@ -566,7 +566,7 @@
         ],
         "responses": {
           "200": {
-            "$ref": "#/responses/ProofResponse"
+            "$ref": "#/responses/TransactionProofResponse"
           },
           "400": {
             "description": "Malformed round number or transaction ID",
@@ -587,7 +587,7 @@
             }
           },
           "500": {
-            "description": "Internal error, including protocol not supporting Merkle proofs.",
+            "description": "Internal error, including protocol not supporting proofs.",
             "schema": {
               "$ref": "#/definitions/ErrorResponse"
             }
@@ -1317,7 +1317,7 @@
           "http"
         ],
         "summary": "Gets a proof for a given light block header inside a state proof commitment",
-        "operationId": "GetProofForLightBlockHeader",
+        "operationId": "GetLightBlockHeaderProof",
         "parameters": [
           {
             "type": "integer",
@@ -2683,9 +2683,7 @@
       ],
       "properties": {
         "Message": {
-          "description": "The encoded message.",
-          "type": "string",
-          "format": "byte"
+          "$ref": "#/definitions/StateProofMessage"
         },
         "StateProof": {
           "description": "The encoded StateProof for the message.",
@@ -2717,6 +2715,44 @@
           "format": "byte"
         }
       }
+    },
+    "StateProofMessage": {
+      "description": "Represents the message that the state proofs are attesting to.",
+      "type": "object",
+      "required": [
+        "BlockHeadersCommitment",
+        "VotersCommitment",
+        "LnProvenWeight",
+        "FirstAttestedRound",
+        "LastAttestedRound"
+      ],
+      "properties": {
+        "BlockHeadersCommitment": {
+          "description": "The vector commitment root on all light block headers within a state proof interval.",
+          "type": "string",
+          "format": "byte"
+        },
+        "VotersCommitment": {
+          "description": "The vector commitment root of the top N accounts to sign the next StateProof.",
+          "type": "string",
+          "format": "byte"
+        },
+        "LnProvenWeight": {
+          "description": "An integer value representing the natural log of the proven weight with 16 bits of precision. This value would be used to verify the next state proof.",
+          "type": "integer",
+          "x-algorand-format": "uint64"
+        },
+        "FirstAttestedRound": {
+          "description": "The first round the message attests to.",
+          "type": "integer",
+          "x-algorand-format": "uint64"
+        },
+        "LastAttestedRound": {
+          "description": "The last round the message attests to.",
+          "type": "integer",
+          "x-algorand-format": "uint64"
+        }
+      }
     }
   },
   "parameters": {
@@ -2976,7 +3012,7 @@
         }
       }
     },
-    "ProofResponse": {
+    "TransactionProofResponse": {
       "description": "Proof of transaction in a block.",
       "schema": {
         "type": "object",
@@ -2989,7 +3025,7 @@
         ],
         "properties": {
           "proof": {
-            "description": "Merkle proof of transaction membership.",
+            "description": "Proof of transaction membership.",
             "type": "string",
             "format": "byte"
           },