go-algorand 3.12.1-beta Release PR

.circleci/config.yml

@@ -7,7 +7,7 @@ version: 2.1
 orbs:
   win: circleci/windows@2.3.0
   go: circleci/go@1.7.0
-  slack: circleci/slack@4.4.2
+  slack: circleci/slack@4.10.1
 
 parameters:
   ubuntu_image:
@@ -64,6 +64,13 @@ executors:
   mac_arm64_large:
     <<: *executor-mac-arm64
 
+slack-fail-stop-step: &slack-fail-post-step
+  post-steps:
+    - slack/notify:
+        branch_pattern: "master,rel/beta,rel/nightly,rel/stable"
+        event: fail
+        template: basic_fail_1
+
 # ===== Workflow Definitions =====
 workflows:
   version: 2
@@ -80,6 +87,7 @@ workflows:
                 - /rel\/.*/
                 - << pipeline.parameters.valid_nightly_branch >>
           context: slack-secrets
+          <<: *slack-fail-post-step
 
       - test:
           name: << matrix.platform >>_test
@@ -91,6 +99,8 @@ workflows:
               ignore:
                 - /rel\/.*/
                 - << pipeline.parameters.valid_nightly_branch >>
+          context: slack-secrets
+          <<: *slack-fail-post-step
 
       - test_nightly:
           name: << matrix.platform >>_test_nightly
@@ -99,13 +109,16 @@ workflows:
           requires:
             - << matrix.platform >>_build_nightly
           context: slack-secrets
+          <<: *slack-fail-post-step
 
       - integration:
           name: << matrix.platform >>_integration
           matrix:
             <<: *matrix-default
           filters:
             <<: *filters-default
+          context: slack-secrets
+          <<: *slack-fail-post-step
 
       - integration_nightly:
           name: << matrix.platform >>_integration_nightly
@@ -114,13 +127,16 @@ workflows:
           requires:
             - << matrix.platform >>_build_nightly
           context: slack-secrets
+          <<: *slack-fail-post-step
 
       - e2e_expect:
           name: << matrix.platform >>_e2e_expect
           matrix:
             <<: *matrix-default
           filters:
             <<: *filters-default
+          context: slack-secrets
+          <<: *slack-fail-post-step
 
       - e2e_expect_nightly:
           name: << matrix.platform >>_e2e_expect_nightly
@@ -129,13 +145,16 @@ workflows:
           requires:
             - << matrix.platform >>_build_nightly
           context: slack-secrets
+          <<: *slack-fail-post-step
 
       - e2e_subs:
           name: << matrix.platform >>_e2e_subs
           matrix:
             <<: *matrix-default
           filters:
             <<: *filters-default
+          context: slack-secrets
+          <<: *slack-fail-post-step
 
       - e2e_subs_nightly:
           name: << matrix.platform >>_e2e_subs_nightly
@@ -146,6 +165,7 @@ workflows:
           context:
             - slack-secrets
             - aws-secrets
+          <<: *slack-fail-post-step
 
       - tests_verification_job:
           name: << matrix.platform >>_<< matrix.job_type >>_verification
@@ -155,6 +175,8 @@ workflows:
               job_type: ["test", "integration", "e2e_expect"]
           requires:
             - << matrix.platform >>_<< matrix.job_type >>
+          context: slack-secrets
+          <<: *slack-fail-post-step
 
       - tests_verification_job_nightly:
           name: << matrix.platform >>_<< matrix.job_type >>_verification
@@ -165,6 +187,7 @@ workflows:
           requires:
             - << matrix.platform >>_<< matrix.job_type >>
           context: slack-secrets
+          <<: *slack-fail-post-step
 
       - upload_binaries:
           name: << matrix.platform >>_upload_binaries
@@ -183,6 +206,7 @@ workflows:
           context:
             - slack-secrets
             - aws-secrets
+          <<: *slack-fail-post-step
 
       #- windows_x64_build
 
@@ -207,9 +231,6 @@ jobs:
             - go
             - gimme
             - .gimme
-      - slack/notify: &slack-fail-event
-          event: fail
-          template: basic_fail_1
 
   test:
     parameters:
@@ -240,9 +261,6 @@ jobs:
           result_subdir: << parameters.platform >>_test_nightly
           no_output_timeout: 45m
       - upload_coverage
-      - slack/notify: &slack-fail-event
-          event: fail
-          template: basic_fail_1
 
   integration:
     parameters:
@@ -275,8 +293,6 @@ jobs:
           platform: << parameters.platform >>
           result_subdir: << parameters.platform >>_integration_nightly
           no_output_timeout: 45m
-      - slack/notify:
-          <<: *slack-fail-event
 
   e2e_expect:
     parameters:
@@ -309,8 +325,6 @@ jobs:
           platform: << parameters.platform >>
           result_subdir: << parameters.platform>>_e2e_expect_nightly
           no_output_timeout: 45m
-      - slack/notify:
-          <<: *slack-fail-event
 
   e2e_subs:
     parameters:
@@ -345,8 +359,6 @@ jobs:
           platform: << parameters.platform >>
           result_subdir: << parameters.platform >>_e2e_subs_nightly
           no_output_timeout: 45m
-      - slack/notify:
-          <<: *slack-fail-event
 
   windows_x64_build:
     executor:
@@ -396,8 +408,6 @@ jobs:
       - checkout
       - tests_verification_command:
           result_subdir: << parameters.platform >>_<< parameters.job_type >>
-      - slack/notify:
-          <<: *slack-fail-event
 
   upload_binaries:
     working_directory: << pipeline.parameters.build_dir >>/project
@@ -410,8 +420,6 @@ jobs:
       - prepare_go
       - upload_binaries_command:
           platform: << parameters.platform >>
-      - slack/notify:
-          <<: *slack-fail-event
 
 # ===== Command Definitions =====
 commands:

.github/workflows/benchmarks.yml

@@ -49,3 +49,9 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           alert-threshold: '200%'
           comment-on-alert: true
+      - name: Slack Notification
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        run: |
+          curl -X POST --data-urlencode "payload={\"text\": \"Benchmark workflow failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK
+        if: ${{ failure() && (contains(github.ref_name, 'rel/nightly') || contains(github.ref_name, 'rel/beta') || contains(github.ref_name, 'rel/stable') || contains(github.ref_name, 'master')) }}

.github/workflows/build.yml

@@ -44,5 +44,5 @@ jobs:
         env:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         run: |
-          curl -X POST --data-urlencode "payload={\"text\": \"Nightly windows build test on Github failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK
-        if: ${{ failure() && (contains(github.base_ref, 'rel/nightly') || contains(github.base_ref, 'rel/beta') || contains(github.base_ref, 'rel/stable')) }}
+          curl -X POST --data-urlencode "payload={\"text\": \"Build windows failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK
+        if: ${{ failure() && (contains(github.ref_name, 'rel/nightly') || contains(github.ref_name, 'rel/beta') || contains(github.ref_name, 'rel/stable') || contains(github.ref_name, 'master')) }}

.github/workflows/codegen_verification.yml

@@ -20,3 +20,9 @@ jobs:
           export GOPATH="${GITHUB_WORKSPACE}/go"
           cd go-algorand
           scripts/travis/codegen_verification.sh
+      - name: Slack Notification
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        run: |
+          curl -X POST --data-urlencode "payload={\"text\": \"Codegen verification failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK
+        if: ${{ failure() && (contains(github.ref_name, 'rel/nightly') || contains(github.ref_name, 'rel/beta') || contains(github.ref_name, 'rel/stable') || contains(github.ref_name, 'master')) }}

.github/workflows/reviewdog.yml

@@ -92,3 +92,9 @@ jobs:
           -filter-mode=added
           -fail-on-error=false
           -level=warning
+      - name: Slack Notification
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        run: |
+          curl -X POST --data-urlencode "payload={\"text\": \"Reviewdog failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK
+        if: ${{ failure() && (contains(github.ref_name, 'rel/nightly') || contains(github.ref_name, 'rel/beta') || contains(github.ref_name, 'rel/stable') || contains(github.ref_name, 'master')) }}

.golangci-warnings.yml

@@ -8,8 +8,8 @@ linters:
     - deadcode
     - partitiontest
     - structcheck
-    - typecheck
     - varcheck
+    - unconvert
     - unused
 
 
@@ -52,24 +52,19 @@ issues:
     # be more lenient with test code
     - path: _test\.go
       linters:
-        - staticcheck
+        - deadcode
         - structcheck
-        - typecheck
         - varcheck
-        - deadcode
-        - gosimple
+        - unconvert
         - unused
     # Add all linters here -- Comment this block out for testing linters
     - path: test/linttest/lintissues\.go
       linters:
-        - staticcheck
+        - deadcode
         - structcheck
-        - typecheck
         - varcheck
-        - deadcode
-        - gosimple
+        - unconvert
         - unused
-        - partitiontest
     - path: crypto/secp256k1/secp256_test\.go
       linters:
         - partitiontest

.golangci.yml

@@ -16,6 +16,7 @@ linters:
     - revive
     - staticcheck
     - typecheck
+    - paralleltest
 
 severity:
   default-severity: error
@@ -115,6 +116,10 @@ issues:
         # - revive
         - staticcheck
         - typecheck
+    # Ignore missing parallel tests in existing packages
+    - path: (agreement|catchup|cmd|config|crypto|daemon|data|gen|ledger|logging|netdeploy|network|node|protocol|rpcs|shared|stateproof|test|tools|util).*_test.go
+      linters:
+        - paralleltest
     # Add all linters here -- Comment this block out for testing linters
     - path: test/linttest/lintissues\.go
       linters:

SECURITY.md

@@ -4,4 +4,4 @@ Algorand takes the security of the platform and of its users very seriously. We
 
 If you believe that you have found a security vulnerability you may disclose it here:
 
-https://www.algorand.com/resources/blog/security
+https://immunefi.com/bounty/algorand/

buildnumber.dat

@@ -1 +1 @@
-0
+1

catchup/catchpointService.go

@@ -53,11 +53,15 @@ type CatchpointCatchupStats struct {
 	TotalAccounts      uint64
 	ProcessedAccounts  uint64
 	VerifiedAccounts   uint64
+	TotalKVs           uint64
+	ProcessedKVs       uint64
+	VerifiedKVs        uint64
 	TotalBlocks        uint64
 	AcquiredBlocks     uint64
 	VerifiedBlocks     uint64
 	ProcessedBytes     uint64
 	TotalAccountHashes uint64
+	TotalKVHashes      uint64
 	StartTime          time.Time
 }
 
@@ -303,7 +307,7 @@ func (cs *CatchpointCatchupService) processStageLedgerDownload() (err error) {
 		if err == nil {
 			cs.log.Infof("ledger downloaded in %d seconds", time.Since(start)/time.Second)
 			start = time.Now()
-			err = cs.ledgerAccessor.BuildMerkleTrie(cs.ctx, cs.updateVerifiedAccounts)
+			err = cs.ledgerAccessor.BuildMerkleTrie(cs.ctx, cs.updateVerifiedCounts)
 			if err == nil {
 				cs.log.Infof("built merkle trie in %d seconds", time.Since(start)/time.Second)
 				break
@@ -335,12 +339,17 @@ func (cs *CatchpointCatchupService) processStageLedgerDownload() (err error) {
 	return nil
 }
 
-// updateVerifiedAccounts update the user's statistics for the given verified accounts
-func (cs *CatchpointCatchupService) updateVerifiedAccounts(addedTrieHashes uint64) {
+// updateVerifiedCounts update the user's statistics for the given verified hashes
+func (cs *CatchpointCatchupService) updateVerifiedCounts(accountCount, kvCount uint64) {
 	cs.statsMu.Lock()
 	defer cs.statsMu.Unlock()
+
 	if cs.stats.TotalAccountHashes > 0 {
-		cs.stats.VerifiedAccounts = cs.stats.TotalAccounts * addedTrieHashes / cs.stats.TotalAccountHashes
+		cs.stats.VerifiedAccounts = cs.stats.TotalAccounts * accountCount / cs.stats.TotalAccountHashes
+	}
+
+	if cs.stats.TotalKVs > 0 {
+		cs.stats.VerifiedKVs = kvCount
 	}
 }
 
@@ -756,6 +765,8 @@ func (cs *CatchpointCatchupService) updateLedgerFetcherProgress(fetcherStats *le
 	defer cs.statsMu.Unlock()
 	cs.stats.TotalAccounts = fetcherStats.TotalAccounts
 	cs.stats.ProcessedAccounts = fetcherStats.ProcessedAccounts
+	cs.stats.TotalKVs = fetcherStats.TotalKVs
+	cs.stats.ProcessedKVs = fetcherStats.ProcessedKVs
 	cs.stats.ProcessedBytes = fetcherStats.ProcessedBytes
 	cs.stats.TotalAccountHashes = fetcherStats.TotalAccountHashes
 }

catchup/fetcher_test.go

@@ -148,7 +148,7 @@ func (b *basicRPCNode) RegisterHandlers(dispatch []network.TaggedMessageHandler)
 
 func (b *basicRPCNode) start() bool {
 	var err error
-	b.listener, err = net.Listen("tcp", "")
+	b.listener, err = net.Listen("tcp", "127.0.0.1:")
 	if err != nil {
 		logging.Base().Error("tcp listen", err)
 		return false

catchup/service.go

@@ -124,6 +124,7 @@ func MakeService(log logging.Logger, config config.Local, net network.GossipNode
 func (s *Service) Start() {
 	s.done = make(chan struct{})
 	s.ctx, s.cancel = context.WithCancel(context.Background())
+	atomic.StoreUint32(&s.initialSyncNotified, 0)
 	s.InitialSyncDone = make(chan struct{})
 	go s.periodicSync()
 }