Fix counting logic

catchup/catchpointService.go

@@ -340,26 +340,16 @@ func (cs *CatchpointCatchupService) processStageLedgerDownload() (err error) {
 }
 
 // updateVerifiedCounts update the user's statistics for the given verified hashes
-func (cs *CatchpointCatchupService) updateVerifiedCounts(hashes [][]byte) {
+func (cs *CatchpointCatchupService) updateVerifiedCounts(kvCount, accountCount uint64) {
 	cs.statsMu.Lock()
 	defer cs.statsMu.Unlock()
 
-	addedTrieAccountHashes := uint64(0) // Accounts include accounts + creatables (assets + apps)
-	addedTrieKVHashes := uint64(0)
-	for _, hash := range hashes {
-		if hash[ledger.HashKindEncodingIndex] == byte(ledger.KV) {
-			addedTrieKVHashes++
-		} else {
-			addedTrieAccountHashes++
-		}
-	}
-
 	if cs.stats.TotalAccountHashes > 0 {
-		cs.stats.VerifiedAccounts = cs.stats.TotalAccounts * addedTrieAccountHashes / cs.stats.TotalAccountHashes
+		cs.stats.VerifiedAccounts = cs.stats.TotalAccounts * accountCount / cs.stats.TotalAccountHashes
 	}
 
-	if cs.stats.TotalKVs > 0 { // TODO Is TotalKVHashes needed?
-		cs.stats.VerifiedKVs = addedTrieKVHashes
+	if cs.stats.TotalKVs > 0 {
+		cs.stats.VerifiedKVs = kvCount
 	}
 }
 

components/mocks/mockCatchpointCatchupAccessor.go

@@ -58,7 +58,7 @@ func (m *MockCatchpointCatchupAccessor) ProcessStagingBalances(ctx context.Conte
 }
 
 // BuildMerkleTrie inserts the account hashes into the merkle trie
-func (m *MockCatchpointCatchupAccessor) BuildMerkleTrie(ctx context.Context, progressUpdates func([][]byte)) (err error) {
+func (m *MockCatchpointCatchupAccessor) BuildMerkleTrie(ctx context.Context, progressUpdates func(uint64, uint64)) (err error) {
 	return nil
 }
 

ledger/catchupaccessor.go

@@ -59,7 +59,7 @@ type CatchpointCatchupAccessor interface {
 	ProcessStagingBalances(ctx context.Context, sectionName string, bytes []byte, progress *CatchpointCatchupAccessorProgress) (err error)
 
 	// BuildMerkleTrie inserts the account hashes into the merkle trie
-	BuildMerkleTrie(ctx context.Context, progressUpdates func([][]byte)) (err error)
+	BuildMerkleTrie(ctx context.Context, progressUpdates func(uint64, uint64)) (err error)
 
 	// GetCatchupBlockRound returns the latest block round matching the current catchpoint
 	GetCatchupBlockRound(ctx context.Context) (round basics.Round, err error)
@@ -593,8 +593,24 @@ func (c *catchpointCatchupAccessorImpl) processStagingBalances(ctx context.Conte
 	return err
 }
 
+// countHashes disambiguates the 2 hash types included in the merkle trie:
+// * accounts + createables (assets + apps)
+// * KVs
+//
+// The function is _not_ a general purpose way to count hashes by hash kind.
+func countHashes(hashes [][]byte) (kvCount, accountCount uint64) {
+	for _, hash := range hashes {
+		if hash[HashKindEncodingIndex] == byte(KV) {
+			kvCount++
+		} else {
+			accountCount++
+		}
+	}
+	return kvCount, accountCount
+}
+
 // BuildMerkleTrie would process the catchpointpendinghashes and insert all the items in it into the merkle trie
-func (c *catchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, progressUpdates func([][]byte)) (err error) {
+func (c *catchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, progressUpdates func(uint64, uint64)) (err error) {
 	wdb := c.ledger.trackerDB().Wdb
 	rdb := c.ledger.trackerDB().Rdb
 	err = wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
@@ -654,6 +670,7 @@ func (c *catchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, pro
 		var trie *merkletrie.Trie
 		uncommitedHashesCount := 0
 		keepWriting := true
+		kvHashesWritten, accountHashesWritten := uint64(0), uint64(0)
 		var mc *MerkleCommitter
 
 		err := wdb.Atomic(func(transactionCtx context.Context, tx *sql.Tx) (err error) {
@@ -703,9 +720,10 @@ func (c *catchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, pro
 
 				}
 				uncommitedHashesCount += len(hashesToWrite)
-				if progressUpdates != nil {
-					progressUpdates(hashesToWrite)
-				}
+
+				kvs, accounts := countHashes(hashesToWrite)
+				kvHashesWritten += kvs
+				accountHashesWritten += accounts
 
 				return nil
 			})
@@ -734,6 +752,10 @@ func (c *catchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, pro
 					continue
 				}
 			}
+
+			if progressUpdates != nil {
+				progressUpdates(kvHashesWritten, accountHashesWritten)
+			}
 		}
 		if err != nil {
 			errChan <- err

ledger/catchupaccessor_test.go

@@ -218,7 +218,7 @@ func TestBuildMerkleTrie(t *testing.T) {
 	catchpointAccessor := MakeCatchpointCatchupAccessor(l, log)
 
 	progressCallCount := 0
-	progressNop := func([][]byte) {
+	progressNop := func(uint64, uint64) {
 		progressCallCount++
 	}
 
@@ -317,7 +317,7 @@ func TestCatchupAccessorBlockdb(t *testing.T) {
 	ctx := context.Background()
 
 	// actual testing...
-	err = catchpointAccessor.BuildMerkleTrie(ctx, func([][]byte) {})
+	err = catchpointAccessor.BuildMerkleTrie(ctx, func(uint64, uint64) {})
 	require.Error(t, err)
 }