refactor: backport remove genesis persistence from state

This avoids saving the genesisDoc to database. When using goleveldb and ~4GiB+ genesis files, it causes a panic during snappy encoding (panic: snappy: decoded block is too large). refs akash-network/support#280 backports: - cometbft#1017 - cometbft#1293 Signed-off-by: Artur Troian <troian.ap@gmail.com>
akash-network · Feb 6, 2025 · 4967515 · 4967515
1 parent 8bcde65
commit 4967515
Show file tree

Hide file tree

Showing 12 changed files with 292 additions and 81 deletions.
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -15,19 +15,19 @@ jobs:
       - name: Split pkgs into 4 files
         run: split -d -n l/4 pkgs.txt pkgs.txt.part.
       # cache multiple
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: "${{ github.sha }}-00"
           path: ./pkgs.txt.part.00
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: "${{ github.sha }}-01"
           path: ./pkgs.txt.part.01
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: "${{ github.sha }}-02"
           path: ./pkgs.txt.part.02
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: "${{ github.sha }}-03"
           path: ./pkgs.txt.part.03
@@ -75,15 +75,15 @@ jobs:
             **/**.go
             go.mod
             go.sum
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: "${{ github.sha }}-${{ matrix.part }}"
         if: env.GIT_DIFF
       - name: test & coverage report creation
         run: |
           cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 8m -race -coverprofile=${{ matrix.part }}profile.out -covermode=atomic
         if: env.GIT_DIFF
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: "${{ github.sha }}-${{ matrix.part }}-coverage"
           path: ./${{ matrix.part }}profile.out
@@ -99,19 +99,19 @@ jobs:
             **/**.go
             go.mod
             go.sum
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: "${{ github.sha }}-00-coverage"
         if: env.GIT_DIFF
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: "${{ github.sha }}-01-coverage"
         if: env.GIT_DIFF
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: "${{ github.sha }}-02-coverage"
         if: env.GIT_DIFF
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: "${{ github.sha }}-03-coverage"
         if: env.GIT_DIFF

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -22,7 +22,10 @@ jobs:
             **/**.go
             go.mod
             go.sum
-
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
       - name: Build
         working-directory: test/e2e
         # Run two make jobs in parallel, since we can't run steps in parallel.

diff --git a/.github/workflows/fuzz-nightly.yml b/.github/workflows/fuzz-nightly.yml
@@ -45,14 +45,14 @@ jobs:
         continue-on-error: true
 
       - name: Archive crashers
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: crashers
           path: test/fuzz/**/crashers
           retention-days: 1
 
       - name: Archive suppressions
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: suppressions
           path: test/fuzz/**/suppressions

diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go
@@ -33,7 +33,7 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-//----------------------------------------------
+// ----------------------------------------------
 // byzantine failures
 
 // Byzantine node sends two different prevotes (nil and blockID) to the same validator
@@ -54,7 +54,9 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
 		stateStore := sm.NewStore(stateDB, sm.StoreOptions{
 			DiscardABCIResponses: false,
 		})
-		state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+		state, err := sm.MakeGenesisState(genDoc)
+		require.NoError(t, err)
+		require.NoError(t, stateStore.Save(state))
 		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
 		defer os.RemoveAll(thisConfig.RootDir)
 		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
@@ -462,7 +464,7 @@ func TestByzantineConflictingProposalsWithPartition(t *testing.T) {
 	}
 }
 
-//-------------------------------
+// -------------------------------
 // byzantine consensus functions
 
 func byzantineDecideProposalFunc(t *testing.T, height int64, round int32, cs *State, sw *p2p.Switch) {
@@ -556,7 +558,7 @@ func sendProposalAndParts(
 	}, cs.Logger)
 }
 
-//----------------------------------------
+// ----------------------------------------
 // byzantine consensus reactor
 
 type ByzantineReactor struct {

diff --git a/consensus/common_test.go b/consensus/common_test.go
@@ -65,7 +65,7 @@ func ResetConfig(name string) *cfg.Config {
 	return cfg.ResetTestRoot(name)
 }
 
-//-------------------------------------------------------------------------------
+// -------------------------------------------------------------------------------
 // validator stub (a kvstore consensus peer we control)
 
 type validatorStub struct {
@@ -190,7 +190,7 @@ func (vss ValidatorStubsByPower) Swap(i, j int) {
 	vss[j].Index = int32(j)
 }
 
-//-------------------------------------------------------------------------------
+// -------------------------------------------------------------------------------
 // Functions for transitioning the consensus state
 
 func startTestRound(cs *State, height int64, round int32) {
@@ -361,7 +361,7 @@ func subscribeToVoter(cs *State, addr []byte) <-chan cmtpubsub.Message {
 	return ch
 }
 
-//-------------------------------------------------------------------------------
+// -------------------------------------------------------------------------------
 // consensus states
 
 func newState(state sm.State, pv types.PrivValidator, app abci.Application) *State {
@@ -475,7 +475,7 @@ func randState(nValidators int) (*State, []*validatorStub) {
 	return cs, vss
 }
 
-//-------------------------------------------------------------------------------
+// -------------------------------------------------------------------------------
 
 func ensureNoNewEvent(ch <-chan cmtpubsub.Message, timeout time.Duration,
 	errorMessage string,
@@ -697,7 +697,7 @@ func ensureNewEventOnChannel(ch <-chan cmtpubsub.Message) {
 	}
 }
 
-//-------------------------------------------------------------------------------
+// -------------------------------------------------------------------------------
 // consensus nets
 
 // consensusLogger is a TestingLogger which uses a different
@@ -722,10 +722,7 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou
 	configRootDirs := make([]string, 0, nValidators)
 	for i := 0; i < nValidators; i++ {
 		stateDB := dbm.NewMemDB() // each state needs its own db
-		stateStore := sm.NewStore(stateDB, sm.StoreOptions{
-			DiscardABCIResponses: false,
-		})
-		state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+		state, _ := sm.MakeGenesisState(genDoc)
 		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
 		configRootDirs = append(configRootDirs, thisConfig.RootDir)
 		for _, opt := range configOpts {
@@ -817,7 +814,7 @@ func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int {
 	panic("didnt find peer in switches")
 }
 
-//-------------------------------------------------------------------------------
+// -------------------------------------------------------------------------------
 // genesis
 
 func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []types.PrivValidator) {
@@ -847,7 +844,7 @@ func randGenesisState(numValidators int, randPower bool, minPower int64) (sm.Sta
 	return s0, privValidators
 }
 
-//------------------------------------
+// ------------------------------------
 // mock ticker
 
 func newMockTickerFunc(onlyOnce bool) func() TimeoutTicker {
@@ -895,7 +892,7 @@ func (m *mockTicker) Chan() <-chan timeoutInfo {
 
 func (*mockTicker) SetLogger(log.Logger) {}
 
-//------------------------------------
+// ------------------------------------
 
 func newCounter() abci.Application {
 	return counter.NewApplication(true)

diff --git a/consensus/replay_test.go b/consensus/replay_test.go
@@ -56,18 +56,17 @@ func TestMain(m *testing.M) {
 // the `Handshake Tests` are for failures in applying the block.
 // With the help of the WAL, we can recover from it all!
 
-//------------------------------------------------------------------------------------------
+// ------------------------------------------------------------------------------------------
 // WAL Tests
 
 // TODO: It would be better to verify explicitly which states we can recover from without the wal
 // and which ones we need the wal for - then we'd also be able to only flush the
 // wal writer when we need to, instead of with every message.
 
-func startNewStateAndWaitForBlock(t *testing.T, consensusReplayConfig *cfg.Config,
-	lastBlockHeight int64, blockDB dbm.DB, stateStore sm.Store,
-) {
+func startNewStateAndWaitForBlock(t *testing.T, consensusReplayConfig *cfg.Config, blockDB dbm.DB) {
 	logger := log.TestingLogger()
-	state, _ := stateStore.LoadFromDBOrGenesisFile(consensusReplayConfig.GenesisFile())
+	state, err := sm.MakeGenesisStateFromFile(consensusReplayConfig.GenesisFile())
+	require.NoError(t, err)
 	privValidator := loadPrivValidator(consensusReplayConfig)
 	cs := newStateWithConfigAndBlockStore(
 		consensusReplayConfig,
@@ -81,7 +80,7 @@ func startNewStateAndWaitForBlock(t *testing.T, consensusReplayConfig *cfg.Confi
 	bytes, _ := os.ReadFile(cs.config.WalFile())
 	t.Logf("====== WAL: \n\r%X\n", bytes)
 
-	err := cs.Start()
+	err = cs.Start()
 	require.NoError(t, err)
 	defer func() {
 		if err := cs.Stop(); err != nil {
@@ -164,9 +163,6 @@ LOOP:
 		logger := log.NewNopLogger()
 		blockDB := dbm.NewMemDB()
 		stateDB := blockDB
-		stateStore := sm.NewStore(stateDB, sm.StoreOptions{
-			DiscardABCIResponses: false,
-		})
 		state, err := sm.MakeGenesisStateFromFile(consensusReplayConfig.GenesisFile())
 		require.NoError(t, err)
 		privValidator := loadPrivValidator(consensusReplayConfig)
@@ -207,7 +203,7 @@ LOOP:
 			t.Logf("WAL panicked: %v", err)
 
 			// make sure we can make blocks after a crash
-			startNewStateAndWaitForBlock(t, consensusReplayConfig, cs.Height, blockDB, stateStore)
+			startNewStateAndWaitForBlock(t, consensusReplayConfig, blockDB)
 
 			// stop consensus state and transactions sender (initFn)
 			cs.Stop() //nolint:errcheck // Logging this error causes failure
@@ -318,7 +314,7 @@ var (
 	sim testSim
 )
 
-//---------------------------------------
+// ---------------------------------------
 // Test handshake/replay
 
 // 0 - all synced up
@@ -1030,7 +1026,7 @@ func (app *badApp) Commit() abci.ResponseCommit {
 	panic("either allHashesAreWrong or onlyLastHashIsWrong must be set")
 }
 
-//--------------------------
+// --------------------------
 // utils for making blocks
 
 func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) {
@@ -1176,7 +1172,7 @@ func stateAndStore(
 	return stateDB, state, store
 }
 
-//----------------------------------
+// ----------------------------------
 // mock block store
 
 type mockBlockStore struct {
@@ -1231,7 +1227,7 @@ func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) {
 	return pruned, nil
 }
 
-//---------------------------------------
+// ---------------------------------------
 // Test handshake/init chain
 
 func TestHandshakeUpdatesValidators(t *testing.T) {

diff --git a/crypto/tmhash/hash.go b/crypto/tmhash/hash.go
@@ -2,7 +2,10 @@ package tmhash
 
 import (
 	"crypto/sha256"
+	"errors"
+	"fmt"
 	"hash"
+	"regexp"
 )
 
 const (
@@ -21,7 +24,7 @@ func Sum(bz []byte) []byte {
 	return h[:]
 }
 
-//-------------------------------------------------------------
+// -------------------------------------------------------------
 
 const (
 	TruncatedSize = 20
@@ -63,3 +66,27 @@ func SumTruncated(bz []byte) []byte {
 	hash := sha256.Sum256(bz)
 	return hash[:TruncatedSize]
 }
+
+// ValidateSHA256 checks if the given string is a syntactically valid SHA256 hash.
+// A valid SHA256 hash is a hex-encoded 64-character string.
+// If the hash isn't valid, it returns an error explaining why.
+func ValidateSHA256(hashStr string) error {
+	const sha256Pattern = `^[a-fA-F0-9]{64}$`
+
+	if len(hashStr) != 64 {
+		return fmt.Errorf("expected 64 characters, but have %d", len(hashStr))
+	}
+
+	match, err := regexp.MatchString(sha256Pattern, hashStr)
+	if err != nil {
+		// if this happens, there is a bug in the regex or some internal regexp
+		// package error.
+		return fmt.Errorf("can't run regex %q: %s", sha256Pattern, err)
+	}
+
+	if !match {
+		return errors.New("contains non-hexadecimal characters")
+	}
+
+	return nil
+}