Make new-state-mgmt canonical

beacon-chain/blockchain/chain_info_norace_test.go

@@ -5,7 +5,9 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 )
 
 func TestHeadSlot_DataRace(t *testing.T) {
@@ -29,6 +31,7 @@ func TestHeadRoot_DataRace(t *testing.T) {
 	s := &Service{
 		beaconDB: db,
 		head:     &head{root: [32]byte{'A'}},
+		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
 	}
 	go func() {
 		s.saveHead(
@@ -47,6 +50,7 @@ func TestHeadBlock_DataRace(t *testing.T) {
 	s := &Service{
 		beaconDB: db,
 		head:     &head{block: &ethpb.SignedBeaconBlock{}},
+		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
 	}
 	go func() {
 		s.saveHead(
@@ -62,6 +66,7 @@ func TestHeadState_DataRace(t *testing.T) {
 	defer testDB.TeardownDB(t, db)
 	s := &Service{
 		beaconDB: db,
+		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
 	}
 	go func() {
 		s.saveHead(

beacon-chain/blockchain/head.go

@@ -59,8 +59,8 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 
 	// If the head state is not available, just return nil.
 	// There's nothing to cache
-	if featureconfig.Get().NewStateMgmt {
-		if !s.stateGen.StateSummaryExists(ctx, headRoot) {
+	if !featureconfig.Get().DisableNewStateMgmt {
+		if !s.stateGen.StateSummaryExists(ctx, headRoot) && !s.beaconDB.HasState(ctx, headRoot) {
 			return nil
 		}
 	} else {
@@ -81,7 +81,7 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 
 	// Get the new head state from cached state or DB.
 	var newHeadState *state.BeaconState
-	if featureconfig.Get().NewStateMgmt {
+	if !featureconfig.Get().DisableNewStateMgmt {
 		newHeadState, err = s.stateGen.StateByRoot(ctx, headRoot)
 		if err != nil {
 			return errors.Wrap(err, "could not retrieve head state in DB")
@@ -121,7 +121,7 @@ func (s *Service) saveHeadNoDB(ctx context.Context, b *ethpb.SignedBeaconBlock,
 
 	var headState *state.BeaconState
 	var err error
-	if featureconfig.Get().NewStateMgmt {
+	if !featureconfig.Get().DisableNewStateMgmt {
 		headState, err = s.stateGen.StateByRoot(ctx, r)
 		if err != nil {
 			return errors.Wrap(err, "could not retrieve head state in DB")

beacon-chain/blockchain/process_attestation_helpers.go

@@ -32,7 +32,7 @@ func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*sta
 	}
 
 	var baseState *stateTrie.BeaconState
-	if featureconfig.Get().NewStateMgmt {
+	if !featureconfig.Get().DisableNewStateMgmt {
 		baseState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(c.Root))
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
@@ -128,7 +128,7 @@ func (s *Service) verifyAttestation(ctx context.Context, baseState *stateTrie.Be
 			// different seeds.
 			var aState *stateTrie.BeaconState
 			var err error
-			if featureconfig.Get().NewStateMgmt {
+			if !featureconfig.Get().DisableNewStateMgmt {
 				aState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
 				return nil, err
 			}

beacon-chain/blockchain/process_attestation_test.go

@@ -8,12 +8,14 @@ import (
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -25,7 +27,11 @@ func TestStore_OnAttestation(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	cfg := &Config{
+		BeaconDB:        db,
+		ForkChoiceStore: protoarray.New(0, 0, [32]byte{}),
+		StateGen:        stategen.New(db, cache.NewStateSummaryCache()),
+	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
 		t.Fatal(err)
@@ -99,7 +105,7 @@ func TestStore_OnAttestation(t *testing.T) {
 			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: BlkWithOutStateRoot[:]}}},
 			s:             &pb.BeaconState{},
 			wantErr:       true,
-			wantErrString: "pre state of target block 0 does not exist",
+			wantErrString: "could not get pre state for slot 0: unknown boundary state",
 		},
 		{
 			name: "process attestation doesn't match current epoch",
@@ -131,7 +137,10 @@ func TestStore_SaveCheckpointState(t *testing.T) {
 	defer testDB.TeardownDB(t, db)
 	params.UseDemoBeaconConfig()
 
-	cfg := &Config{BeaconDB: db}
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
 		t.Fatal(err)
@@ -155,13 +164,18 @@ func TestStore_SaveCheckpointState(t *testing.T) {
 	if err := service.beaconDB.SaveState(ctx, s, r); err != nil {
 		t.Fatal(err)
 	}
+
 	service.justifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 
+	r = bytesutil.ToBytes32([]byte{'A'})
 	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'A'}, 32)}
 	service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'A'}))
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'A'}, 32)}); err != nil {
+		t.Fatal(err)
+	}
 	s1, err := service.getAttPreState(ctx, cp1)
 	if err != nil {
 		t.Fatal(err)
@@ -172,6 +186,9 @@ func TestStore_SaveCheckpointState(t *testing.T) {
 
 	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: bytesutil.PadTo([]byte{'B'}, 32)}
 	service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'B'}))
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'B'}, 32)}); err != nil {
+		t.Fatal(err)
+	}
 	s2, err := service.getAttPreState(ctx, cp2)
 	if err != nil {
 		t.Fatal(err)
@@ -211,6 +228,9 @@ func TestStore_SaveCheckpointState(t *testing.T) {
 	service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	cp3 := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'C'}, 32)}
 	service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'C'}))
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'C'}, 32)}); err != nil {
+		t.Fatal(err)
+	}
 	s3, err := service.getAttPreState(ctx, cp3)
 	if err != nil {
 		t.Fatal(err)
@@ -225,7 +245,10 @@ func TestStore_UpdateCheckpointState(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	cfg := &Config{BeaconDB: db}
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
 		t.Fatal(err)

beacon-chain/blockchain/process_block.go

@@ -93,7 +93,7 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock)
 		return nil, errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
 	}
 
-	if featureconfig.Get().NewStateMgmt {
+	if !featureconfig.Get().DisableNewStateMgmt {
 		if err := s.stateGen.SaveState(ctx, root, postState); err != nil {
 			return nil, errors.Wrap(err, "could not save state")
 		}
@@ -116,7 +116,7 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock)
 			return nil, errors.Wrap(err, "could not save finalized checkpoint")
 		}
 
-		if !featureconfig.Get().NewStateMgmt {
+		if featureconfig.Get().DisableNewStateMgmt {
 			startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
 			endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
 			if endSlot > startSlot {
@@ -139,7 +139,7 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock)
 			return nil, errors.Wrap(err, "could not save new justified")
 		}
 
-		if featureconfig.Get().NewStateMgmt {
+		if !featureconfig.Get().DisableNewStateMgmt {
 			fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
 			fBlock, err := s.beaconDB.Block(ctx, fRoot)
 			if err != nil {
@@ -231,7 +231,7 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 		return errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
 	}
 
-	if featureconfig.Get().NewStateMgmt {
+	if !featureconfig.Get().DisableNewStateMgmt {
 		if err := s.stateGen.SaveState(ctx, root, postState); err != nil {
 			return errors.Wrap(err, "could not save state")
 		}
@@ -266,7 +266,7 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 
 	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
 	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
-		if !featureconfig.Get().NewStateMgmt {
+		if featureconfig.Get().DisableNewStateMgmt {
 			startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
 			endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
 			if endSlot > startSlot {
@@ -299,7 +299,7 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 			return errors.Wrap(err, "could not save new justified")
 		}
 
-		if featureconfig.Get().NewStateMgmt {
+		if !featureconfig.Get().DisableNewStateMgmt {
 			fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
 			fBlock, err := s.beaconDB.Block(ctx, fRoot)
 			if err != nil {
@@ -316,7 +316,7 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 		return errors.Wrap(err, "could not save new validators")
 	}
 
-	if !featureconfig.Get().NewStateMgmt {
+	if featureconfig.Get().DisableNewStateMgmt {
 		numOfStates := len(s.boundaryRoots)
 		if numOfStates > initialSyncCacheSize {
 			if err = s.persistCachedStates(ctx, numOfStates); err != nil {
@@ -341,7 +341,7 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 			return err
 		}
 
-		if !featureconfig.Get().NewStateMgmt && helpers.IsEpochStart(postState.Slot()) {
+		if featureconfig.Get().DisableNewStateMgmt && helpers.IsEpochStart(postState.Slot()) {
 			if err := s.beaconDB.SaveState(ctx, postState, root); err != nil {
 				return errors.Wrap(err, "could not save state")
 			}

beacon-chain/blockchain/process_block_helpers.go

@@ -61,8 +61,12 @@ func (s *Service) verifyBlkPreState(ctx context.Context, b *ethpb.BeaconBlock) (
 	ctx, span := trace.StartSpan(ctx, "chainService.verifyBlkPreState")
 	defer span.End()
 
-	if featureconfig.Get().NewStateMgmt {
-		preState, err := s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(b.ParentRoot))
+	if !featureconfig.Get().DisableNewStateMgmt {
+		parentRoot := bytesutil.ToBytes32(b.ParentRoot)
+		if !s.stateGen.StateSummaryExists(ctx, parentRoot) {
+			return nil, errors.New("provided block root does not have block saved in the db")
+		}
+		preState, err := s.stateGen.StateByRoot(ctx, parentRoot)
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
 		}
@@ -288,7 +292,7 @@ func (s *Service) updateJustified(ctx context.Context, state *stateTrie.BeaconSt
 		s.justifiedCheckpt = cpt
 	}
 
-	if !featureconfig.Get().NewStateMgmt {
+	if featureconfig.Get().DisableNewStateMgmt {
 		justifiedRoot := bytesutil.ToBytes32(cpt.Root)
 
 		justifiedState := s.initSyncState[justifiedRoot]

beacon-chain/blockchain/process_block_test.go

@@ -9,12 +9,14 @@ import (
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -26,7 +28,10 @@ func TestStore_OnBlock(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	cfg := &Config{BeaconDB: db}
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
 		t.Fatal(err)
@@ -60,10 +65,16 @@ func TestStore_OnBlock(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: st.Slot(), Root: randomParentRoot[:]}); err != nil {
+		t.Fatal(err)
+	}
 	if err := service.beaconDB.SaveState(ctx, st.Copy(), randomParentRoot); err != nil {
 		t.Fatal(err)
 	}
 	randomParentRoot2 := roots[1]
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: st.Slot(), Root: randomParentRoot2[:]}); err != nil {
+		t.Fatal(err)
+	}
 	if err := service.beaconDB.SaveState(ctx, st.Copy(), bytesutil.ToBytes32(randomParentRoot2)); err != nil {
 		t.Fatal(err)
 	}
@@ -307,12 +318,15 @@ func TestShouldUpdateJustified_ReturnFalse(t *testing.T) {
 	}
 }
 
-func TestCachedPreState_CanGetFromCache(t *testing.T) {
+func TestCachedPreState_CanGetFromStateSummary(t *testing.T) {
 	ctx := context.Background()
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	cfg := &Config{BeaconDB: db}
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
 		t.Fatal(err)
@@ -321,7 +335,8 @@ func TestCachedPreState_CanGetFromCache(t *testing.T) {
 	s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: 1, GenesisValidatorsRoot: params.BeaconConfig().ZeroHash[:]})
 	r := [32]byte{'A'}
 	b := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r[:]}
-	service.initSyncState[r] = s
+	service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: 1, Root: r[:]})
+	service.stateGen.SaveState(ctx, r, s)
 
 	received, err := service.verifyBlkPreState(ctx, b)
 	if err != nil {
@@ -337,7 +352,10 @@ func TestCachedPreState_CanGetFromDB(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	cfg := &Config{BeaconDB: db}
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
 		t.Fatal(err)
@@ -348,19 +366,20 @@ func TestCachedPreState_CanGetFromDB(t *testing.T) {
 
 	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	_, err = service.verifyBlkPreState(ctx, b)
-	wanted := "pre state of slot 1 does not exist"
+	wanted := "provided block root does not have block saved in the db"
 	if err.Error() != wanted {
 		t.Error("Did not get wanted error")
 	}
 
 	s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: 1})
-	service.beaconDB.SaveState(ctx, s, r)
+	service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: 1, Root: r[:]})
+	service.stateGen.SaveState(ctx, r, s)
 
 	received, err := service.verifyBlkPreState(ctx, b)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(s, received) {
+	if s.Slot() != received.Slot() {
 		t.Error("cached state not the same")
 	}
 }

beacon-chain/blockchain/receive_attestation.go

@@ -91,7 +91,7 @@ func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
 			atts := s.attPool.ForkchoiceAttestations()
 			for _, a := range atts {
 				var hasState bool
-				if featureconfig.Get().NewStateMgmt {
+				if !featureconfig.Get().DisableNewStateMgmt {
 					hasState = s.stateGen.StateSummaryExists(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
 				} else {
 					hasState = s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot)) && s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.Target.Root))