Release custom state ssz (#4421)

* Release custom state ssz, change all HTR of beacon state to use custom method

* typo

* use mainnet config

Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

beacon-chain/blockchain/forkchoice/BUILD.bazel

@@ -25,6 +25,7 @@ go_library(
         "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/params:go_default_library",
+        "//shared/stateutil:go_default_library",
         "//shared/traceutil:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
@@ -61,6 +62,7 @@ go_test(
         "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/params:go_default_library",
+        "//shared/stateutil:go_default_library",
         "//shared/testutil:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_prysmaticlabs_go_bitfield//:go_default_library",

beacon-chain/blockchain/forkchoice/lmd_ghost_yaml_test.go

@@ -14,6 +14,7 @@ import (
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"gopkg.in/yaml.v2"
 )
 
@@ -38,6 +39,8 @@ func TestGetHeadFromYaml(t *testing.T) {
 	var c *Config
 	err = yaml.Unmarshal(yamlFile, &c)
 
+	params.UseMainnetConfig()
+
 	for _, test := range c.TestCases {
 		db := testDB.SetupDB(t)
 		defer testDB.TeardownDB(t, db)

beacon-chain/blockchain/forkchoice/process_block_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/stateutil"
 )
 
 func TestStore_OnBlock(t *testing.T) {
@@ -27,7 +28,7 @@ func TestStore_OnBlock(t *testing.T) {
 
 	store := NewForkChoiceService(ctx, db)
 
-	genesisStateRoot, err := ssz.HashTreeRoot(&pb.BeaconState{})
+	genesisStateRoot, err := stateutil.HashTreeRootState(&pb.BeaconState{})
 	if err != nil {
 		t.Error(err)
 	}

beacon-chain/blockchain/forkchoice/service.go

@@ -19,6 +19,7 @@ import (
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/stateutil"
 	"go.opencensus.io/trace"
 )
 
@@ -124,7 +125,7 @@ func (s *Store) cacheGenesisState(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	stateRoot, err := ssz.HashTreeRoot(genesisState)
+	stateRoot, err := stateutil.HashTreeRootState(genesisState)
 	if err != nil {
 		return errors.Wrap(err, "could not tree hash genesis state")
 	}

beacon-chain/blockchain/forkchoice/service_test.go

@@ -16,6 +16,7 @@ import (
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/stateutil"
 )
 
 func TestStore_GenesisStoreOk(t *testing.T) {
@@ -27,7 +28,7 @@ func TestStore_GenesisStoreOk(t *testing.T) {
 
 	genesisTime := time.Unix(9999, 0)
 	genesisState := &pb.BeaconState{GenesisTime: uint64(genesisTime.Unix())}
-	genesisStateRoot, err := ssz.HashTreeRoot(genesisState)
+	genesisStateRoot, err := stateutil.HashTreeRootState(genesisState)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -39,6 +40,9 @@ func TestStore_GenesisStoreOk(t *testing.T) {
 	if err := db.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
 		t.Fatal(err)
 	}
+	if err := db.SaveGenesisBlockRoot(ctx, genesisBlkRoot); err != nil {
+		t.Fatal(err)
+	}
 
 	checkPoint := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
 	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
@@ -150,7 +154,7 @@ func TestStore_LatestAttestingBalance(t *testing.T) {
 	}
 
 	s := &pb.BeaconState{Validators: validators}
-	stateRoot, err := ssz.HashTreeRoot(s)
+	stateRoot, err := stateutil.HashTreeRootState(s)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -162,6 +166,9 @@ func TestStore_LatestAttestingBalance(t *testing.T) {
 	if err := db.SaveState(ctx, s, blkRoot); err != nil {
 		t.Fatal(err)
 	}
+	if err := db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
+		t.Fatal(err)
+	}
 
 	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
 	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
@@ -253,7 +260,7 @@ func TestStore_GetHead(t *testing.T) {
 	}
 
 	s := &pb.BeaconState{Validators: validators}
-	stateRoot, err := ssz.HashTreeRoot(s)
+	stateRoot, err := stateutil.HashTreeRootState(s)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -265,6 +272,9 @@ func TestStore_GetHead(t *testing.T) {
 	if err := store.db.SaveState(ctx, s, blkRoot); err != nil {
 		t.Fatal(err)
 	}
+	if err := store.db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
+		t.Fatal(err)
+	}
 
 	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
 
@@ -373,7 +383,7 @@ func TestStore_GetFilterBlockTree_CorrectLeaf(t *testing.T) {
 	}
 
 	s := &pb.BeaconState{}
-	stateRoot, err := ssz.HashTreeRoot(s)
+	stateRoot, err := stateutil.HashTreeRootState(s)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -385,6 +395,9 @@ func TestStore_GetFilterBlockTree_CorrectLeaf(t *testing.T) {
 	if err := store.db.SaveState(ctx, s, blkRoot); err != nil {
 		t.Fatal(err)
 	}
+	if err := store.db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
+		t.Fatal(err)
+	}
 
 	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
 
@@ -433,7 +446,7 @@ func TestStore_GetFilterBlockTree_IncorrectLeaf(t *testing.T) {
 	}
 
 	s := &pb.BeaconState{}
-	stateRoot, err := ssz.HashTreeRoot(s)
+	stateRoot, err := stateutil.HashTreeRootState(s)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -445,6 +458,9 @@ func TestStore_GetFilterBlockTree_IncorrectLeaf(t *testing.T) {
 	if err := store.db.SaveState(ctx, s, blkRoot); err != nil {
 		t.Fatal(err)
 	}
+	if err := store.db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
+		t.Fatal(err)
+	}
 
 	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
 

beacon-chain/blockchain/receive_block_test.go

@@ -13,6 +13,7 @@ import (
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/stateutil"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 	logTest "github.com/sirupsen/logrus/hooks/test"
 )
@@ -154,7 +155,7 @@ func TestReceiveBlockNoPubsubForkchoice_ProcessCorrectly(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	stateRoot, err := ssz.HashTreeRoot(beaconState)
+	stateRoot, err := stateutil.HashTreeRootState(beaconState)
 	if err != nil {
 		t.Fatal(err)
 	}

beacon-chain/core/state/transition.go

@@ -70,17 +70,9 @@ func ExecuteStateTransition(
 	interop.WriteBlockToDisk(signed, false)
 	interop.WriteStateToDisk(state)
 
-	var postStateRoot [32]byte
-	if featureconfig.Get().EnableCustomStateSSZ {
-		postStateRoot, err = stateutil.HashTreeRootState(state)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not tree hash processed state")
-		}
-	} else {
-		postStateRoot, err = ssz.HashTreeRoot(state)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not tree hash processed state")
-		}
+	postStateRoot, err := stateutil.HashTreeRootState(state)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not tree hash processed state")
 	}
 	if !bytes.Equal(postStateRoot[:], signed.Block.StateRoot) {
 		return state, fmt.Errorf("validate state root failed, wanted: %#x, received: %#x",
@@ -184,10 +176,7 @@ func CalculateStateRoot(
 		return [32]byte{}, errors.Wrap(err, "could not process block")
 	}
 
-	if featureconfig.Get().EnableCustomStateSSZ {
-		return stateutil.HashTreeRootState(stateCopy)
-	}
-	return ssz.HashTreeRoot(stateCopy)
+	return stateutil.HashTreeRootState(stateCopy)
 }
 
 // ProcessSlot happens every slot and focuses on the slot counter and block roots record updates.
@@ -211,20 +200,10 @@ func ProcessSlot(ctx context.Context, state *pb.BeaconState) (*pb.BeaconState, e
 	defer span.End()
 	span.AddAttributes(trace.Int64Attribute("slot", int64(state.Slot)))
 
-	var prevStateRoot [32]byte
-	var err error
-	if featureconfig.Get().EnableCustomStateSSZ {
-		prevStateRoot, err = stateutil.HashTreeRootState(state)
-		if err != nil {
-			traceutil.AnnotateError(span, err)
-			return nil, errors.Wrap(err, "could not tree hash prev state root")
-		}
-	} else {
-		prevStateRoot, err = ssz.HashTreeRoot(state)
-		if err != nil {
-			traceutil.AnnotateError(span, err)
-			return nil, errors.Wrap(err, "could not tree hash prev state root")
-		}
+	prevStateRoot, err := stateutil.HashTreeRootState(state)
+	if err != nil {
+		traceutil.AnnotateError(span, err)
+		return nil, errors.Wrap(err, "could not tree hash prev state root")
 	}
 	state.StateRoots[state.Slot%params.BeaconConfig().SlotsPerHistoricalRoot] = prevStateRoot[:]
 

beacon-chain/interop-cold-start/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//shared:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/interop:go_default_library",
+        "//shared/stateutil:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_prysmaticlabs_go_ssz//:go_default_library",

beacon-chain/interop-cold-start/service.go

@@ -16,6 +16,7 @@ import (
 	"github.com/prysmaticlabs/prysm/shared"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/interop"
+	"github.com/prysmaticlabs/prysm/shared/stateutil"
 )
 
 var _ = shared.Service(&Service{})
@@ -139,7 +140,7 @@ func (s *Service) DepositsNumberAndRootAtHeight(ctx context.Context, blockHeight
 
 func (s *Service) saveGenesisState(ctx context.Context, genesisState *pb.BeaconState) error {
 	s.chainStartDeposits = make([]*ethpb.Deposit, len(genesisState.Validators))
-	stateRoot, err := ssz.HashTreeRoot(genesisState)
+	stateRoot, err := stateutil.HashTreeRootState(genesisState)
 	if err != nil {
 		return errors.Wrap(err, "could not tree hash genesis state")
 	}

beacon-chain/rpc/validator/BUILD.bazel

@@ -86,6 +86,7 @@ go_test(
         "//shared/featureconfig:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",
+        "//shared/stateutil:go_default_library",
         "//shared/testutil:go_default_library",
         "//shared/trieutil:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",

beacon-chain/rpc/validator/proposer_test.go

@@ -3,6 +3,8 @@ package validator
 import (
 	"context"
 	dbpb "github.com/prysmaticlabs/prysm/proto/beacon/db"
+	"github.com/prysmaticlabs/prysm/shared/stateutil"
+
 	"math/big"
 	"strings"
 	"testing"
@@ -82,7 +84,7 @@ func TestComputeStateRoot_OK(t *testing.T) {
 
 	beaconState, privKeys := testutil.DeterministicGenesisState(t, 100)
 
-	stateRoot, err := ssz.HashTreeRoot(beaconState)
+	stateRoot, err := stateutil.HashTreeRootState(beaconState)
 	if err != nil {
 		t.Fatalf("Could not hash genesis state: %v", err)
 	}

shared/featureconfig/config.go

@@ -33,7 +33,6 @@ type Flags struct {
 	EnableBackupWebhook       bool   // EnableBackupWebhook to allow database backups to trigger from monitoring port /db/backup.
 	PruneEpochBoundaryStates  bool   // PruneEpochBoundaryStates prunes the epoch boundary state before last finalized check point.
 	EnableSnappyDBCompression bool   // EnableSnappyDBCompression in the database.
-	EnableCustomStateSSZ      bool   // EnableCustomStateSSZ in the the state transition function.
 	InitSyncCacheState        bool   // InitSyncCacheState caches state during initial sync.
 	KafkaBootstrapServers     string // KafkaBootstrapServers to find kafka servers to stream blocks, attestations, etc.
 	NewCommitteeAssignments   bool   // NewCommitteeAssignments uses the new committee assignments algorithm.
@@ -87,10 +86,6 @@ func ConfigureBeaconChain(ctx *cli.Context) {
 		log.Warn("Enabled unsafe eth1 data vote cache")
 		cfg.EnableEth1DataVoteCache = true
 	}
-	if ctx.GlobalBool(EnableCustomStateSSZ.Name) {
-		log.Warn("Enabled custom state ssz for the state transition function")
-		cfg.EnableCustomStateSSZ = true
-	}
 	if ctx.GlobalBool(initSyncVerifyEverythingFlag.Name) {
 		log.Warn("Initial syncing with verifying all block's content signatures.")
 		cfg.InitSyncNoVerify = false

shared/featureconfig/flags.go

@@ -29,11 +29,6 @@ var (
 		Name:  "enable-eth1-data-vote-cache",
 		Usage: "Enable unsafe cache mechanism. See https://github.com/prysmaticlabs/prysm/issues/3106",
 	}
-	// EnableCustomStateSSZ see https://github.com/prysmaticlabs/prysm/pull/4077.
-	EnableCustomStateSSZ = cli.BoolFlag{
-		Name:  "enable-custom-state-ssz",
-		Usage: "Enable custom hash_tree_root(state) for Prysm. See https://github.com/prysmaticlabs/prysm/issues/4077",
-	}
 	enableShuffledIndexCache = cli.BoolFlag{
 		Name:  "enable-shuffled-index-cache",
 		Usage: "Enable unsafe cache mechanism. See https://github.com/prysmaticlabs/prysm/issues/3106",
@@ -132,6 +127,12 @@ var (
 		Usage:  deprecatedUsage,
 		Hidden: true,
 	}
+
+	deprecatedEnableCustomStateSSZ = cli.BoolFlag{
+		Name:   "enable-custom-state-ssz",
+		Usage:  deprecatedUsage,
+		Hidden: true,
+	}
 	deprecatedEnableCommitteeCacheFlag = cli.BoolFlag{
 		Name:   "enable-committee-cache",
 		Usage:  deprecatedUsage,
@@ -154,6 +155,7 @@ var deprecatedFlags = []cli.Flag{
 	deprecatedEnablePruneBoundaryStateFlag,
 	deprecatedEnableActiveIndicesCacheFlag,
 	deprecatedEnableActiveCountCacheFlag,
+	deprecatedEnableCustomStateSSZ,
 	deprecatedEnableCommitteeCacheFlag,
 	deprecatedEnableBLSPubkeyCacheFlag,
 }
@@ -170,7 +172,6 @@ var BeaconChainFlags = append(deprecatedFlags, []cli.Flag{
 	writeSSZStateTransitionsFlag,
 	EnableAttestationCacheFlag,
 	EnableEth1DataVoteCacheFlag,
-	EnableCustomStateSSZ,
 	initSyncVerifyEverythingFlag,
 	initSyncCacheState,
 	NewCacheFlag,

shared/params/spectest/BUILD.bazel

@@ -6,10 +6,7 @@ go_library(
     srcs = ["config.go"],
     importpath = "github.com/prysmaticlabs/prysm/shared/params/spectest",
     visibility = ["//visibility:public"],
-    deps = [
-        "//shared/featureconfig:go_default_library",
-        "//shared/params:go_default_library",
-    ],
+    deps = ["//shared/params:go_default_library"],
 )
 
 go_test(

shared/params/spectest/config.go

@@ -4,15 +4,11 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 // SetConfig sets the global params for spec tests depending on the option chosen.
 func SetConfig(config string) error {
-	c := featureconfig.Get()
-	c.EnableCustomStateSSZ = true
-	featureconfig.Init(c)
 	switch config {
 	case "minimal":
 		newConfig := params.MinimalSpecConfig()

shared/stateutil/state_root.go

@@ -44,6 +44,9 @@ func HashTreeRootState(state *pb.BeaconState) ([32]byte, error) {
 }
 
 func (h *stateRootHasher) hashTreeRootState(state *pb.BeaconState) ([32]byte, error) {
+	if state == nil {
+		return [32]byte{}, errors.New("nil state")
+	}
 	// There are 20 fields in the beacon state.
 	fieldRoots := make([][]byte, 20)