Implement Final Updates 0.6 (#2562)

* ValidatorStatus Estimating Activation RPC Server (#2469)

* fix spacing

* working on position in queue

* fmt

* spacing

* feedback

* tests

* rename

* Only Perform Initial Sync With a Single Peer (#2471)

* fix spacing

* use send instead of broadcast in initial sync

* Fix Estimation of Deposit Inclusion Slot in ValidatorActivationStatus (#2472)

* fix spacing

* fix time estimates

* correct slot estimation

* naming

* Update beacon-chain/rpc/validator_server.go

Co-Authored-By: rauljordan <raul@prysmaticlabs.com>

* SSZ web api for decoding input data (#2473)

* first pass ssz server for decoding deposit input data

* fix decoding

* revert viz change on helper

* add image target

* use /api prefix, add deployment for cluster

* fix lint

* standardize slot numbers (#2475)

* Add CORS for ssz api (#2476)

* first pass ssz server for decoding deposit input data

* fix decoding

* revert viz change on helper

* add image target

* use /api prefix, add deployment for cluster

* fix lint

* needed CORS

* Allow Client to Retrieve Multiple Validator Statuses (#2474)

* multiple validator statuses

* gazelle

* context

* fixing bugs

* remove old way of checking

* fix logging

* make activation queue more accurate

* fix rpc test

* add test

* fix remaining tests

* lint

* comment

* review comments

* Update Prysm README (#2477)

* README updated

* readme updates

* no err throw (#2479)

* Fix Status Nil Pointer Error (#2480)

* no err throw

* nil errors

* 3.175 (#2482)

* Better Error Message if Failing to Exit Initial Sync (#2483)

* no err throw

* nil errors

* better error on init sync

* Only Log Active Balances (#2485)

* only log active balance

* dont need ()

* change logging (#2487)

* fix chainstart waiting on rpc server (#2488)

* shift ticker to after activation (#2489)

* Add drain script (#2418)

* Add drain script

* Fix script to drain contracts from newest to oldest

* Add README

* remove comments

* Only after block 400k, look up by deposit event

* issue warn log on disconnecting peer instead of error (#2491)

* Display Only Active Validator Data (#2490)

* Fix Validator Status Field in RPC Server (#2492)

* fix status of key

* status test fix

* fmt

* Estimate the Time Till Follow Distance Is Completed (#2486)

* use estimation instead

* fix test

* fixing another test

* fix tests and preston's comments

* remove unused var

* fix condition

* Revert "fix condition"

This reverts commit dee0e31.

* dont return error

* add production config for testnet release (#2493)

* Lookup Validator Index in State in Status Check (#2494)

* state lookup

* refactor duplicate code

* refactor with mapping

* fix broken tests

* finish refactor

* merged master

* updated EpochCommitteeCount and fixed tests

* implemented ShardDelta

* test for ShardDelta

* implemented EpochStartShard

* added epoch out of bound test

* test for accurate start shard

* lint

* implemented process_final_updates

* move to the top of the file

* add comment back

* gaz

* Test for process final updates

* fixed tests

* fixed all the tests

beacon-chain/core/epoch/BUILD.bazel

@@ -18,6 +18,7 @@ go_library(
         "//shared/mathutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/sliceutil:go_default_library",
+        "//shared/ssz:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
     ],

beacon-chain/core/epoch/epoch_operations_test.go

@@ -16,19 +16,35 @@ func buildState(slot uint64, validatorCount uint64) *pb.BeaconState {
 	validators := make([]*pb.Validator, validatorCount)
 	for i := 0; i < len(validators); i++ {
 		validators[i] = &pb.Validator{
-			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			ExitEpoch:        params.BeaconConfig().FarFutureEpoch,
+			EffectiveBalance: params.BeaconConfig().MaxDepositAmount,
 		}
 	}
 	validatorBalances := make([]uint64, len(validators))
 	for i := 0; i < len(validatorBalances); i++ {
 		validatorBalances[i] = params.BeaconConfig().MaxDepositAmount
 	}
+	latestActiveIndexRoots := make(
+		[][]byte,
+		params.BeaconConfig().LatestActiveIndexRootsLength,
+	)
+	for i := 0; i < len(latestActiveIndexRoots); i++ {
+		latestActiveIndexRoots[i] = params.BeaconConfig().ZeroHash[:]
+	}
+	latestRandaoMixes := make(
+		[][]byte,
+		params.BeaconConfig().LatestRandaoMixesLength,
+	)
+	for i := 0; i < len(latestRandaoMixes); i++ {
+		latestRandaoMixes[i] = params.BeaconConfig().ZeroHash[:]
+	}
 	return &pb.BeaconState{
 		Slot:                   slot,
 		Balances:               validatorBalances,
 		ValidatorRegistry:      validators,
 		LatestRandaoMixes:      make([][]byte, params.BeaconConfig().LatestRandaoMixesLength),
 		LatestActiveIndexRoots: make([][]byte, params.BeaconConfig().LatestActiveIndexRootsLength),
+		LatestSlashedBalances:  make([]uint64, params.BeaconConfig().LatestSlashedExitLength),
 	}
 }
 
@@ -285,3 +301,55 @@ func TestInclusionSlot_SlotNotFound(t *testing.T) {
 		t.Errorf("Expected %s, received %v", want, err)
 	}
 }
+
+func TestProcessFinalUpdates_CanProcess(t *testing.T) {
+	s := buildState(params.BeaconConfig().SlotsPerHistoricalRoot-1, params.BeaconConfig().SlotsPerEpoch)
+	ce := helpers.CurrentEpoch(s)
+	ne := ce + 1
+	s.Eth1DataVotes = []*pb.Eth1DataVote{{VoteCount: 100}}
+	s.Balances[0] = 29 * 1e9
+	s.LatestSlashedBalances[ce] = 100
+	s.LatestRandaoMixes[ce] = []byte{'A'}
+	newS, err := ProcessFinalUpdates(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify we can reset ETH1 data votes.
+	if len(newS.Eth1DataVotes) != 0 {
+		t.Errorf("Eth1 data votes didnt get reset, got %d", len(newS.Eth1DataVotes))
+	}
+
+	// Verify effective balance is correctly updated.
+	if newS.ValidatorRegistry[0].EffectiveBalance != 29*1e9 {
+		t.Errorf("effective balance incorrectly updated, got %d", s.ValidatorRegistry[0].EffectiveBalance)
+	}
+
+	// Verify start shard is correctly updated.
+	if newS.LatestStartShard != 64 {
+		t.Errorf("start shard incorrectly updated, got %d", 64)
+	}
+
+	// Verify latest active index root is correctly updated in the right position.
+	pos := (ne + params.BeaconConfig().ActivationExitDelay) % params.BeaconConfig().LatestActiveIndexRootsLength
+	if bytes.Equal(newS.LatestActiveIndexRoots[pos], params.BeaconConfig().ZeroHash[:]) {
+		t.Error("latest active index roots still zero hashes")
+	}
+
+	// Verify slashed balances correctly updated.
+	if newS.LatestSlashedBalances[ce] != newS.LatestSlashedBalances[ne] {
+		t.Errorf("wanted slashed balance %d, got %d",
+			newS.LatestSlashedBalances[ce],
+			newS.LatestSlashedBalances[ne])
+	}
+
+	// Verify randao is correctly updated in the right position.
+	if bytes.Equal(newS.LatestRandaoMixes[ne], params.BeaconConfig().ZeroHash[:]) {
+		t.Error("latest RANDAO still zero hashes")
+	}
+
+	// Verify historical root accumulator was appended.
+	if len(newS.HistoricalRoots) != 1 {
+		t.Errorf("wanted slashed balance %d, got %d", 1, len(newS.HistoricalRoots[ce]))
+	}
+}

beacon-chain/core/epoch/epoch_processing.go

@@ -19,6 +19,7 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/mathutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/sliceutil"
+	"github.com/prysmaticlabs/prysm/shared/ssz"
 	"github.com/sirupsen/logrus"
 )
 
@@ -369,6 +370,112 @@ func ProcessJustificationFinalization(state *pb.BeaconState, prevAttestedBal uin
 	return state, nil
 }
 
+// ProcessFinalUpdates processes the final updates during epoch processing.
+//
+// Spec pseudocode definition:
+//	def process_final_updates(state: BeaconState) -> None:
+//    current_epoch = get_current_epoch(state)
+//    next_epoch = current_epoch + 1
+//    # Reset eth1 data votes
+//    if (state.slot + 1) % SLOTS_PER_ETH1_VOTING_PERIOD == 0:
+//        state.eth1_data_votes = []
+//    # Update effective balances with hysteresis
+//    for index, validator in enumerate(state.validator_registry):
+//        balance = state.balances[index]
+//        HALF_INCREMENT = EFFECTIVE_BALANCE_INCREMENT // 2
+//        if balance < validator.effective_balance or validator.effective_balance + 3 * HALF_INCREMENT < balance:
+//            validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE)
+//    # Update start shard
+//    state.latest_start_shard = (state.latest_start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT
+//    # Set active index root
+//    index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % LATEST_ACTIVE_INDEX_ROOTS_LENGTH
+//    state.latest_active_index_roots[index_root_position] = hash_tree_root(
+//        get_active_validator_indices(state, next_epoch + ACTIVATION_EXIT_DELAY)
+//    )
+//    # Set total slashed balances
+//    state.latest_slashed_balances[next_epoch % LATEST_SLASHED_EXIT_LENGTH] = (
+//        state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH]
+//    )
+//    # Set randao mix
+//    state.latest_randao_mixes[next_epoch % LATEST_RANDAO_MIXES_LENGTH] = get_randao_mix(state, current_epoch)
+//    # Set historical root accumulator
+//    if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0:
+//        historical_batch = HistoricalBatch(
+//            block_roots=state.latest_block_roots,
+//            state_roots=state.latest_state_roots,
+//        )
+//        state.historical_roots.append(hash_tree_root(historical_batch))
+//    # Rotate current/previous epoch attestations
+//    state.previous_epoch_attestations = state.current_epoch_attestations
+//    state.current_epoch_attestations = []
+func ProcessFinalUpdates(state *pb.BeaconState) (*pb.BeaconState, error) {
+	currentEpoch := helpers.CurrentEpoch(state)
+	nextEpoch := currentEpoch + 1
+
+	// Reset ETH1 data votes.
+	if (state.Slot+1)%params.BeaconConfig().SlotsPerHistoricalRoot == 0 {
+		state.Eth1DataVotes = nil
+	}
+
+	// Update effective balances with hysteresis.
+	for i, v := range state.ValidatorRegistry {
+		balance := state.Balances[i]
+		halfInc := params.BeaconConfig().EffectiveBalanceIncrement / 2
+		if balance < v.EffectiveBalance || v.EffectiveBalance+3*halfInc < balance {
+			v.EffectiveBalance = params.BeaconConfig().MaxEffectiveBalance
+			if v.EffectiveBalance > balance-balance%params.BeaconConfig().EffectiveBalanceIncrement {
+				v.EffectiveBalance = balance - balance%params.BeaconConfig().EffectiveBalanceIncrement
+			}
+		}
+	}
+
+	// Update start shard.
+	state.LatestStartShard = (state.LatestStartShard + helpers.ShardDelta(state, currentEpoch)) %
+		params.BeaconConfig().ShardCount
+
+	// Set active index root.
+	activationDelay := params.BeaconConfig().ActivationExitDelay
+	idxRootPosition := (nextEpoch + activationDelay) % params.BeaconConfig().LatestActiveIndexRootsLength
+	idxRoot, err := ssz.TreeHash(helpers.ActiveValidatorIndices(state, nextEpoch+activationDelay))
+	if err != nil {
+		return nil, fmt.Errorf("could not tree hash active indices: %v", err)
+	}
+	state.LatestActiveIndexRoots[idxRootPosition] = idxRoot[:]
+
+	// Set total slashed balances.
+	slashedExitLength := params.BeaconConfig().LatestSlashedExitLength
+	state.LatestSlashedBalances[nextEpoch%slashedExitLength] =
+		state.LatestSlashedBalances[currentEpoch%slashedExitLength]
+
+	// Set RANDAO mix.
+	randaoMixLength := params.BeaconConfig().LatestRandaoMixesLength
+	mix, err := helpers.RandaoMix(state, currentEpoch)
+	if err != nil {
+		return nil, fmt.Errorf("could not get randao mix: %v", err)
+	}
+	state.LatestRandaoMixes[nextEpoch%randaoMixLength] = mix
+
+	// Set historical root accumulator.
+	epochsPerHistoricalRoot := params.BeaconConfig().SlotsPerHistoricalRoot / params.BeaconConfig().SlotsPerEpoch
+	if nextEpoch%epochsPerHistoricalRoot == 0 {
+		historicalBatch := &pb.HistoricalBatch{
+			BlockRoots: state.LatestBlockRoots,
+			StateRoots: state.LatestStateRoots,
+		}
+		batchRoot, err := hashutil.HashProto(historicalBatch)
+		if err != nil {
+			return nil, fmt.Errorf("could not hash historical batch: %v", err)
+		}
+		state.HistoricalRoots = append(state.HistoricalRoots, batchRoot[:])
+	}
+
+	// Rotate current and previous epoch attestations.
+	state.PreviousEpochAttestations = state.CurrentEpochAttestations
+	state.CurrentEpochAttestations = nil
+
+	return state, nil
+}
+
 // UpdateLatestSlashedBalances updates the latest slashed balances. It transfers
 // the amount from the current epoch index to next epoch index.
 //

beacon-chain/core/helpers/committee.go

@@ -43,7 +43,6 @@ type CrosslinkCommittee struct {
 //        )
 //    ) * SLOTS_PER_EPOCH
 func EpochCommitteeCount(state *pb.BeaconState, epoch uint64) uint64 {
-
 	minCommitteePerSlot := uint64(1)
 	activeIndices := ActiveValidatorIndices(state, epoch)
 	// Max committee count per slot will be 0 when shard count is less than epoch length, this