Upgrade the Beacon API e2e evaluator (#13868)

* GET

* POST

* Revert "Auxiliary commit to revert individual files from 615feb1"

This reverts commit 55cf071c684019f3d6124179154c10b2277fda49.

* comment fix

* deepsource

config values

block protos

get_committee_indices

proto and ssz

attestation interface

Revert "Auxiliary commit to revert individual files from deadb21"

This reverts commit 32ad5009537bc5ec0e6caf9f52143d380d00be85.

todos

get_attesting_indices

Revert "Auxiliary commit to revert individual files from dd27897"

This reverts commit f39644ed3cb6f3964fc6c86fdf4bd5de2a9668c8.

beacon spec changes

Fix pending attestation. Build ok

beacon-chain/blockchain/head.go

@@ -401,7 +401,7 @@ func (s *Service) saveOrphanedOperations(ctx context.Context, orphanedRoot [32]b
 		}
 		for _, a := range orphanedBlk.Block().Body().Attestations() {
 			// if the attestation is one epoch older, it wouldn't been useful to save it.
-			if a.Data.Slot+params.BeaconConfig().SlotsPerEpoch < s.CurrentSlot() {
+			if a.GetData().Slot+params.BeaconConfig().SlotsPerEpoch < s.CurrentSlot() {
 				continue
 			}
 			if helpers.IsAggregated(a) {

beacon-chain/blockchain/metrics.go

@@ -369,6 +369,6 @@ func reportEpochMetrics(ctx context.Context, postState, headState state.BeaconSt
 
 func reportAttestationInclusion(blk interfaces.ReadOnlyBeaconBlock) {
 	for _, att := range blk.Body().Attestations() {
-		attestationInclusionDelay.Observe(float64(blk.Slot() - att.Data.Slot))
+		attestationInclusionDelay.Observe(float64(blk.Slot() - att.GetData().Slot))
 	}
 }

beacon-chain/blockchain/process_attestation.go

@@ -6,9 +6,12 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/v5/consensus-types/interfaces"
+	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
 	ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1/attestation"
+	"github.com/prysmaticlabs/prysm/v5/runtime/version"
 	"github.com/prysmaticlabs/prysm/v5/time/slots"
 	"go.opencensus.io/trace"
 )
@@ -36,17 +39,17 @@ import (
 //
 //	 # Update latest messages for attesting indices
 //	 update_latest_messages(store, indexed_attestation.attesting_indices, attestation)
-func (s *Service) OnAttestation(ctx context.Context, a *ethpb.Attestation, disparity time.Duration) error {
+func (s *Service) OnAttestation(ctx context.Context, a interfaces.Attestation, disparity time.Duration) error {
 	ctx, span := trace.StartSpan(ctx, "blockChain.onAttestation")
 	defer span.End()
 
 	if err := helpers.ValidateNilAttestation(a); err != nil {
 		return err
 	}
-	if err := helpers.ValidateSlotTargetEpoch(a.Data); err != nil {
+	if err := helpers.ValidateSlotTargetEpoch(a.GetData()); err != nil {
 		return err
 	}
-	tgt := ethpb.CopyCheckpoint(a.Data.Target)
+	tgt := ethpb.CopyCheckpoint(a.GetData().Target)
 
 	// Note that target root check is ignored here because it was performed in sync's validation pipeline:
 	// validate_aggregate_proof.go and validate_beacon_attestation.go
@@ -67,24 +70,38 @@ func (s *Service) OnAttestation(ctx context.Context, a *ethpb.Attestation, dispa
 	}
 
 	// Verify attestation beacon block is known and not from the future.
-	if err := s.verifyBeaconBlock(ctx, a.Data); err != nil {
+	if err := s.verifyBeaconBlock(ctx, a.GetData()); err != nil {
 		return errors.Wrap(err, "could not verify attestation beacon block")
 	}
 
 	// Note that LMD GHOST and FFG consistency check is ignored because it was performed in sync's validation pipeline:
 	// validate_aggregate_proof.go and validate_beacon_attestation.go
 
 	// Verify attestations can only affect the fork choice of subsequent slots.
-	if err := slots.VerifyTime(genesisTime, a.Data.Slot+1, disparity); err != nil {
+	if err := slots.VerifyTime(genesisTime, a.GetData().Slot+1, disparity); err != nil {
 		return err
 	}
 
 	// Use the target state to verify attesting indices are valid.
-	committee, err := helpers.BeaconCommitteeFromState(ctx, baseState, a.Data.Slot, a.Data.CommitteeIndex)
-	if err != nil {
-		return err
+	var committees [][]primitives.ValidatorIndex
+	if a.Version() < version.Electra {
+		committee, err := helpers.BeaconCommitteeFromState(ctx, baseState, a.GetData().Slot, a.GetData().CommitteeIndex)
+		if err != nil {
+			return err
+		}
+		committees = [][]primitives.ValidatorIndex{committee}
+	} else {
+		committeeIndices := a.GetCommitteeBits().BitIndices()
+		committees = make([][]primitives.ValidatorIndex, len(committeeIndices))
+		for i, ci := range committeeIndices {
+			committees[i], err = helpers.BeaconCommitteeFromState(ctx, baseState, a.GetData().Slot, primitives.CommitteeIndex(ci))
+			if err != nil {
+				return err
+			}
+		}
 	}
-	indexedAtt, err := attestation.ConvertToIndexed(ctx, a, committee)
+
+	indexedAtt, err := attestation.ConvertToIndexed(ctx, a, committees)
 	if err != nil {
 		return err
 	}
@@ -97,7 +114,7 @@ func (s *Service) OnAttestation(ctx context.Context, a *ethpb.Attestation, dispa
 	// We assume trusted attestation in this function has verified signature.
 
 	// Update forkchoice store with the new attestation for updating weight.
-	s.cfg.ForkChoiceStore.ProcessAttestation(ctx, indexedAtt.AttestingIndices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
+	s.cfg.ForkChoiceStore.ProcessAttestation(ctx, indexedAtt.AttestingIndices, bytesutil.ToBytes32(a.GetData().BeaconBlockRoot), a.GetData().Target.Epoch)
 
 	return nil
 }

beacon-chain/blockchain/process_block.go

@@ -366,17 +366,31 @@ func (s *Service) handleEpochBoundary(ctx context.Context, slot primitives.Slot,
 func (s *Service) handleBlockAttestations(ctx context.Context, blk interfaces.ReadOnlyBeaconBlock, st state.BeaconState) error {
 	// Feed in block's attestations to fork choice store.
 	for _, a := range blk.Body().Attestations() {
-		committee, err := helpers.BeaconCommitteeFromState(ctx, st, a.Data.Slot, a.Data.CommitteeIndex)
-		if err != nil {
-			return err
+		var committees [][]primitives.ValidatorIndex
+		if a.Version() < version.Electra {
+			committee, err := helpers.BeaconCommitteeFromState(ctx, st, a.GetData().Slot, a.GetData().CommitteeIndex)
+			if err != nil {
+				return err
+			}
+			committees = [][]primitives.ValidatorIndex{committee}
+		} else {
+			committeeIndices := a.GetCommitteeBits().BitIndices()
+			committees = make([][]primitives.ValidatorIndex, len(committeeIndices))
+			var err error
+			for i, ci := range committeeIndices {
+				committees[i], err = helpers.BeaconCommitteeFromState(ctx, st, a.GetData().Slot, primitives.CommitteeIndex(ci))
+				if err != nil {
+					return err
+				}
+			}
 		}
-		indices, err := attestation.AttestingIndices(a.AggregationBits, committee)
+		indices, err := attestation.AttestingIndices(a, committees)
 		if err != nil {
 			return err
 		}
-		r := bytesutil.ToBytes32(a.Data.BeaconBlockRoot)
+		r := bytesutil.ToBytes32(a.GetData().BeaconBlockRoot)
 		if s.cfg.ForkChoiceStore.HasNode(r) {
-			s.cfg.ForkChoiceStore.ProcessAttestation(ctx, indices, r, a.Data.Target.Epoch)
+			s.cfg.ForkChoiceStore.ProcessAttestation(ctx, indices, r, a.GetData().Target.Epoch)
 		} else if err := s.cfg.AttPool.SaveBlockAttestation(a); err != nil {
 			return err
 		}

beacon-chain/blockchain/receive_attestation.go

@@ -10,6 +10,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/v5/config/params"
+	"github.com/prysmaticlabs/prysm/v5/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
 	ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
@@ -31,7 +32,7 @@ type AttestationStateFetcher interface {
 // AttestationReceiver interface defines the methods of chain service receive and processing new attestations.
 type AttestationReceiver interface {
 	AttestationStateFetcher
-	VerifyLmdFfgConsistency(ctx context.Context, att *ethpb.Attestation) error
+	VerifyLmdFfgConsistency(ctx context.Context, att interfaces.Attestation) error
 	InForkchoice([32]byte) bool
 }
 
@@ -51,13 +52,13 @@ func (s *Service) AttestationTargetState(ctx context.Context, target *ethpb.Chec
 }
 
 // VerifyLmdFfgConsistency verifies that attestation's LMD and FFG votes are consistency to each other.
-func (s *Service) VerifyLmdFfgConsistency(ctx context.Context, a *ethpb.Attestation) error {
-	r, err := s.TargetRootForEpoch([32]byte(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
+func (s *Service) VerifyLmdFfgConsistency(ctx context.Context, a interfaces.Attestation) error {
+	r, err := s.TargetRootForEpoch([32]byte(a.GetData().BeaconBlockRoot), a.GetData().Target.Epoch)
 	if err != nil {
 		return err
 	}
-	if !bytes.Equal(a.Data.Target.Root, r[:]) {
-		return fmt.Errorf("FFG and LMD votes are not consistent, block root: %#x, target root: %#x, canonical target root: %#x", a.Data.BeaconBlockRoot, a.Data.Target.Root, r)
+	if !bytes.Equal(a.GetData().Target.Root, r[:]) {
+		return fmt.Errorf("FFG and LMD votes are not consistent, block root: %#x, target root: %#x, canonical target root: %#x", a.GetData().BeaconBlockRoot, a.GetData().Target.Root, r)
 	}
 	return nil
 }
@@ -170,13 +171,13 @@ func (s *Service) processAttestations(ctx context.Context, disparity time.Durati
 		// Based on the spec, don't process the attestation until the subsequent slot.
 		// This delays consideration in the fork choice until their slot is in the past.
 		// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md#validate_on_attestation
-		nextSlot := a.Data.Slot + 1
+		nextSlot := a.GetData().Slot + 1
 		if err := slots.VerifyTime(uint64(s.genesisTime.Unix()), nextSlot, disparity); err != nil {
 			continue
 		}
 
-		hasState := s.cfg.BeaconDB.HasStateSummary(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
-		hasBlock := s.hasBlock(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
+		hasState := s.cfg.BeaconDB.HasStateSummary(ctx, bytesutil.ToBytes32(a.GetData().BeaconBlockRoot))
+		hasBlock := s.hasBlock(ctx, bytesutil.ToBytes32(a.GetData().BeaconBlockRoot))
 		if !(hasState && hasBlock) {
 			continue
 		}
@@ -185,17 +186,17 @@ func (s *Service) processAttestations(ctx context.Context, disparity time.Durati
 			log.WithError(err).Error("Could not delete fork choice attestation in pool")
 		}
 
-		if !helpers.VerifyCheckpointEpoch(a.Data.Target, s.genesisTime) {
+		if !helpers.VerifyCheckpointEpoch(a.GetData().Target, s.genesisTime) {
 			continue
 		}
 
 		if err := s.receiveAttestationNoPubsub(ctx, a, disparity); err != nil {
 			log.WithFields(logrus.Fields{
-				"slot":             a.Data.Slot,
-				"committeeIndex":   a.Data.CommitteeIndex,
-				"beaconBlockRoot":  fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.BeaconBlockRoot)),
-				"targetRoot":       fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.Target.Root)),
-				"aggregationCount": a.AggregationBits.Count(),
+				"slot":             a.GetData().Slot,
+				"committeeIndex":   a.GetData().CommitteeIndex,
+				"beaconBlockRoot":  fmt.Sprintf("%#x", bytesutil.Trunc(a.GetData().BeaconBlockRoot)),
+				"targetRoot":       fmt.Sprintf("%#x", bytesutil.Trunc(a.GetData().Target.Root)),
+				"aggregationCount": a.GetAggregationBits().Count(),
 			}).WithError(err).Warn("Could not process attestation for fork choice")
 		}
 	}
@@ -206,7 +207,7 @@ func (s *Service) processAttestations(ctx context.Context, disparity time.Durati
 //  1. Validate attestation, update validator's latest vote
 //  2. Apply fork choice to the processed attestation
 //  3. Save latest head info
-func (s *Service) receiveAttestationNoPubsub(ctx context.Context, att *ethpb.Attestation, disparity time.Duration) error {
+func (s *Service) receiveAttestationNoPubsub(ctx context.Context, att interfaces.Attestation, disparity time.Duration) error {
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.receiveAttestationNoPubsub")
 	defer span.End()
 

beacon-chain/blockchain/receive_block.go

@@ -479,12 +479,27 @@ func (s *Service) sendBlockAttestationsToSlasher(signed interfaces.ReadOnlySigne
 	// is done in the background to avoid adding more load to this critical code path.
 	ctx := context.TODO()
 	for _, att := range signed.Block().Body().Attestations() {
-		committee, err := helpers.BeaconCommitteeFromState(ctx, preState, att.Data.Slot, att.Data.CommitteeIndex)
-		if err != nil {
-			log.WithError(err).Error("Could not get attestation committee")
-			return
+		var committees [][]primitives.ValidatorIndex
+		if att.Version() < version.Electra {
+			committee, err := helpers.BeaconCommitteeFromState(ctx, preState, att.GetData().Slot, att.GetData().CommitteeIndex)
+			if err != nil {
+				log.WithError(err).Error("Could not get attestation committee")
+				return
+			}
+			committees = [][]primitives.ValidatorIndex{committee}
+		} else {
+			committeeIndices := att.GetCommitteeBits().BitIndices()
+			committees = make([][]primitives.ValidatorIndex, len(committeeIndices))
+			var err error
+			for i, ci := range committeeIndices {
+				committees[i], err = helpers.BeaconCommitteeFromState(ctx, preState, att.GetData().Slot, primitives.CommitteeIndex(ci))
+				if err != nil {
+					log.WithError(err).Error("Could not get attestation committee")
+					return
+				}
+			}
 		}
-		indexedAtt, err := attestation.ConvertToIndexed(ctx, att, committee)
+		indexedAtt, err := attestation.ConvertToIndexed(ctx, att, committees)
 		if err != nil {
 			log.WithError(err).Error("Could not convert to indexed attestation")
 			return

beacon-chain/core/altair/BUILD.bazel

@@ -36,6 +36,7 @@ go_library(
         "//math:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/attestation:go_default_library",
+        "//runtime/version:go_default_library",
         "//time/slots:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",

beacon-chain/core/altair/attestation.go

@@ -16,6 +16,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
 	ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1/attestation"
+	"github.com/prysmaticlabs/prysm/v5/runtime/version"
 	"go.opencensus.io/trace"
 )
 
@@ -48,7 +49,7 @@ func ProcessAttestationsNoVerifySignature(
 func ProcessAttestationNoVerifySignature(
 	ctx context.Context,
 	beaconState state.BeaconState,
-	att *ethpb.Attestation,
+	att interfaces.Attestation,
 	totalBalance uint64,
 ) (state.BeaconState, error) {
 	ctx, span := trace.StartSpan(ctx, "altair.ProcessAttestationNoVerifySignature")
@@ -58,24 +59,38 @@ func ProcessAttestationNoVerifySignature(
 		return nil, err
 	}
 
-	delay, err := beaconState.Slot().SafeSubSlot(att.Data.Slot)
+	delay, err := beaconState.Slot().SafeSubSlot(att.GetData().Slot)
 	if err != nil {
-		return nil, fmt.Errorf("att slot %d can't be greater than state slot %d", att.Data.Slot, beaconState.Slot())
+		return nil, fmt.Errorf("att slot %d can't be greater than state slot %d", att.GetData().Slot, beaconState.Slot())
 	}
-	participatedFlags, err := AttestationParticipationFlagIndices(beaconState, att.Data, delay)
+	participatedFlags, err := AttestationParticipationFlagIndices(beaconState, att.GetData(), delay)
 	if err != nil {
 		return nil, err
 	}
-	committee, err := helpers.BeaconCommitteeFromState(ctx, beaconState, att.Data.Slot, att.Data.CommitteeIndex)
-	if err != nil {
-		return nil, err
+
+	var committees [][]primitives.ValidatorIndex
+	if att.Version() < version.Electra {
+		committee, err := helpers.BeaconCommitteeFromState(ctx, beaconState, att.GetData().Slot, att.GetData().CommitteeIndex)
+		if err != nil {
+			return nil, err
+		}
+		committees = [][]primitives.ValidatorIndex{committee}
+	} else {
+		committeeIndices := helpers.CommitteeIndices(att.GetCommitteeBits())
+		committees = make([][]primitives.ValidatorIndex, len(committeeIndices))
+		for i, ci := range committeeIndices {
+			committees[i], err = helpers.BeaconCommitteeFromState(ctx, beaconState, att.GetData().Slot, ci)
+			if err != nil {
+				return nil, err
+			}
+		}
 	}
-	indices, err := attestation.AttestingIndices(att.AggregationBits, committee)
+
+	indices, err := attestation.AttestingIndices(att, committees)
 	if err != nil {
 		return nil, err
 	}
-
-	return SetParticipationAndRewardProposer(ctx, beaconState, att.Data.Target.Epoch, indices, participatedFlags, totalBalance)
+	return SetParticipationAndRewardProposer(ctx, beaconState, att.GetData().Target.Epoch, indices, participatedFlags, totalBalance)
 }
 
 // SetParticipationAndRewardProposer retrieves and sets the epoch participation bits in state. Based on the epoch participation, it rewards

beacon-chain/core/altair/upgrade.go

@@ -8,8 +8,10 @@ import (
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
 	state_native "github.com/prysmaticlabs/prysm/v5/beacon-chain/state/state-native"
 	"github.com/prysmaticlabs/prysm/v5/config/params"
+	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
 	ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1/attestation"
+	"github.com/prysmaticlabs/prysm/v5/runtime/version"
 )
 
 // UpgradeToAltair updates input state to return the version Altair state.
@@ -154,11 +156,24 @@ func TranslateParticipation(ctx context.Context, state state.BeaconState, atts [
 		if err != nil {
 			return nil, err
 		}
-		committee, err := helpers.BeaconCommitteeFromState(ctx, state, att.Data.Slot, att.Data.CommitteeIndex)
-		if err != nil {
-			return nil, err
+		var committees [][]primitives.ValidatorIndex
+		if att.Version() < version.Electra {
+			committee, err := helpers.BeaconCommitteeFromState(ctx, state, att.GetData().Slot, att.GetData().CommitteeIndex)
+			if err != nil {
+				return nil, err
+			}
+			committees = [][]primitives.ValidatorIndex{committee}
+		} else {
+			committeeIndices := helpers.CommitteeIndices(att.GetCommitteeBits())
+			committees = make([][]primitives.ValidatorIndex, len(committeeIndices))
+			for i, ci := range committeeIndices {
+				committees[i], err = helpers.BeaconCommitteeFromState(ctx, state, att.GetData().Slot, ci)
+				if err != nil {
+					return nil, err
+				}
+			}
 		}
-		indices, err := attestation.AttestingIndices(att.AggregationBits, committee)
+		indices, err := attestation.AttestingIndices(att, committees)
 		if err != nil {
 			return nil, err
 		}