Web3signer: Bellatrix Block (#10590)

* initial commit

* fixing bazel

* removing extra space

* adding blindedbeaconblock

* adding unit tests

* fixing bazel build

* switching to switch statement

* fixing function parameters

* fixing ineffectual err error

* deleting unused files

* updating web3signer version to support bellatrix

* adding metrics

* testing longer run

* changing log level to help find errors

* changing logging back to all to better understand the issue

* testing better way to get public keys for web3signer on validator

* fixing bazel

* rolling back changes

* missed rolling back 1 thing

* adding flag back in

* adding comments back in

* testing lower participation

* adding logs to see web3signer keys for comparison

* fix bazel

* adding more logs temporariliy

* switching hex utility function

* web3signer doesn't support deposits, so changing this

* rolling back unintended unit test

* rolling back some changes for debugging

* Update validator/keymanager/remote-web3signer/v1/web3signer_types.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update testing/endtoend/components/web3remotesigner.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update testing/endtoend/endtoend_test.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update testing/endtoend/endtoend_test.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* fixing merge conflict

* Update validator/keymanager/remote-web3signer/v1/requests.go

* Update testing/endtoend/endtoend_test.go

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

* Update validator/keymanager/remote-web3signer/v1/requests.go

Co-authored-by: terencechain <terence@prysmaticlabs.com>

* prysm doesn't currently support deposits for web3signer

* reverting back to old implementation

Co-authored-by: Radosław Kapka <rkapka@wp.pl>
Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>
Co-authored-by: terencechain <terence@prysmaticlabs.com>

testing/endtoend/components/validator.go

@@ -3,7 +3,6 @@ package components
 import (
 	"bytes"
 	"context"
-	"encoding/hex"
 	"fmt"
 	"math/big"
 	"os"
@@ -14,6 +13,7 @@ import (
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/pkg/errors"
@@ -61,7 +61,6 @@ func (s *ValidatorNodeSet) Start(ctx context.Context) error {
 		return errors.New("validator count is not easily divisible by beacon node count")
 	}
 	validatorsPerNode := validatorNum / beaconNodeNum
-
 	// Create validator nodes.
 	nodes := make([]e2etypes.ComponentRunner, prysmBeaconNodeNum)
 	for i := 0; i < prysmBeaconNodeNum; i++ {
@@ -159,7 +158,7 @@ func (v *ValidatorNode) Start(ctx context.Context) error {
 		}
 		var hexPubs []string
 		for _, pub := range pubs {
-			hexPubs = append(hexPubs, "0x"+hex.EncodeToString(pub.Marshal()))
+			hexPubs = append(hexPubs, hexutil.Encode(pub.Marshal()))
 		}
 		args = append(args, fmt.Sprintf("--%s=%s", flags.Web3SignerPublicValidatorKeysFlag.Name, strings.Join(hexPubs, ",")))
 	} else {

testing/endtoend/components/web3remotesigner.go

@@ -216,6 +216,9 @@ func writeKeystoreKeys(ctx context.Context, keystorePath string, numKeys uint64)
 	if err != nil {
 		return err
 	}
+	for i, p := range pub {
+		log.Infof("web3signer file added %s, key index %v", hexutil.Encode(p.Marshal()), i)
+	}
 	for i, pk := range priv {
 		if ctx.Err() != nil {
 			return ctx.Err()

testing/endtoend/deps.bzl

@@ -6,11 +6,10 @@ lighthouse_archive_name = "lighthouse-%s-x86_64-unknown-linux-gnu-portable.tar.g
 def e2e_deps():
     http_archive(
         name = "web3signer",
-        # Built from commit 17d253b which has important unreleased changes.
-        urls = ["https://prysmaticlabs.com/uploads/web3signer-17d253b.tar.gz"],
-        sha256 = "bf450a59a0845c1ce8100b3192c7fec021b565efe8b1ab46bed9f71cb994a6d7",
+        urls = ["https://artifacts.consensys.net/public/web3signer/raw/names/web3signer.tar.gz/versions/22.5.0/web3signer-22.5.0.tar.gz"],
+        sha256 = "3a954f5302e424b34acb4bb024f275caf722d8b116c639617f7a2e0f9c9ddc78",
         build_file = "@prysm//testing/endtoend:web3signer.BUILD",
-        strip_prefix = "web3signer-develop",
+        strip_prefix = "web3signer-22.5.0",
     )
 
     http_archive(

testing/endtoend/endtoend_test.go

@@ -178,7 +178,7 @@ func (r *testRunner) run() {
 			comps = append(comps, web3RemoteSigner)
 		}
 		if err := helpers.ComponentsStarted(ctx, comps); err != nil {
-			return errors.Wrap(err, "validator nodes require beacon nodes to run")
+			return errors.Wrap(err, "validator nodes require components to run")
 		}
 		if err := validatorNodes.Start(ctx); err != nil {
 			return errors.Wrap(err, "failed to start validator nodes")
@@ -368,6 +368,8 @@ func (r *testRunner) runEvaluators(conns []*grpc.ClientConn, tickingStartTime ti
 func (r *testRunner) testDepositsAndTx(ctx context.Context, g *errgroup.Group,
 	keystorePath string, requiredNodes []e2etypes.ComponentRunner) {
 	minGenesisActiveCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount)
+	// prysm web3signer doesn't support deposits
+	r.config.UseWeb3RemoteSigner = false
 	depositCheckValidator := components.NewValidatorNode(r.config, int(e2e.DepositCount), e2e.TestParams.BeaconNodeCount, minGenesisActiveCount)
 	g.Go(func() error {
 		if err := helpers.ComponentsStarted(ctx, requiredNodes); err != nil {

testing/endtoend/minimal_e2e_test.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/prysmaticlabs/prysm/config/params"
 	ev "github.com/prysmaticlabs/prysm/testing/endtoend/evaluators"
-	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
 	e2eParams "github.com/prysmaticlabs/prysm/testing/endtoend/params"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/types"
 	"github.com/prysmaticlabs/prysm/testing/require"
@@ -34,10 +33,6 @@ func e2eMinimal(t *testing.T, useWeb3RemoteSigner bool, extraEpochs uint64) {
 		epochsToRun, err = strconv.Atoi(epochStr)
 		require.NoError(t, err)
 	}
-	// TODO(#10053): Web3signer does not support bellatrix yet.
-	if useWeb3RemoteSigner {
-		epochsToRun = helpers.BellatrixE2EForkEpoch - 1
-	}
 	seed := 0
 	seedStr, isValid := os.LookupEnv("E2E_SEED")
 	if isValid {
@@ -53,13 +48,13 @@ func e2eMinimal(t *testing.T, useWeb3RemoteSigner bool, extraEpochs uint64) {
 		ev.ValidatorsAreActive,
 		ev.ValidatorsParticipatingAtEpoch(2),
 		ev.FinalizationOccurs(3),
+		ev.VerifyBlockGraffiti,
 		ev.PeersCheck,
+		ev.ProposeVoluntaryExit,
+		ev.ValidatorHasExited,
 		ev.ProcessesDepositsInBlocks,
-		ev.VerifyBlockGraffiti,
 		ev.ActivatesDepositedValidators,
 		ev.DepositedValidatorsAreActive,
-		ev.ProposeVoluntaryExit,
-		ev.ValidatorHasExited,
 		ev.ValidatorsVoteWithTheMajority,
 		ev.ColdStateCheckpoint,
 		ev.AltairForkTransition,

validator/keymanager/remote-web3signer/keymanager.go

@@ -20,7 +20,7 @@ import (
 	"github.com/prysmaticlabs/prysm/validator/keymanager"
 	remote_utils "github.com/prysmaticlabs/prysm/validator/keymanager/remote-utils"
 	"github.com/prysmaticlabs/prysm/validator/keymanager/remote-web3signer/internal"
-	v1 "github.com/prysmaticlabs/prysm/validator/keymanager/remote-web3signer/v1"
+	web3signerv1 "github.com/prysmaticlabs/prysm/validator/keymanager/remote-web3signer/v1"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -113,7 +113,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 	}
 	switch request.Object.(type) {
 	case *validatorpb.SignRequest_Block:
-		bockSignRequest, err := v1.GetBlockSignRequest(request, genesisValidatorsRoot)
+		bockSignRequest, err := web3signerv1.GetBlockSignRequest(request, genesisValidatorsRoot)
 		if err != nil {
 			return nil, err
 		}
@@ -123,7 +123,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		blockSignRequestsTotal.Inc()
 		return json.Marshal(bockSignRequest)
 	case *validatorpb.SignRequest_AttestationData:
-		attestationSignRequest, err := v1.GetAttestationSignRequest(request, genesisValidatorsRoot)
+		attestationSignRequest, err := web3signerv1.GetAttestationSignRequest(request, genesisValidatorsRoot)
 		if err != nil {
 			return nil, err
 		}
@@ -133,7 +133,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		attestationSignRequestsTotal.Inc()
 		return json.Marshal(attestationSignRequest)
 	case *validatorpb.SignRequest_AggregateAttestationAndProof:
-		aggregateAndProofSignRequest, err := v1.GetAggregateAndProofSignRequest(request, genesisValidatorsRoot)
+		aggregateAndProofSignRequest, err := web3signerv1.GetAggregateAndProofSignRequest(request, genesisValidatorsRoot)
 		if err != nil {
 			return nil, err
 		}
@@ -143,7 +143,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		aggregateAndProofSignRequestsTotal.Inc()
 		return json.Marshal(aggregateAndProofSignRequest)
 	case *validatorpb.SignRequest_Slot:
-		aggregationSlotSignRequest, err := v1.GetAggregationSlotSignRequest(request, genesisValidatorsRoot)
+		aggregationSlotSignRequest, err := web3signerv1.GetAggregationSlotSignRequest(request, genesisValidatorsRoot)
 		if err != nil {
 			return nil, err
 		}
@@ -153,30 +153,43 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		aggregationSlotSignRequestsTotal.Inc()
 		return json.Marshal(aggregationSlotSignRequest)
 	case *validatorpb.SignRequest_BlockV2:
-		blocv2AltairSignRequest, err := v1.GetBlockV2AltairSignRequest(request, genesisValidatorsRoot)
+		blockv2AltairSignRequest, err := web3signerv1.GetBlockV2AltairSignRequest(request, genesisValidatorsRoot)
 		if err != nil {
 			return nil, err
 		}
-		if err = validator.StructCtx(ctx, blocv2AltairSignRequest); err != nil {
+		if err = validator.StructCtx(ctx, blockv2AltairSignRequest); err != nil {
 			return nil, err
 		}
 		blockV2SignRequestsTotal.Inc()
-		return json.Marshal(blocv2AltairSignRequest)
-	// TODO(#10053): Need to add support for merge blocks.
-
-	/*
-		case *validatorpb.SignRequest_BlockV3:
-		return "BLOCK_V3", nil
-	*/
-
+		return json.Marshal(blockv2AltairSignRequest)
+	case *validatorpb.SignRequest_BlockV3:
+		blockv2BellatrixSignRequest, err := web3signerv1.GetBlockV2BellatrixSignRequest(request, genesisValidatorsRoot)
+		if err != nil {
+			return nil, err
+		}
+		if err = validator.StructCtx(ctx, blockv2BellatrixSignRequest); err != nil {
+			return nil, err
+		}
+		blockV3SignRequestsTotal.Inc()
+		return json.Marshal(blockv2BellatrixSignRequest)
+	case *validatorpb.SignRequest_BlindedBlockV3:
+		blindedBlockv2SignRequest, err := web3signerv1.GetBlockV2BellatrixSignRequest(request, genesisValidatorsRoot)
+		if err != nil {
+			return nil, err
+		}
+		if err = validator.StructCtx(ctx, blindedBlockv2SignRequest); err != nil {
+			return nil, err
+		}
+		blindedblockV3SignRequestsTotal.Inc()
+		return json.Marshal(blindedBlockv2SignRequest)
 	// We do not support "DEPOSIT" type.
 	/*
 		case *validatorpb.:
 		return "DEPOSIT", nil
 	*/
 
 	case *validatorpb.SignRequest_Epoch:
-		randaoRevealSignRequest, err := v1.GetRandaoRevealSignRequest(request, genesisValidatorsRoot)
+		randaoRevealSignRequest, err := web3signerv1.GetRandaoRevealSignRequest(request, genesisValidatorsRoot)
 		if err != nil {
 			return nil, err
 		}
@@ -186,7 +199,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		randaoRevealSignRequestsTotal.Inc()
 		return json.Marshal(randaoRevealSignRequest)
 	case *validatorpb.SignRequest_Exit:
-		voluntaryExitRequest, err := v1.GetVoluntaryExitSignRequest(request, genesisValidatorsRoot)
+		voluntaryExitRequest, err := web3signerv1.GetVoluntaryExitSignRequest(request, genesisValidatorsRoot)
 		if err != nil {
 			return nil, err
 		}
@@ -196,7 +209,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		voluntaryExitSignRequestsTotal.Inc()
 		return json.Marshal(voluntaryExitRequest)
 	case *validatorpb.SignRequest_SyncMessageBlockRoot:
-		syncCommitteeMessageRequest, err := v1.GetSyncCommitteeMessageSignRequest(request, genesisValidatorsRoot)
+		syncCommitteeMessageRequest, err := web3signerv1.GetSyncCommitteeMessageSignRequest(request, genesisValidatorsRoot)
 		if err != nil {
 			return nil, err
 		}
@@ -206,7 +219,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		syncCommitteeMessageSignRequestsTotal.Inc()
 		return json.Marshal(syncCommitteeMessageRequest)
 	case *validatorpb.SignRequest_SyncAggregatorSelectionData:
-		syncCommitteeSelectionProofRequest, err := v1.GetSyncCommitteeSelectionProofSignRequest(request, genesisValidatorsRoot)
+		syncCommitteeSelectionProofRequest, err := web3signerv1.GetSyncCommitteeSelectionProofSignRequest(request, genesisValidatorsRoot)
 		if err != nil {
 			return nil, err
 		}
@@ -216,7 +229,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		syncCommitteeSelectionProofSignRequestsTotal.Inc()
 		return json.Marshal(syncCommitteeSelectionProofRequest)
 	case *validatorpb.SignRequest_ContributionAndProof:
-		contributionAndProofRequest, err := v1.GetSyncCommitteeContributionAndProofSignRequest(request, genesisValidatorsRoot)
+		contributionAndProofRequest, err := web3signerv1.GetSyncCommitteeContributionAndProofSignRequest(request, genesisValidatorsRoot)
 		if err != nil {
 			return nil, err
 		}

validator/keymanager/remote-web3signer/metrics.go

@@ -34,6 +34,14 @@ var (
 		Name: "remote_web3signer_block_v2_sign_requests_total",
 		Help: "Total number of block v2 sign requests",
 	})
+	blockV3SignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_block_v3_sign_requests_total",
+		Help: "Total number of block v3 sign requests",
+	})
+	blindedblockV3SignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_blinded_block_v3_sign_requests_total",
+		Help: "Total number of blinded block v3 sign requests",
+	})
 	randaoRevealSignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "remote_web3signer_randao_reveal_sign_requests_total",
 		Help: "Total number of randao reveal sign requests",

validator/keymanager/remote-web3signer/v1/BUILD.bazel

@@ -10,7 +10,9 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/validator/keymanager/remote-web3signer/v1",
     visibility = ["//visibility:public"],
     deps = [
+        "//consensus-types/interfaces:go_default_library",
         "//consensus-types/primitives:go_default_library",
+        "//consensus-types/wrapper:go_default_library",
         "//network/forks:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/validator-client:go_default_library",

validator/keymanager/remote-web3signer/v1/mock/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
         "//config/fieldparams:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/validator-client:go_default_library",
+        "//testing/util:go_default_library",
         "//validator/keymanager/remote-web3signer/v1:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_prysmaticlabs_go_bitfield//:go_default_library",

validator/keymanager/remote-web3signer/v1/mock/mocks.go

@@ -8,6 +8,7 @@ import (
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	eth "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	validatorpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/validator-client"
+	"github.com/prysmaticlabs/prysm/testing/util"
 	v1 "github.com/prysmaticlabs/prysm/validator/keymanager/remote-web3signer/v1"
 )
 
@@ -326,6 +327,24 @@ func GetMockSignRequest(t string) *validatorpb.SignRequest {
 			},
 			SigningSlot: 0,
 		}
+	case "BLOCK_V2_BELLATRIX":
+		return &validatorpb.SignRequest{
+			PublicKey:       make([]byte, fieldparams.BLSPubkeyLength),
+			SigningRoot:     make([]byte, fieldparams.RootLength),
+			SignatureDomain: make([]byte, 4),
+			Object: &validatorpb.SignRequest_BlockV3{
+				BlockV3: util.HydrateBeaconBlockBellatrix(&eth.BeaconBlockBellatrix{}),
+			},
+		}
+	case "BLOCK_V2_BLINDED_BELLATRIX":
+		return &validatorpb.SignRequest{
+			PublicKey:       make([]byte, fieldparams.BLSPubkeyLength),
+			SigningRoot:     make([]byte, fieldparams.RootLength),
+			SignatureDomain: make([]byte, 4),
+			Object: &validatorpb.SignRequest_BlindedBlockV3{
+				BlindedBlockV3: util.HydrateBlindedBeaconBlockBellatrix(&eth.BlindedBeaconBlockBellatrix{}),
+			},
+		}
 	case "RANDAO_REVEAL":
 		return &validatorpb.SignRequest{
 			PublicKey:       make([]byte, fieldparams.BLSPubkeyLength),
@@ -461,6 +480,24 @@ func MockBlockV2AltairSignRequest() *v1.BlockV2AltairSignRequest {
 	}
 }
 
+func MockBlockV2BellatrixSignRequest(bodyRoot string) *v1.BlockV2BellatrixSignRequest {
+	return &v1.BlockV2BellatrixSignRequest{
+		Type:        "BLOCK_V2",
+		ForkInfo:    MockForkInfo(),
+		SigningRoot: hexutil.Encode(make([]byte, fieldparams.RootLength)),
+		BeaconBlock: &v1.BeaconBlockBellatrixBlockV2{
+			Version: "BELLATRIX",
+			BlockHeader: &v1.BeaconBlockHeader{
+				Slot:          "0",
+				ProposerIndex: "0",
+				ParentRoot:    hexutil.Encode(make([]byte, fieldparams.RootLength)),
+				StateRoot:     hexutil.Encode(make([]byte, fieldparams.RootLength)),
+				BodyRoot:      bodyRoot,
+			},
+		},
+	}
+}
+
 // MockRandaoRevealSignRequest is a mock implementation of the RandaoRevealSignRequest.
 func MockRandaoRevealSignRequest() *v1.RandaoRevealSignRequest {
 	return &v1.RandaoRevealSignRequest{