From cfd3e548cd925cfbebcb542c59de446e10b7b634 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Mon, 22 Apr 2019 19:08:20 -0500 Subject: [PATCH 01/54] Use Map for Inclusion Distance in EpochProcessing (#2346) --- .../core/balances/rewards_penalties.go | 8 +-- .../core/balances/rewards_penalties_test.go | 39 ++++++++++++--- beacon-chain/core/epoch/epoch_operations.go | 23 --------- .../core/epoch/epoch_operations_test.go | 50 ------------------- 4 files changed, 36 insertions(+), 84 deletions(-) diff --git a/beacon-chain/core/balances/rewards_penalties.go b/beacon-chain/core/balances/rewards_penalties.go index 7c25fd9ffc2b..91d5a856f283 100644 --- a/beacon-chain/core/balances/rewards_penalties.go +++ b/beacon-chain/core/balances/rewards_penalties.go @@ -139,9 +139,9 @@ func InclusionDistance( baseRewardQuotient := helpers.BaseRewardQuotient(totalBalance) for _, index := range attesterIndices { - inclusionDistance, err := epoch.InclusionDistance(state, index) + inclusionDistance, err := blocks.AttsInclusionDistance(index) if err != nil { - return nil, fmt.Errorf("could not get inclusion distance: %v", err) + return nil, err } if inclusionDistance == 0 { return nil, errors.New("could not process inclusion distance: 0") @@ -266,9 +266,9 @@ func InactivityInclusionDistance( baseRewardQuotient := helpers.BaseRewardQuotient(totalBalance) for _, index := range attesterIndices { - inclusionDistance, err := epoch.InclusionDistance(state, index) + inclusionDistance, err := blocks.AttsInclusionDistance(index) if err != nil { - return nil, fmt.Errorf("could not get inclusion distance: %v", err) + return nil, err } baseReward := helpers.BaseReward(state, index, baseRewardQuotient) state.ValidatorBalances[index] -= baseReward - diff --git a/beacon-chain/core/balances/rewards_penalties_test.go b/beacon-chain/core/balances/rewards_penalties_test.go index cfa3aa50304f..74a1f735e6f5 100644 --- a/beacon-chain/core/balances/rewards_penalties_test.go +++ b/beacon-chain/core/balances/rewards_penalties_test.go @@ -140,10 +140,16 @@ func TestInclusionDistRewards_AccurateRewards(t *testing.T) { participationBitfield = append(participationBitfield, byte(0xff)) } - attestation := []*pb.PendingAttestation{ - {Data: &pb.AttestationData{Slot: params.BeaconConfig().GenesisSlot}, + attestations := []*pb.PendingAttestation{ + {Data: &pb.AttestationData{ + Slot: params.BeaconConfig().GenesisSlot, + JustifiedBlockRootHash32: []byte{}, + Shard: 0, + CrosslinkDataRootHash32: params.BeaconConfig().ZeroHash[:], + }, AggregationBitfield: participationBitfield, - InclusionSlot: params.BeaconConfig().GenesisSlot + 5}, + InclusionSlot: params.BeaconConfig().GenesisSlot + 5, + }, } tests := []struct { @@ -158,10 +164,29 @@ func TestInclusionDistRewards_AccurateRewards(t *testing.T) { validatorBalances[i] = params.BeaconConfig().MaxDepositAmount } state := &pb.BeaconState{ - Slot: params.BeaconConfig().GenesisSlot, - ValidatorRegistry: validators, - ValidatorBalances: validatorBalances, - LatestAttestations: attestation, + Slot: params.BeaconConfig().GenesisSlot + 5, + ValidatorRegistry: validators, + ValidatorBalances: validatorBalances, + LatestAttestations: attestations, + PreviousJustifiedRoot: []byte{}, + LatestCrosslinks: []*pb.Crosslink{ + { + CrosslinkDataRootHash32: params.BeaconConfig().ZeroHash[:], + Epoch: params.BeaconConfig().GenesisEpoch, + }, + }, + } + block := &pb.BeaconBlock{ + Body: &pb.BeaconBlockBody{ + Attestations: []*pb.Attestation{ + { + Data: attestations[0].Data, + }, + }, + }, + } + if _, err := blocks.ProcessBlockAttestations(state, block, false /* verify sig */); err != nil { + t.Fatal(err) } state, err := InclusionDistance( state, diff --git a/beacon-chain/core/epoch/epoch_operations.go b/beacon-chain/core/epoch/epoch_operations.go index 863737809f89..c7099164d1d2 100644 --- a/beacon-chain/core/epoch/epoch_operations.go +++ b/beacon-chain/core/epoch/epoch_operations.go @@ -65,29 +65,6 @@ func InclusionSlot(state *pb.BeaconState, validatorIndex uint64) (uint64, error) return lowestSlotIncluded, nil } -// InclusionDistance returns the difference in slot number of when attestation -// gets submitted and when it gets included. -// -// Spec pseudocode definition: -// Let inclusion_distance(state, index) = -// a.slot_included - a.data.slot where a is the above attestation same as -// inclusion_slot. -func InclusionDistance(state *pb.BeaconState, validatorIndex uint64) (uint64, error) { - - for _, attestation := range state.LatestAttestations { - participatedValidators, err := helpers.AttestationParticipants(state, attestation.Data, attestation.AggregationBitfield) - if err != nil { - return 0, fmt.Errorf("could not get attestation participants: %v", err) - } - for _, index := range participatedValidators { - if index == validatorIndex { - return attestation.InclusionSlot - attestation.Data.Slot, nil - } - } - } - return 0, fmt.Errorf("could not find inclusion distance for validator index %d", validatorIndex) -} - // AttestingValidators returns the validators of the winning root. // // Spec pseudocode definition: diff --git a/beacon-chain/core/epoch/epoch_operations_test.go b/beacon-chain/core/epoch/epoch_operations_test.go index e968b0c87106..196c1ea1260a 100644 --- a/beacon-chain/core/epoch/epoch_operations_test.go +++ b/beacon-chain/core/epoch/epoch_operations_test.go @@ -254,53 +254,3 @@ func TestInclusionSlot_SlotNotFound(t *testing.T) { t.Errorf("Expected %s, received %v", want, err) } } - -func TestInclusionDistance_CorrectDistance(t *testing.T) { - state := buildState(params.BeaconConfig().GenesisSlot, params.BeaconConfig().DepositsForChainStart) - var participationBitfield []byte - for i := 0; i < 16; i++ { - participationBitfield = append(participationBitfield, byte(0xff)) - } - - state.LatestAttestations = []*pb.PendingAttestation{ - {Data: &pb.AttestationData{Slot: params.BeaconConfig().GenesisSlot}, - AggregationBitfield: participationBitfield, - InclusionSlot: params.BeaconConfig().GenesisSlot + 100}, - } - distance, err := InclusionDistance(state, 251) - if err != nil { - t.Fatalf("Could not execute InclusionDistance: %v", err) - } - - // Inclusion distance is 100 because input validator index is 45, - // validator 45's attested slot 0 and got included slot 100. - if distance != 100 { - t.Errorf("Incorrect distance. Wanted: %d, got: %d", - 100, distance) - } -} - -func TestInclusionDistance_InvalidBitfield(t *testing.T) { - state := buildState(params.BeaconConfig().GenesisSlot, params.BeaconConfig().DepositsForChainStart) - - state.LatestAttestations = []*pb.PendingAttestation{ - {Data: &pb.AttestationData{Slot: params.BeaconConfig().GenesisSlot}, - AggregationBitfield: []byte{}, - InclusionSlot: 100}, - } - - want := fmt.Sprintf("wanted participants bitfield length %d, got: %d", 16, 0) - if _, err := InclusionDistance(state, 0); !strings.Contains(err.Error(), want) { - t.Errorf("Expected %s, received %v", want, err) - } -} - -func TestInclusionDistance_NotFound(t *testing.T) { - state := buildState(0, params.BeaconConfig().SlotsPerEpoch) - - badIndex := uint64(10000) - want := fmt.Sprintf("could not find inclusion distance for validator index %d", badIndex) - if _, err := InclusionDistance(state, badIndex); !strings.Contains(err.Error(), want) { - t.Errorf("Expected %s, received %v", want, err) - } -} From 2d8dd203f1ba6eac90a11364473ace8acf5558ab Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Mon, 22 Apr 2019 19:59:02 -0500 Subject: [PATCH 02/54] Optimize LMD-GHOST Vote Counting (#2333) * use att targets * risky head request * timing lmd * fixed ancestor cache hits * tests pass * comment on func * advance state transition * advance N empty state transitions * imports * imports * use risky head state * use validator registry * revert timing * imports --- beacon-chain/attestation/service.go | 15 +- beacon-chain/attestation/service_test.go | 7 +- beacon-chain/blockchain/BUILD.bazel | 1 - beacon-chain/blockchain/fork_choice.go | 95 ++-- beacon-chain/blockchain/fork_choice_test.go | 218 ++++++-- beacon-chain/cache/block.go | 4 +- beacon-chain/cache/block_test.go | 24 +- proto/beacon/p2p/v1/types.pb.go | 556 +++++++++++++++----- proto/beacon/p2p/v1/types.proto | 6 + 9 files changed, 690 insertions(+), 236 deletions(-) diff --git a/beacon-chain/attestation/service.go b/beacon-chain/attestation/service.go index c4944d31a451..8040c7318f28 100644 --- a/beacon-chain/attestation/service.go +++ b/beacon-chain/attestation/service.go @@ -121,7 +121,7 @@ func (a *Service) LatestAttestation(ctx context.Context, index uint64) (*pb.Atte // Let `get_latest_attestation_target(store: Store, validator_index: ValidatorIndex) -> // BeaconBlock` be the target block in the attestation // `get_latest_attestation(store, validator_index)`. -func (a *Service) LatestAttestationTarget(ctx context.Context, index uint64) (*pb.BeaconBlock, error) { +func (a *Service) LatestAttestationTarget(ctx context.Context, index uint64) (*pb.AttestationTarget, error) { attestation, err := a.LatestAttestation(ctx, index) if err != nil { return nil, fmt.Errorf("could not get attestation: %v", err) @@ -129,12 +129,19 @@ func (a *Service) LatestAttestationTarget(ctx context.Context, index uint64) (*p if attestation == nil { return nil, nil } - targetBlockHash := bytesutil.ToBytes32(attestation.Data.BeaconBlockRootHash32) - targetBlock, err := a.beaconDB.Block(targetBlockHash) + targetRoot := bytesutil.ToBytes32(attestation.Data.BeaconBlockRootHash32) + targetBlock, err := a.beaconDB.Block(targetRoot) if err != nil { return nil, fmt.Errorf("could not get target block: %v", err) } - return targetBlock, nil + if targetBlock == nil { + return nil, nil + } + return &pb.AttestationTarget{ + Slot: targetBlock.Slot, + BlockRoot: targetRoot[:], + ParentRoot: targetBlock.ParentRootHash32, + }, nil } // attestationPool takes an newly received attestation from sync service diff --git a/beacon-chain/attestation/service_test.go b/beacon-chain/attestation/service_test.go index 20515bd18c87..f42aca9c8159 100644 --- a/beacon-chain/attestation/service_test.go +++ b/beacon-chain/attestation/service_test.go @@ -1,6 +1,7 @@ package attestation import ( + "bytes" "context" "fmt" "reflect" @@ -244,12 +245,12 @@ func TestLatestAttestationTarget_ReturnsLatestAttestedBlock(t *testing.T) { pubKey48 := bytesutil.ToBytes48(pubKey) service.store.m[pubKey48] = attestation - latestAttestedBlock, err := service.LatestAttestationTarget(ctx, 0) + latestAttestedTarget, err := service.LatestAttestationTarget(ctx, 0) if err != nil { t.Fatalf("Could not get latest attestation: %v", err) } - if !reflect.DeepEqual(block, latestAttestedBlock) { - t.Errorf("Wanted: %v, got: %v", block, latestAttestedBlock) + if !bytes.Equal(blockRoot[:], latestAttestedTarget.BlockRoot) { + t.Errorf("Wanted: %v, got: %v", blockRoot[:], latestAttestedTarget.BlockRoot) } } diff --git a/beacon-chain/blockchain/BUILD.bazel b/beacon-chain/blockchain/BUILD.bazel index 13cb84c6ad91..fc3b537c3716 100644 --- a/beacon-chain/blockchain/BUILD.bazel +++ b/beacon-chain/blockchain/BUILD.bazel @@ -26,7 +26,6 @@ go_library( "//shared/hashutil:go_default_library", "//shared/p2p:go_default_library", "//shared/params:go_default_library", - "@com_github_gogo_protobuf//proto:go_default_library", "@com_github_prometheus_client_golang//prometheus:go_default_library", "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", diff --git a/beacon-chain/blockchain/fork_choice.go b/beacon-chain/blockchain/fork_choice.go index 00940cba52b8..a90bb4af3529 100644 --- a/beacon-chain/blockchain/fork_choice.go +++ b/beacon-chain/blockchain/fork_choice.go @@ -1,10 +1,10 @@ package blockchain import ( + "bytes" "context" "fmt" - "github.com/gogo/protobuf/proto" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prysmaticlabs/prysm/beacon-chain/cache" @@ -15,6 +15,7 @@ import ( "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/params" + "github.com/sirupsen/logrus" "go.opencensus.io/trace" ) @@ -181,7 +182,9 @@ func (c *ChainService) ApplyForkChoiceRule( if err != nil { return fmt.Errorf("could not hash head: %v", err) } - log.WithField("headRoot", fmt.Sprintf("0x%x", h)).Info("Chain head block and state updated") + log.WithFields(logrus.Fields{ + "headRoot": fmt.Sprintf("0x%x", h), + }).Info("Chain head block and state updated") return nil } @@ -217,10 +220,9 @@ func (c *ChainService) lmdGhost( ctx context.Context, startBlock *pb.BeaconBlock, startState *pb.BeaconState, - voteTargets map[uint64]*pb.BeaconBlock, + voteTargets map[uint64]*pb.AttestationTarget, ) (*pb.BeaconBlock, error) { highestSlot := c.beaconDB.HighestBlockSlot() - head := startBlock for { children, err := c.blockChildren(ctx, head, highestSlot) @@ -290,18 +292,18 @@ func (c *ChainService) blockChildren(ctx context.Context, block *pb.BeaconBlock, // attestationTargets retrieves the list of attestation targets since last finalized epoch, // each attestation target consists of validator index and its attestation target (i.e. the block // which the validator attested to) -func (c *ChainService) attestationTargets(ctx context.Context, state *pb.BeaconState) (map[uint64]*pb.BeaconBlock, error) { +func (c *ChainService) attestationTargets(ctx context.Context, state *pb.BeaconState) (map[uint64]*pb.AttestationTarget, error) { indices := helpers.ActiveValidatorIndices(state.ValidatorRegistry, helpers.CurrentEpoch(state)) - attestationTargets := make(map[uint64]*pb.BeaconBlock) + attestationTargets := make(map[uint64]*pb.AttestationTarget) for i, index := range indices { - block, err := c.attsService.LatestAttestationTarget(ctx, index) + target, err := c.attsService.LatestAttestationTarget(ctx, index) if err != nil { return nil, fmt.Errorf("could not retrieve attestation target: %v", err) } - if block == nil { + if target == nil { continue } - attestationTargets[uint64(i)] = block + attestationTargets[uint64(i)] = target } return attestationTargets, nil } @@ -316,20 +318,25 @@ func (c *ChainService) attestationTargets(ctx context.Context, state *pb.BeaconS // for validator_index, target in attestation_targets // if get_ancestor(store, target, block.slot) == block // ) -func VoteCount(block *pb.BeaconBlock, state *pb.BeaconState, targets map[uint64]*pb.BeaconBlock, beaconDB *db.BeaconDB) (int, error) { +func VoteCount(block *pb.BeaconBlock, state *pb.BeaconState, targets map[uint64]*pb.AttestationTarget, beaconDB *db.BeaconDB) (int, error) { balances := 0 - var ancestor *pb.BeaconBlock + var ancestorRoot []byte var err error - for validatorIndex, targetBlock := range targets { + blockRoot, err := hashutil.HashBeaconBlock(block) + if err != nil { + return 0, err + } + + for validatorIndex, target := range targets { if featureconfig.FeatureConfig().EnableBlockAncestorCache { - ancestor, err = cachedAncestorBlock(targetBlock, block.Slot, beaconDB) + ancestorRoot, err = cachedAncestor(target, block.Slot, beaconDB) if err != nil { return 0, err } } else { // if block ancestor cache was not enabled, retrieve the ancestor recursively. - ancestor, err = BlockAncestor(targetBlock, block.Slot, beaconDB) + ancestorRoot, err = BlockAncestor(target, block.Slot, beaconDB) if err != nil { return 0, err } @@ -339,11 +346,11 @@ func VoteCount(block *pb.BeaconBlock, state *pb.BeaconState, targets map[uint64] // block older than current block 5. // B4 - B5 - B6 // \ - - - - - B7 - if ancestor == nil { + if ancestorRoot == nil { continue } - if proto.Equal(ancestor, block) { + if bytes.Equal(blockRoot[:], ancestorRoot) { balances += int(helpers.EffectiveBalance(state, validatorIndex)) } } @@ -363,53 +370,63 @@ func VoteCount(block *pb.BeaconBlock, state *pb.BeaconState, targets map[uint64] // return None // else: // return get_ancestor(store, store.get_parent(block), slot) -func BlockAncestor(block *pb.BeaconBlock, slot uint64, beaconDB *db.BeaconDB) (*pb.BeaconBlock, error) { - if block.Slot == slot { - return block, nil +func BlockAncestor(targetBlock *pb.AttestationTarget, slot uint64, beaconDB *db.BeaconDB) ([]byte, error) { + if targetBlock.Slot == slot { + return targetBlock.BlockRoot[:], nil } - if block.Slot < slot { + if targetBlock.Slot < slot { return nil, nil } - parentHash := bytesutil.ToBytes32(block.ParentRootHash32) - parent, err := beaconDB.Block(parentHash) + parentRoot := bytesutil.ToBytes32(targetBlock.ParentRoot) + parent, err := beaconDB.Block(parentRoot) if err != nil { return nil, fmt.Errorf("could not get parent block: %v", err) } if parent == nil { return nil, fmt.Errorf("parent block does not exist: %v", err) } - return BlockAncestor(parent, slot, beaconDB) + newTarget := &pb.AttestationTarget{ + Slot: parent.Slot, + BlockRoot: parentRoot[:], + ParentRoot: parent.ParentRootHash32, + } + return BlockAncestor(newTarget, slot, beaconDB) } -// cachedAncestorBlock retrieves the cached ancestor block from block ancestor cache, +// cachedAncestor retrieves the cached ancestor target from block ancestor cache, // if it's not there it looks up the block tree get it and cache it. -func cachedAncestorBlock(targetBlk *pb.BeaconBlock, height uint64, beaconDB *db.BeaconDB) (*pb.BeaconBlock, error) { - var ancestor *pb.BeaconBlock - +func cachedAncestor(target *pb.AttestationTarget, height uint64, beaconDB *db.BeaconDB) ([]byte, error) { // check if the ancestor block of from a given block height was cached. - targetHash, err := hashutil.HashBeaconBlock(targetBlk) - if err != nil { - return nil, err - } - cachedAncestorBlock, err := blkAncestorCache.AncestorBySlot(targetHash[:], height) + cachedAncestorInfo, err := blkAncestorCache.AncestorBySlot(target.BlockRoot, height) if err != nil { return nil, nil } - if cachedAncestorBlock != nil { - return cachedAncestorBlock.Block, nil + if cachedAncestorInfo != nil { + return cachedAncestorInfo.Target.BlockRoot, nil } - // add the ancestor to the cache if it was not cached. - ancestor, err = BlockAncestor(targetBlk, height, beaconDB) + ancestorRoot, err := BlockAncestor(target, height, beaconDB) + if err != nil { + return nil, err + } + ancestor, err := beaconDB.Block(bytesutil.ToBytes32(ancestorRoot)) if err != nil { return nil, err } + if ancestor == nil { + return nil, nil + } + ancestorTarget := &pb.AttestationTarget{ + Slot: ancestor.Slot, + BlockRoot: ancestorRoot, + ParentRoot: ancestor.ParentRootHash32, + } if err := blkAncestorCache.AddBlockAncestor(&cache.AncestorInfo{ - Hash: targetHash[:], Height: height, - Block: ancestor, + Hash: target.BlockRoot, + Target: ancestorTarget, }); err != nil { return nil, err } - return ancestor, nil + return ancestorRoot, nil } diff --git a/beacon-chain/blockchain/fork_choice_test.go b/beacon-chain/blockchain/fork_choice_test.go index 1f03653971d2..9d448b23d55d 100644 --- a/beacon-chain/blockchain/fork_choice_test.go +++ b/beacon-chain/blockchain/fork_choice_test.go @@ -140,9 +140,17 @@ func TestVoteCount_ParentDoesNotExistNoVoteCount(t *testing.T) { if err := beaconDB.SaveBlock(potentialHead); err != nil { t.Fatal(err) } + headRoot, err := hashutil.HashBeaconBlock(potentialHead) + if err != nil { + t.Fatal(err) + } - voteTargets := make(map[uint64]*pb.BeaconBlock) - voteTargets[0] = potentialHead + voteTargets := make(map[uint64]*pb.AttestationTarget) + voteTargets[0] = &pb.AttestationTarget{ + Slot: potentialHead.Slot, + BlockRoot: headRoot[:], + ParentRoot: potentialHead.ParentRootHash32, + } count, err := VoteCount(genesisBlock, &pb.BeaconState{}, voteTargets, beaconDB) if err != nil { t.Fatalf("Could not get vote count: %v", err) @@ -168,10 +176,19 @@ func TestVoteCount_IncreaseCountCorrectly(t *testing.T) { Slot: params.BeaconConfig().GenesisSlot + 5, ParentRootHash32: genesisRoot[:], } + headRoot1, err := hashutil.HashBeaconBlock(potentialHead) + if err != nil { + t.Fatal(err) + } + potentialHead2 := &pb.BeaconBlock{ Slot: params.BeaconConfig().GenesisSlot + 6, ParentRootHash32: genesisRoot[:], } + headRoot2, err := hashutil.HashBeaconBlock(potentialHead2) + if err != nil { + t.Fatal(err) + } // We store these potential heads in the DB. if err := beaconDB.SaveBlock(potentialHead); err != nil { t.Fatal(err) @@ -180,9 +197,17 @@ func TestVoteCount_IncreaseCountCorrectly(t *testing.T) { t.Fatal(err) } beaconState := &pb.BeaconState{ValidatorBalances: []uint64{1e9, 1e9}} - voteTargets := make(map[uint64]*pb.BeaconBlock) - voteTargets[0] = potentialHead - voteTargets[1] = potentialHead2 + voteTargets := make(map[uint64]*pb.AttestationTarget) + voteTargets[0] = &pb.AttestationTarget{ + Slot: potentialHead.Slot, + BlockRoot: headRoot1[:], + ParentRoot: potentialHead.ParentRootHash32, + } + voteTargets[1] = &pb.AttestationTarget{ + Slot: potentialHead2.Slot, + BlockRoot: headRoot2[:], + ParentRoot: potentialHead2.ParentRootHash32, + } count, err := VoteCount(genesisBlock, beaconState, voteTargets, beaconDB) if err != nil { t.Fatalf("Could not fetch vote balances: %v", err) @@ -479,6 +504,10 @@ func TestLMDGhost_TrivialHeadUpdate(t *testing.T) { Slot: 2, ParentRootHash32: root1[:], } + block2Root, err := hashutil.HashBeaconBlock(block2) + if err != nil { + t.Fatal(err) + } if err = chainService.beaconDB.SaveBlock(block2); err != nil { t.Fatalf("Could not save block: %v", err) } @@ -487,8 +516,12 @@ func TestLMDGhost_TrivialHeadUpdate(t *testing.T) { } // The only vote is on block 2. - voteTargets := make(map[uint64]*pb.BeaconBlock) - voteTargets[0] = block2 + voteTargets := make(map[uint64]*pb.AttestationTarget) + voteTargets[0] = &pb.AttestationTarget{ + Slot: block2.Slot, + BlockRoot: block2Root[:], + ParentRoot: block2.ParentRootHash32, + } // LMDGhost should pick block 2. head, err := chainService.lmdGhost(ctx, block1, beaconState, voteTargets) @@ -540,6 +573,10 @@ func TestLMDGhost_3WayChainSplitsSameHeight(t *testing.T) { Slot: 2, ParentRootHash32: root1[:], } + root2, err := hashutil.HashBeaconBlock(block2) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } if err = chainService.beaconDB.SaveBlock(block2); err != nil { t.Fatalf("Could not save block: %v", err) } @@ -551,6 +588,10 @@ func TestLMDGhost_3WayChainSplitsSameHeight(t *testing.T) { Slot: 3, ParentRootHash32: root1[:], } + root3, err := hashutil.HashBeaconBlock(block3) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } if err = chainService.beaconDB.SaveBlock(block3); err != nil { t.Fatalf("Could not save block: %v", err) } @@ -562,6 +603,10 @@ func TestLMDGhost_3WayChainSplitsSameHeight(t *testing.T) { Slot: 4, ParentRootHash32: root1[:], } + root4, err := hashutil.HashBeaconBlock(block4) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } if err = chainService.beaconDB.SaveBlock(block4); err != nil { t.Fatalf("Could not save block: %v", err) } @@ -570,11 +615,27 @@ func TestLMDGhost_3WayChainSplitsSameHeight(t *testing.T) { } // Give block 4 the most votes (2). - voteTargets := make(map[uint64]*pb.BeaconBlock) - voteTargets[0] = block2 - voteTargets[1] = block3 - voteTargets[2] = block4 - voteTargets[3] = block4 + voteTargets := make(map[uint64]*pb.AttestationTarget) + voteTargets[0] = &pb.AttestationTarget{ + Slot: block2.Slot, + BlockRoot: root2[:], + ParentRoot: block2.ParentRootHash32, + } + voteTargets[1] = &pb.AttestationTarget{ + Slot: block3.Slot, + BlockRoot: root3[:], + ParentRoot: block3.ParentRootHash32, + } + voteTargets[2] = &pb.AttestationTarget{ + Slot: block4.Slot, + BlockRoot: root4[:], + ParentRoot: block4.ParentRootHash32, + } + voteTargets[3] = &pb.AttestationTarget{ + Slot: block4.Slot, + BlockRoot: root4[:], + ParentRoot: block4.ParentRootHash32, + } // LMDGhost should pick block 4. head, err := chainService.lmdGhost(ctx, block1, beaconState, voteTargets) if err != nil { @@ -669,6 +730,10 @@ func TestLMDGhost_2WayChainSplitsDiffHeight(t *testing.T) { Slot: 5, ParentRootHash32: root3[:], } + root5, err := hashutil.HashBeaconBlock(block5) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } if err = chainService.beaconDB.SaveBlock(block5); err != nil { t.Fatalf("Could not save block: %v", err) } @@ -680,6 +745,10 @@ func TestLMDGhost_2WayChainSplitsDiffHeight(t *testing.T) { Slot: 6, ParentRootHash32: root4[:], } + root6, err := hashutil.HashBeaconBlock(block6) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } if err = chainService.beaconDB.SaveBlock(block6); err != nil { t.Fatalf("Could not save block: %v", err) } @@ -688,10 +757,22 @@ func TestLMDGhost_2WayChainSplitsDiffHeight(t *testing.T) { } // Give block 5 the most votes (2). - voteTargets := make(map[uint64]*pb.BeaconBlock) - voteTargets[0] = block6 - voteTargets[1] = block5 - voteTargets[2] = block5 + voteTargets := make(map[uint64]*pb.AttestationTarget) + voteTargets[0] = &pb.AttestationTarget{ + Slot: block6.Slot, + BlockRoot: root6[:], + ParentRoot: block6.ParentRootHash32, + } + voteTargets[1] = &pb.AttestationTarget{ + Slot: block5.Slot, + BlockRoot: root5[:], + ParentRoot: block5.ParentRootHash32, + } + voteTargets[2] = &pb.AttestationTarget{ + Slot: block5.Slot, + BlockRoot: root5[:], + ParentRoot: block5.ParentRootHash32, + } // LMDGhost should pick block 5. head, err := chainService.lmdGhost(ctx, block1, beaconState, voteTargets) if err != nil { @@ -758,9 +839,19 @@ func BenchmarkLMDGhost_8Slots_8Validators(b *testing.B) { } } - voteTargets := make(map[uint64]*pb.BeaconBlock) + blockRoot, err := hashutil.HashBeaconBlock(block) + if err != nil { + b.Fatal(err) + } + + voteTargets := make(map[uint64]*pb.AttestationTarget) + target := &pb.AttestationTarget{ + Slot: block.Slot, + BlockRoot: blockRoot[:], + ParentRoot: block.ParentRootHash32, + } for i := 0; i < validatorCount; i++ { - voteTargets[uint64(i)] = block + voteTargets[uint64(i)] = target } for i := 0; i < b.N; i++ { @@ -829,9 +920,19 @@ func BenchmarkLMDGhost_32Slots_8Validators(b *testing.B) { } } - voteTargets := make(map[uint64]*pb.BeaconBlock) + blockRoot, err := hashutil.HashBeaconBlock(block) + if err != nil { + b.Fatal(err) + } + + voteTargets := make(map[uint64]*pb.AttestationTarget) + target := &pb.AttestationTarget{ + Slot: block.Slot, + BlockRoot: blockRoot[:], + ParentRoot: block.ParentRootHash32, + } for i := 0; i < validatorCount; i++ { - voteTargets[uint64(i)] = block + voteTargets[uint64(i)] = target } for i := 0; i < b.N; i++ { @@ -898,9 +999,19 @@ func BenchmarkLMDGhost_32Slots_64Validators(b *testing.B) { } } - voteTargets := make(map[uint64]*pb.BeaconBlock) + blockRoot, err := hashutil.HashBeaconBlock(block) + if err != nil { + b.Fatal(err) + } + + voteTargets := make(map[uint64]*pb.AttestationTarget) + target := &pb.AttestationTarget{ + Slot: block.Slot, + BlockRoot: blockRoot[:], + ParentRoot: block.ParentRootHash32, + } for i := 0; i < validatorCount; i++ { - voteTargets[uint64(i)] = block + voteTargets[uint64(i)] = target } for i := 0; i < b.N; i++ { @@ -967,9 +1078,19 @@ func BenchmarkLMDGhost_64Slots_16384Validators(b *testing.B) { } } - voteTargets := make(map[uint64]*pb.BeaconBlock) + blockRoot, err := hashutil.HashBeaconBlock(block) + if err != nil { + b.Fatal(err) + } + + voteTargets := make(map[uint64]*pb.AttestationTarget) + target := &pb.AttestationTarget{ + Slot: block.Slot, + BlockRoot: blockRoot[:], + ParentRoot: block.ParentRootHash32, + } for i := 0; i < validatorCount; i++ { - voteTargets[uint64(i)] = block + voteTargets[uint64(i)] = target } for i := 0; i < b.N; i++ { @@ -1341,10 +1462,18 @@ func TestVoteCount_CacheEnabledAndMiss(t *testing.T) { Slot: params.BeaconConfig().GenesisSlot + 5, ParentRootHash32: genesisRoot[:], } + pHeadHash, err := hashutil.HashBeaconBlock(potentialHead) + if err != nil { + t.Fatal(err) + } potentialHead2 := &pb.BeaconBlock{ Slot: params.BeaconConfig().GenesisSlot + 6, ParentRootHash32: genesisRoot[:], } + pHeadHash2, err := hashutil.HashBeaconBlock(potentialHead2) + if err != nil { + t.Fatal(err) + } // We store these potential heads in the DB. if err := beaconDB.SaveBlock(potentialHead); err != nil { t.Fatal(err) @@ -1353,9 +1482,17 @@ func TestVoteCount_CacheEnabledAndMiss(t *testing.T) { t.Fatal(err) } beaconState := &pb.BeaconState{ValidatorBalances: []uint64{1e9, 1e9}} - voteTargets := make(map[uint64]*pb.BeaconBlock) - voteTargets[0] = potentialHead - voteTargets[1] = potentialHead2 + voteTargets := make(map[uint64]*pb.AttestationTarget) + voteTargets[0] = &pb.AttestationTarget{ + Slot: potentialHead.Slot, + BlockRoot: pHeadHash[:], + ParentRoot: potentialHead.ParentRootHash32, + } + voteTargets[1] = &pb.AttestationTarget{ + Slot: potentialHead2.Slot, + BlockRoot: pHeadHash2[:], + ParentRoot: potentialHead2.ParentRootHash32, + } count, err := VoteCount(genesisBlock, beaconState, voteTargets, beaconDB) if err != nil { t.Fatalf("Could not fetch vote balances: %v", err) @@ -1371,8 +1508,7 @@ func TestVoteCount_CacheEnabledAndMiss(t *testing.T) { t.Fatal(err) } // Verify the cached block ancestor is genesis block. - h, _ = hashutil.HashBeaconBlock(cachedInfo.Block) - if h != genesisRoot { + if bytesutil.ToBytes32(cachedInfo.Target.BlockRoot) != genesisRoot { t.Error("could not retrieve the correct ancestor block") } } @@ -1400,20 +1536,30 @@ func TestVoteCount_CacheEnabledAndHit(t *testing.T) { pHeadHash2, _ := hashutil.HashBeaconBlock(potentialHead2) beaconState := &pb.BeaconState{ValidatorBalances: []uint64{1e9, 1e9}} - voteTargets := make(map[uint64]*pb.BeaconBlock) - voteTargets[0] = potentialHead - voteTargets[1] = potentialHead2 + voteTargets := make(map[uint64]*pb.AttestationTarget) + voteTargets[0] = &pb.AttestationTarget{ + Slot: potentialHead.Slot, + BlockRoot: pHeadHash[:], + ParentRoot: potentialHead.ParentRootHash32, + } + voteTargets[1] = &pb.AttestationTarget{ + Slot: potentialHead2.Slot, + BlockRoot: pHeadHash2[:], + ParentRoot: potentialHead2.ParentRootHash32, + } aInfo := &cache.AncestorInfo{ - Height: genesisBlock.Slot, - Hash: pHeadHash[:], - Block: genesisBlock, + Target: &pb.AttestationTarget{ + Slot: genesisBlock.Slot, + BlockRoot: genesisRoot[:], + ParentRoot: genesisBlock.ParentRootHash32, + }, } // Presave cached ancestor blocks before running vote count. if err := blkAncestorCache.AddBlockAncestor(aInfo); err != nil { t.Fatal(err) } - aInfo.Hash = pHeadHash2[:] + aInfo.Target.BlockRoot = pHeadHash2[:] if err := blkAncestorCache.AddBlockAncestor(aInfo); err != nil { t.Fatal(err) } diff --git a/beacon-chain/cache/block.go b/beacon-chain/cache/block.go index b007466d45fe..3d4e3c15f233 100644 --- a/beacon-chain/cache/block.go +++ b/beacon-chain/cache/block.go @@ -32,9 +32,9 @@ var ( // AncestorInfo defines the cached ancestor block object for height. type AncestorInfo struct { - Hash []byte Height uint64 - Block *pb.BeaconBlock + Hash []byte + Target *pb.AttestationTarget } // AncestorBlockCache structs with 1 queue for looking up block ancestor by height. diff --git a/beacon-chain/cache/block_test.go b/beacon-chain/cache/block_test.go index 0f0773047180..6897a4561121 100644 --- a/beacon-chain/cache/block_test.go +++ b/beacon-chain/cache/block_test.go @@ -14,6 +14,10 @@ func TestHeightHeightFn_OK(t *testing.T) { aInfo := &AncestorInfo{ Height: height, Hash: hash, + Target: &pb.AttestationTarget{ + Slot: height, + BlockRoot: hash, + }, } key, err := heightKeyFn(aInfo) @@ -21,7 +25,7 @@ func TestHeightHeightFn_OK(t *testing.T) { t.Fatal(err) } - strHeightKey := string(aInfo.Hash) + strconv.Itoa(int(aInfo.Height)) + strHeightKey := string(aInfo.Target.BlockRoot) + strconv.Itoa(int(aInfo.Target.Slot)) if key != strHeightKey { t.Errorf("Incorrect hash key: %s, expected %s", key, strHeightKey) } @@ -42,7 +46,10 @@ func TestAncestorCache_AncestorInfoByHeight(t *testing.T) { aInfo := &AncestorInfo{ Height: height, Hash: hash, - Block: &pb.BeaconBlock{Slot: height}, + Target: &pb.AttestationTarget{ + Slot: height, + BlockRoot: hash, + }, } fetchedInfo, err := cache.AncestorBySlot(hash, height) @@ -66,15 +73,15 @@ func TestAncestorCache_AncestorInfoByHeight(t *testing.T) { if fetchedInfo.Height != height { t.Errorf( "Expected fetched slot number to be %d, got %d", - aInfo.Height, - fetchedInfo.Height, + aInfo.Target.Slot, + fetchedInfo.Target.Slot, ) } - if !reflect.DeepEqual(fetchedInfo.Block, aInfo.Block) { + if !reflect.DeepEqual(fetchedInfo.Target, aInfo.Target) { t.Errorf( "Expected fetched info committee to be %v, got %v", - aInfo.Block, - fetchedInfo.Block, + aInfo.Target, + fetchedInfo.Target, ) } } @@ -85,6 +92,9 @@ func TestBlockAncestor_maxSize(t *testing.T) { for i := 0; i < maxCacheSize+10; i++ { aInfo := &AncestorInfo{ Height: uint64(i), + Target: &pb.AttestationTarget{ + Slot: uint64(i), + }, } if err := cache.AddBlockAncestor(aInfo); err != nil { t.Fatal(err) diff --git a/proto/beacon/p2p/v1/types.pb.go b/proto/beacon/p2p/v1/types.pb.go index 9064a2a1cc52..0995a83a4919 100755 --- a/proto/beacon/p2p/v1/types.pb.go +++ b/proto/beacon/p2p/v1/types.pb.go @@ -48,7 +48,7 @@ func (x Validator_StatusFlags) String() string { } func (Validator_StatusFlags) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{6, 0} + return fileDescriptor_e719e7d82cfa7b0d, []int{7, 0} } type BeaconState struct { @@ -638,6 +638,69 @@ func (m *AttestationData) GetJustifiedBlockRootHash32() []byte { return nil } +type AttestationTarget struct { + Slot uint64 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` + BlockRoot []byte `protobuf:"bytes,2,opt,name=block_root,json=blockRoot,proto3" json:"block_root,omitempty"` + ParentRoot []byte `protobuf:"bytes,3,opt,name=parent_root,json=parentRoot,proto3" json:"parent_root,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *AttestationTarget) Reset() { *m = AttestationTarget{} } +func (m *AttestationTarget) String() string { return proto.CompactTextString(m) } +func (*AttestationTarget) ProtoMessage() {} +func (*AttestationTarget) Descriptor() ([]byte, []int) { + return fileDescriptor_e719e7d82cfa7b0d, []int{5} +} +func (m *AttestationTarget) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *AttestationTarget) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_AttestationTarget.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *AttestationTarget) XXX_Merge(src proto.Message) { + xxx_messageInfo_AttestationTarget.Merge(m, src) +} +func (m *AttestationTarget) XXX_Size() int { + return m.Size() +} +func (m *AttestationTarget) XXX_DiscardUnknown() { + xxx_messageInfo_AttestationTarget.DiscardUnknown(m) +} + +var xxx_messageInfo_AttestationTarget proto.InternalMessageInfo + +func (m *AttestationTarget) GetSlot() uint64 { + if m != nil { + return m.Slot + } + return 0 +} + +func (m *AttestationTarget) GetBlockRoot() []byte { + if m != nil { + return m.BlockRoot + } + return nil +} + +func (m *AttestationTarget) GetParentRoot() []byte { + if m != nil { + return m.ParentRoot + } + return nil +} + type AttestationDataAndCustodyBit struct { Data *AttestationData `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` CustodyBit bool `protobuf:"varint,2,opt,name=custody_bit,json=custodyBit,proto3" json:"custody_bit,omitempty"` @@ -650,7 +713,7 @@ func (m *AttestationDataAndCustodyBit) Reset() { *m = AttestationDataAnd func (m *AttestationDataAndCustodyBit) String() string { return proto.CompactTextString(m) } func (*AttestationDataAndCustodyBit) ProtoMessage() {} func (*AttestationDataAndCustodyBit) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{5} + return fileDescriptor_e719e7d82cfa7b0d, []int{6} } func (m *AttestationDataAndCustodyBit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -710,7 +773,7 @@ func (m *Validator) Reset() { *m = Validator{} } func (m *Validator) String() string { return proto.CompactTextString(m) } func (*Validator) ProtoMessage() {} func (*Validator) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{6} + return fileDescriptor_e719e7d82cfa7b0d, []int{7} } func (m *Validator) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -801,7 +864,7 @@ func (m *ShardReassignmentRecord) Reset() { *m = ShardReassignmentRecord func (m *ShardReassignmentRecord) String() string { return proto.CompactTextString(m) } func (*ShardReassignmentRecord) ProtoMessage() {} func (*ShardReassignmentRecord) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{7} + return fileDescriptor_e719e7d82cfa7b0d, []int{8} } func (m *ShardReassignmentRecord) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -863,7 +926,7 @@ func (m *Crosslink) Reset() { *m = Crosslink{} } func (m *Crosslink) String() string { return proto.CompactTextString(m) } func (*Crosslink) ProtoMessage() {} func (*Crosslink) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{8} + return fileDescriptor_e719e7d82cfa7b0d, []int{9} } func (m *Crosslink) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -923,7 +986,7 @@ func (m *BeaconBlock) Reset() { *m = BeaconBlock{} } func (m *BeaconBlock) String() string { return proto.CompactTextString(m) } func (*BeaconBlock) ProtoMessage() {} func (*BeaconBlock) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{9} + return fileDescriptor_e719e7d82cfa7b0d, []int{10} } func (m *BeaconBlock) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1016,7 +1079,7 @@ func (m *BeaconBlockBody) Reset() { *m = BeaconBlockBody{} } func (m *BeaconBlockBody) String() string { return proto.CompactTextString(m) } func (*BeaconBlockBody) ProtoMessage() {} func (*BeaconBlockBody) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{10} + return fileDescriptor_e719e7d82cfa7b0d, []int{11} } func (m *BeaconBlockBody) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1093,7 +1156,7 @@ func (m *DepositInput) Reset() { *m = DepositInput{} } func (m *DepositInput) String() string { return proto.CompactTextString(m) } func (*DepositInput) ProtoMessage() {} func (*DepositInput) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{11} + return fileDescriptor_e719e7d82cfa7b0d, []int{12} } func (m *DepositInput) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1156,7 +1219,7 @@ func (m *ProposalSignedData) Reset() { *m = ProposalSignedData{} } func (m *ProposalSignedData) String() string { return proto.CompactTextString(m) } func (*ProposalSignedData) ProtoMessage() {} func (*ProposalSignedData) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{12} + return fileDescriptor_e719e7d82cfa7b0d, []int{13} } func (m *ProposalSignedData) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1220,7 +1283,7 @@ func (m *SlashableAttestation) Reset() { *m = SlashableAttestation{} } func (m *SlashableAttestation) String() string { return proto.CompactTextString(m) } func (*SlashableAttestation) ProtoMessage() {} func (*SlashableAttestation) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{13} + return fileDescriptor_e719e7d82cfa7b0d, []int{14} } func (m *SlashableAttestation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1290,7 +1353,7 @@ func (m *DepositData) Reset() { *m = DepositData{} } func (m *DepositData) String() string { return proto.CompactTextString(m) } func (*DepositData) ProtoMessage() {} func (*DepositData) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{14} + return fileDescriptor_e719e7d82cfa7b0d, []int{15} } func (m *DepositData) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1355,7 +1418,7 @@ func (m *ProposerSlashing) Reset() { *m = ProposerSlashing{} } func (m *ProposerSlashing) String() string { return proto.CompactTextString(m) } func (*ProposerSlashing) ProtoMessage() {} func (*ProposerSlashing) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{15} + return fileDescriptor_e719e7d82cfa7b0d, []int{16} } func (m *ProposerSlashing) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1431,7 +1494,7 @@ func (m *AttesterSlashing) Reset() { *m = AttesterSlashing{} } func (m *AttesterSlashing) String() string { return proto.CompactTextString(m) } func (*AttesterSlashing) ProtoMessage() {} func (*AttesterSlashing) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{16} + return fileDescriptor_e719e7d82cfa7b0d, []int{17} } func (m *AttesterSlashing) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1487,7 +1550,7 @@ func (m *Deposit) Reset() { *m = Deposit{} } func (m *Deposit) String() string { return proto.CompactTextString(m) } func (*Deposit) ProtoMessage() {} func (*Deposit) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{17} + return fileDescriptor_e719e7d82cfa7b0d, []int{18} } func (m *Deposit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1550,7 +1613,7 @@ func (m *VoluntaryExit) Reset() { *m = VoluntaryExit{} } func (m *VoluntaryExit) String() string { return proto.CompactTextString(m) } func (*VoluntaryExit) ProtoMessage() {} func (*VoluntaryExit) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{18} + return fileDescriptor_e719e7d82cfa7b0d, []int{19} } func (m *VoluntaryExit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1612,7 +1675,7 @@ func (m *Eth1Data) Reset() { *m = Eth1Data{} } func (m *Eth1Data) String() string { return proto.CompactTextString(m) } func (*Eth1Data) ProtoMessage() {} func (*Eth1Data) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{19} + return fileDescriptor_e719e7d82cfa7b0d, []int{20} } func (m *Eth1Data) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1667,7 +1730,7 @@ func (m *Eth1DataVote) Reset() { *m = Eth1DataVote{} } func (m *Eth1DataVote) String() string { return proto.CompactTextString(m) } func (*Eth1DataVote) ProtoMessage() {} func (*Eth1DataVote) Descriptor() ([]byte, []int) { - return fileDescriptor_e719e7d82cfa7b0d, []int{20} + return fileDescriptor_e719e7d82cfa7b0d, []int{21} } func (m *Eth1DataVote) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1717,6 +1780,7 @@ func init() { proto.RegisterType((*PendingAttestation)(nil), "ethereum.beacon.p2p.v1.PendingAttestation") proto.RegisterType((*Attestation)(nil), "ethereum.beacon.p2p.v1.Attestation") proto.RegisterType((*AttestationData)(nil), "ethereum.beacon.p2p.v1.AttestationData") + proto.RegisterType((*AttestationTarget)(nil), "ethereum.beacon.p2p.v1.AttestationTarget") proto.RegisterType((*AttestationDataAndCustodyBit)(nil), "ethereum.beacon.p2p.v1.AttestationDataAndCustodyBit") proto.RegisterType((*Validator)(nil), "ethereum.beacon.p2p.v1.Validator") proto.RegisterType((*ShardReassignmentRecord)(nil), "ethereum.beacon.p2p.v1.ShardReassignmentRecord") @@ -1738,133 +1802,135 @@ func init() { func init() { proto.RegisterFile("proto/beacon/p2p/v1/types.proto", fileDescriptor_e719e7d82cfa7b0d) } var fileDescriptor_e719e7d82cfa7b0d = []byte{ - // 2009 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x58, 0x59, 0x6f, 0x1c, 0x59, - 0x15, 0xa6, 0xdc, 0x5e, 0x4f, 0xaf, 0xbe, 0x8e, 0xdd, 0x45, 0x36, 0x3b, 0x95, 0x0c, 0x71, 0xc2, - 0x4c, 0x9b, 0xee, 0x91, 0x88, 0x20, 0x8c, 0x84, 0x3b, 0xf6, 0x30, 0x86, 0xcc, 0x8c, 0x55, 0x6d, - 0x12, 0x1e, 0x80, 0xd2, 0xed, 0xae, 0xdb, 0xdd, 0x15, 0x57, 0xd7, 0x2d, 0xd5, 0xbd, 0xdd, 0x13, - 0x23, 0xfe, 0x00, 0x83, 0xc4, 0x1b, 0x0f, 0xf0, 0xc6, 0xf2, 0x27, 0xd8, 0x79, 0x41, 0xe2, 0x91, - 0x75, 0x10, 0x12, 0x42, 0x28, 0xcf, 0xec, 0xfc, 0x01, 0x74, 0x97, 0x5a, 0x7a, 0xb3, 0x13, 0x86, - 0x97, 0x79, 0x6a, 0xd5, 0x39, 0xdf, 0x39, 0xf7, 0x9c, 0x73, 0xcf, 0x76, 0x1b, 0xb6, 0xc3, 0x88, - 0x72, 0xba, 0xd7, 0x26, 0xb8, 0x43, 0x83, 0xbd, 0xb0, 0x11, 0xee, 0x8d, 0xea, 0x7b, 0xfc, 0x2c, - 0x24, 0xac, 0x26, 0x39, 0x68, 0x8b, 0xf0, 0x3e, 0x89, 0xc8, 0x70, 0x50, 0x53, 0x98, 0x5a, 0xd8, - 0x08, 0x6b, 0xa3, 0xfa, 0xe5, 0x2b, 0x4a, 0xb0, 0x43, 0x07, 0x03, 0x1a, 0xec, 0x0d, 0x08, 0x63, - 0xb8, 0x17, 0x0b, 0x59, 0xff, 0x29, 0x42, 0xbe, 0x29, 0xe1, 0x2d, 0x8e, 0x39, 0x41, 0xc7, 0x80, - 0x46, 0xd8, 0xf7, 0x5c, 0xcc, 0x69, 0xe4, 0x44, 0xa4, 0xe7, 0x31, 0x1e, 0x9d, 0x99, 0xc6, 0x4e, - 0x6e, 0x37, 0xdf, 0xb8, 0x51, 0x9b, 0x7d, 0x42, 0xed, 0x51, 0x2c, 0x61, 0xaf, 0x27, 0xc2, 0xb6, - 0x96, 0x45, 0x87, 0xb0, 0x3d, 0xad, 0xd1, 0x19, 0x86, 0x2e, 0xe6, 0xc4, 0x21, 0x21, 0xed, 0xf4, - 0xcd, 0x85, 0x1d, 0x63, 0x77, 0xd1, 0xbe, 0x3a, 0x25, 0xfb, 0x79, 0x09, 0x3a, 0x14, 0x18, 0xf4, - 0x4a, 0xd6, 0xb0, 0x36, 0xf6, 0x71, 0xd0, 0x21, 0xcc, 0xcc, 0xed, 0xe4, 0x76, 0x17, 0x33, 0xa7, - 0x36, 0x35, 0x03, 0xed, 0xc1, 0x86, 0x8f, 0x39, 0x61, 0xdc, 0x89, 0x70, 0xe0, 0x62, 0xea, 0x0c, - 0xbc, 0xa7, 0x84, 0x99, 0x7f, 0x5d, 0xd9, 0xc9, 0xed, 0x16, 0xec, 0x75, 0xc5, 0xb3, 0x25, 0xeb, - 0x4d, 0xc1, 0x41, 0x07, 0x70, 0x3d, 0x8c, 0xc8, 0xc8, 0xa3, 0x43, 0xe6, 0xb0, 0xfe, 0xb0, 0xdb, - 0xf5, 0xbd, 0xa0, 0xe7, 0x30, 0x8e, 0x23, 0xee, 0xb0, 0x3e, 0x8e, 0x5c, 0xf3, 0x6f, 0x2b, 0xd2, - 0xcc, 0x2b, 0x31, 0xac, 0x15, 0xa3, 0x5a, 0x02, 0xd4, 0x12, 0x18, 0xd4, 0x84, 0x6b, 0x9d, 0x61, - 0x14, 0x91, 0x80, 0xcf, 0x51, 0xf2, 0x77, 0xa5, 0xe4, 0xb2, 0x46, 0xcd, 0xd2, 0xf1, 0x09, 0x30, - 0x67, 0x58, 0xa2, 0x22, 0xf5, 0x0f, 0x25, 0xbe, 0x35, 0x65, 0x83, 0x0a, 0xd2, 0x3d, 0xa8, 0x4e, - 0x1f, 0xaf, 0x24, 0xff, 0xa9, 0x24, 0x37, 0x27, 0x0f, 0x56, 0x82, 0x73, 0xbc, 0x27, 0xc4, 0x75, - 0xfa, 0x98, 0xf5, 0x5f, 0x6d, 0x98, 0xff, 0x12, 0xf2, 0x85, 0x59, 0xde, 0x13, 0xe2, 0xbe, 0x21, - 0x31, 0x73, 0xbc, 0xcf, 0x28, 0xf9, 0xb7, 0x52, 0x32, 0xed, 0x7d, 0xaa, 0x23, 0xeb, 0xfd, 0x93, - 0x21, 0xe3, 0x5e, 0xd7, 0x23, 0xae, 0xf6, 0xe1, 0xd7, 0xe5, 0x71, 0xef, 0x3f, 0x1b, 0xf3, 0x13, - 0xef, 0x67, 0x88, 0x46, 0x94, 0x72, 0xf3, 0x37, 0x65, 0x79, 0xf0, 0xe6, 0x94, 0xa4, 0x4d, 0x29, - 0x47, 0xbb, 0x50, 0x9e, 0x3c, 0xea, 0xb7, 0xea, 0xa8, 0xd2, 0x93, 0xf1, 0x23, 0x3e, 0x02, 0xa5, - 0x09, 0xcd, 0xbf, 0x53, 0x9a, 0x8b, 0x4f, 0xc6, 0x34, 0x7e, 0x1c, 0xb6, 0x34, 0xa1, 0x83, 0xb9, - 0x47, 0x03, 0xa7, 0xed, 0xf1, 0xae, 0x47, 0x7c, 0xd7, 0xfc, 0xbd, 0x52, 0xbc, 0x39, 0xc6, 0x6e, - 0x6a, 0xae, 0xb0, 0xa4, 0xeb, 0x05, 0xd8, 0xf7, 0xbe, 0x92, 0x58, 0xf2, 0x9e, 0xb6, 0x24, 0xa1, - 0x27, 0x96, 0xa4, 0x48, 0x69, 0xc9, 0x1f, 0xb4, 0x25, 0x09, 0x59, 0x5a, 0xf2, 0x36, 0xe8, 0x64, - 0x77, 0x3a, 0x11, 0x65, 0xcc, 0xf7, 0x82, 0x53, 0x66, 0xfe, 0xa0, 0x7a, 0x7e, 0x41, 0x3f, 0x88, - 0xa1, 0x76, 0x45, 0x09, 0x27, 0x04, 0x86, 0x3e, 0x09, 0x1f, 0xd6, 0x0a, 0xdb, 0x3e, 0xed, 0x9c, - 0xca, 0xb3, 0xf5, 0xfd, 0x32, 0xf3, 0x47, 0x55, 0x59, 0x5f, 0x5b, 0x0a, 0xd1, 0x14, 0x00, 0x61, - 0x85, 0xba, 0x5b, 0x86, 0x3e, 0x05, 0x97, 0xdb, 0x98, 0x77, 0xfa, 0xc4, 0x9d, 0x25, 0xfc, 0x63, - 0x25, 0x5c, 0xd5, 0x90, 0x29, 0xe9, 0x7b, 0x50, 0xd5, 0x27, 0x33, 0x1f, 0x33, 0xa9, 0x24, 0xee, - 0x03, 0x3f, 0xa9, 0xca, 0x46, 0xb0, 0xa9, 0xf8, 0x2d, 0xc5, 0x4e, 0x9a, 0xc1, 0x17, 0x93, 0x66, - 0x80, 0xb9, 0xf8, 0x91, 0x31, 0x67, 0xe6, 0x4f, 0x55, 0x14, 0xee, 0xce, 0x8b, 0xc2, 0x31, 0x09, - 0x5c, 0x2f, 0xe8, 0xed, 0xa7, 0x32, 0x36, 0x52, 0x7a, 0x32, 0xa4, 0x6c, 0x40, 0xbc, 0xc0, 0x25, - 0x4f, 0xc7, 0x7d, 0xfa, 0xd9, 0x58, 0x40, 0x8e, 0x04, 0x20, 0xeb, 0xd2, 0x67, 0xa0, 0x90, 0x0d, - 0xa6, 0xf9, 0xf3, 0xea, 0x8e, 0xb1, 0x9b, 0x6f, 0xdc, 0x9c, 0x67, 0x92, 0x6a, 0xd5, 0x2a, 0x32, - 0xf9, 0x4c, 0x90, 0xd1, 0xe7, 0x40, 0xdf, 0x94, 0x43, 0x78, 0xbf, 0xee, 0xb8, 0x98, 0x63, 0xf3, - 0x3b, 0xdb, 0x52, 0xd9, 0xce, 0x3c, 0x65, 0x87, 0xbc, 0x5f, 0x3f, 0xc0, 0x1c, 0xdb, 0x25, 0x25, - 0x1a, 0x7f, 0xa3, 0x37, 0xa1, 0x9c, 0x68, 0x71, 0x46, 0x94, 0x13, 0x66, 0x7e, 0x77, 0x5b, 0xc6, - 0xea, 0xd6, 0x45, 0xba, 0x1e, 0x51, 0x4e, 0xec, 0x22, 0xc9, 0x7c, 0x31, 0x74, 0x0b, 0x8a, 0x2e, - 0x09, 0x29, 0xf3, 0x74, 0x84, 0xcc, 0xef, 0x6d, 0xcb, 0x94, 0x2e, 0x68, 0xaa, 0x8c, 0x0a, 0xb2, - 0xa0, 0xd0, 0x23, 0x01, 0x61, 0x1e, 0x73, 0xb8, 0x37, 0x20, 0xe6, 0xd7, 0x6e, 0x4b, 0x50, 0x5e, - 0x13, 0x4f, 0xbc, 0x01, 0x41, 0x75, 0x58, 0xec, 0xd2, 0xe8, 0xd4, 0x7c, 0xf7, 0xb6, 0xf4, 0xec, - 0xea, 0x3c, 0x6b, 0x5e, 0xa7, 0xd1, 0xa9, 0x2d, 0xa1, 0x68, 0x03, 0x16, 0x99, 0x4f, 0xb9, 0xf9, - 0x75, 0xa5, 0x4e, 0x7e, 0x58, 0x21, 0x2c, 0x0a, 0x08, 0xba, 0x03, 0x95, 0xa4, 0x63, 0x8c, 0x48, - 0xc4, 0x3c, 0x1a, 0x98, 0x86, 0xc4, 0x95, 0x63, 0xfa, 0x23, 0x45, 0x46, 0xb7, 0xa1, 0x1c, 0xf7, - 0xb6, 0x18, 0xa9, 0xc6, 0x56, 0x49, 0x93, 0x63, 0xe0, 0x25, 0x58, 0x52, 0x85, 0x9b, 0x93, 0x6c, - 0xf5, 0x61, 0xbd, 0x67, 0x00, 0x9a, 0xce, 0x27, 0x74, 0x1f, 0x16, 0xe5, 0x55, 0x19, 0xd2, 0x9f, - 0xdb, 0xf3, 0xfc, 0xc9, 0x88, 0xc8, 0x0b, 0x93, 0x42, 0xa8, 0x0e, 0x97, 0x70, 0xaf, 0x17, 0x91, - 0xde, 0x44, 0x8b, 0x59, 0x90, 0x7d, 0x60, 0x23, 0xc3, 0x4b, 0xfa, 0xcb, 0x1d, 0xa8, 0x74, 0x86, - 0x8c, 0x53, 0xf7, 0x2c, 0x85, 0xe7, 0x24, 0xbc, 0xac, 0xe9, 0x09, 0xf4, 0x25, 0x28, 0x79, 0x41, - 0xc7, 0x1f, 0x0a, 0xa7, 0x1c, 0x19, 0xc2, 0x45, 0xe9, 0x50, 0x31, 0xa1, 0xb6, 0x44, 0x28, 0xff, - 0x68, 0x40, 0xfe, 0x03, 0xe2, 0xd1, 0x1e, 0x24, 0x1a, 0x88, 0xc3, 0xbc, 0x5e, 0x80, 0xf9, 0x30, - 0x22, 0xd2, 0xad, 0x82, 0x8d, 0x12, 0x56, 0x2b, 0xe6, 0x58, 0xdf, 0xcf, 0x41, 0x79, 0xc2, 0x50, - 0x84, 0x74, 0x3e, 0x19, 0x69, 0x3a, 0x89, 0x2b, 0x57, 0xd3, 0x5d, 0x65, 0x84, 0xfa, 0x40, 0xf7, - 0xc0, 0x54, 0x3e, 0x4f, 0xf7, 0x3a, 0x6d, 0xe1, 0x66, 0x3b, 0x53, 0xce, 0x49, 0x57, 0x40, 0xf7, - 0xe1, 0xb2, 0x4c, 0x1a, 0xa7, 0x4d, 0x87, 0x81, 0x8b, 0xa3, 0xb3, 0x31, 0x51, 0x65, 0x6e, 0x55, - 0x22, 0x9a, 0x1a, 0x30, 0x2e, 0x9c, 0x34, 0x7a, 0x55, 0xc0, 0x59, 0xe1, 0x25, 0x25, 0x9c, 0x20, - 0x64, 0xec, 0x53, 0xe1, 0x87, 0x49, 0x17, 0x49, 0x10, 0xe6, 0xb2, 0xbc, 0xc8, 0xe7, 0x18, 0x15, - 0xe5, 0x89, 0x51, 0x21, 0x4a, 0x66, 0x72, 0xac, 0xae, 0xcc, 0x9c, 0xaa, 0xaf, 0xc1, 0x95, 0x14, - 0x38, 0x1d, 0xac, 0x55, 0x69, 0xb4, 0x99, 0x40, 0x26, 0xe2, 0x65, 0x7d, 0x15, 0xae, 0x4e, 0xdc, - 0xd2, 0x7e, 0xe0, 0x3e, 0x48, 0x2e, 0xff, 0xfd, 0xa5, 0xe4, 0x36, 0xe4, 0x33, 0xf9, 0x25, 0x6f, - 0x78, 0xd5, 0x86, 0x34, 0xb5, 0xac, 0x6f, 0xe6, 0x60, 0x2d, 0x59, 0x80, 0xd1, 0x16, 0x2c, 0x87, - 0xc3, 0xf6, 0x29, 0x39, 0x93, 0xa7, 0x15, 0x6c, 0xfd, 0x25, 0x56, 0xa3, 0x77, 0x3c, 0xde, 0x77, - 0x23, 0xfc, 0x0e, 0xf6, 0x9d, 0x4e, 0x44, 0x5c, 0x12, 0x70, 0x0f, 0xfb, 0x2c, 0x76, 0x52, 0xa5, - 0xf8, 0x95, 0x14, 0xf4, 0x20, 0xc5, 0xe8, 0xdb, 0xb9, 0x03, 0x15, 0xdc, 0xe1, 0xde, 0x48, 0x15, - 0x87, 0x0a, 0xe8, 0x92, 0xea, 0x56, 0x29, 0x5d, 0x45, 0xf4, 0x1a, 0x00, 0x79, 0xea, 0x71, 0x0d, - 0x5a, 0x96, 0xa0, 0x35, 0x41, 0x51, 0xec, 0x3b, 0x50, 0xc9, 0x58, 0x93, 0xbd, 0x9a, 0x72, 0x4a, - 0x57, 0xd0, 0x9b, 0x50, 0x8c, 0xa7, 0xad, 0xc2, 0xad, 0xaa, 0xde, 0xad, 0x89, 0x0a, 0x74, 0x0c, - 0x05, 0x11, 0xb9, 0x21, 0x73, 0xba, 0x3e, 0xee, 0x31, 0x73, 0x6d, 0xc7, 0xd8, 0x2d, 0x35, 0x5e, - 0xb9, 0xf0, 0xbd, 0x50, 0x6b, 0x49, 0xa9, 0xd7, 0x85, 0x90, 0x9d, 0x67, 0xe9, 0x87, 0xf5, 0x69, - 0xc8, 0x67, 0x78, 0x28, 0x0f, 0x2b, 0x47, 0x6f, 0x1d, 0x9d, 0x1c, 0xed, 0x3f, 0xac, 0x7c, 0x08, - 0x21, 0x28, 0xa9, 0x8f, 0x93, 0xc3, 0x03, 0xe7, 0xf0, 0x0b, 0x47, 0x27, 0x15, 0x03, 0x55, 0xa0, - 0xf0, 0xf8, 0xe8, 0xe4, 0x8d, 0x03, 0x7b, 0xff, 0xf1, 0x7e, 0xf3, 0xe1, 0x61, 0x65, 0xc1, 0xf2, - 0xa1, 0x2a, 0xf7, 0x69, 0x9b, 0x60, 0x26, 0x8a, 0x7d, 0x40, 0x02, 0x6e, 0x93, 0x0e, 0x8d, 0x5c, - 0x91, 0x98, 0xe9, 0x5b, 0x42, 0x8d, 0x24, 0x55, 0xce, 0xa5, 0x84, 0xac, 0x66, 0xd2, 0xec, 0xc2, - 0x8e, 0x5b, 0x40, 0x2e, 0x33, 0x51, 0xbe, 0x0c, 0x6b, 0x69, 0xe2, 0x27, 0x23, 0xc0, 0xc8, 0x8c, - 0x80, 0x0b, 0x2a, 0x73, 0xe1, 0xdc, 0xca, 0xb4, 0x7e, 0xb8, 0x10, 0xbf, 0xd3, 0xd4, 0xbc, 0x9f, - 0xd5, 0x86, 0x5e, 0x06, 0x14, 0x62, 0x39, 0xa1, 0xa6, 0x15, 0x57, 0x14, 0x27, 0x53, 0xeb, 0x77, - 0x61, 0x5d, 0x04, 0x9c, 0xcc, 0xe8, 0x4b, 0x65, 0xc9, 0xc8, 0x60, 0x6f, 0x42, 0x51, 0x3f, 0xa3, - 0x22, 0x32, 0x22, 0xd8, 0xd7, 0x4d, 0xa8, 0xa0, 0x88, 0xb6, 0xa4, 0xa1, 0xd7, 0x60, 0x2d, 0xdd, - 0x3d, 0x96, 0x9e, 0x73, 0xf5, 0x58, 0x8d, 0x57, 0x05, 0x74, 0x15, 0xd6, 0xd2, 0x9e, 0xbc, 0x2c, - 0xf5, 0xa7, 0x04, 0x51, 0xc3, 0x6d, 0xea, 0x9e, 0xc9, 0x2c, 0x3d, 0xa7, 0x86, 0x33, 0x21, 0x6a, - 0x52, 0xf7, 0xcc, 0x96, 0x42, 0xd6, 0xb7, 0x72, 0x50, 0x9e, 0xe0, 0x88, 0xcd, 0x6b, 0x6c, 0x19, - 0x54, 0x4f, 0xdc, 0x9b, 0xcf, 0xd1, 0x1c, 0xec, 0x31, 0x41, 0xf4, 0x18, 0x50, 0x18, 0xd1, 0x90, - 0x32, 0x12, 0xa9, 0xbd, 0xd4, 0x0b, 0x7a, 0xcc, 0x5c, 0x90, 0xea, 0x76, 0xe7, 0xae, 0x96, 0x5a, - 0xa2, 0xa5, 0x05, 0xec, 0xf5, 0x70, 0x82, 0x22, 0x15, 0xab, 0x83, 0xc6, 0x14, 0xe7, 0xce, 0x57, - 0xbc, 0xaf, 0x25, 0x52, 0xc5, 0x78, 0x82, 0xc2, 0xd0, 0x7d, 0x58, 0xd5, 0x9b, 0x17, 0x33, 0x17, - 0xa5, 0xba, 0xed, 0x79, 0xea, 0x0e, 0x14, 0xce, 0x4e, 0x04, 0xd0, 0x5b, 0x50, 0x1e, 0x51, 0x7f, - 0x18, 0x70, 0x31, 0x97, 0x44, 0x47, 0x61, 0xe6, 0x92, 0xd4, 0xf1, 0xd2, 0xdc, 0x6a, 0x8f, 0xe1, - 0x87, 0x4f, 0x3d, 0x6e, 0x97, 0x46, 0xd9, 0x4f, 0x66, 0x7d, 0xdb, 0x80, 0xc2, 0x41, 0xbc, 0x07, - 0x86, 0x43, 0x3e, 0xb7, 0x83, 0xd6, 0x60, 0x23, 0x8c, 0x28, 0xed, 0x3a, 0xb4, 0xeb, 0x84, 0x94, - 0x31, 0xc2, 0x92, 0x25, 0xac, 0x20, 0xc3, 0x47, 0xbb, 0x6f, 0x77, 0x8f, 0x13, 0xc6, 0xc5, 0x1d, - 0x37, 0x77, 0x61, 0xc7, 0xb5, 0x9e, 0x00, 0x52, 0x37, 0x85, 0x7d, 0xb1, 0x15, 0x10, 0xf7, 0x05, - 0x57, 0x80, 0xbb, 0xb0, 0x3e, 0x6f, 0xf6, 0x97, 0xdb, 0x13, 0x53, 0xec, 0x4f, 0x06, 0x5c, 0x92, - 0x77, 0x84, 0xdb, 0x3e, 0xc9, 0x6e, 0x54, 0x1f, 0x85, 0xf5, 0xb1, 0x6e, 0xe5, 0x89, 0x07, 0x8f, - 0x21, 0xdf, 0x3b, 0x95, 0x6c, 0xbf, 0x12, 0xf4, 0x99, 0xeb, 0xd0, 0xc2, 0xec, 0x75, 0x28, 0x1e, - 0x8b, 0xb9, 0xff, 0x65, 0x2c, 0xbe, 0xf0, 0x2e, 0xf5, 0x0d, 0x03, 0xf2, 0xfa, 0x9e, 0x65, 0x10, - 0x8f, 0xb2, 0x8f, 0x82, 0x70, 0xc8, 0xf5, 0x74, 0xbe, 0x75, 0x41, 0x26, 0xca, 0x1c, 0xc9, 0xbc, - 0x1c, 0x74, 0xc6, 0xe0, 0x01, 0x1d, 0x06, 0x5c, 0x07, 0x5f, 0x7f, 0x89, 0x8e, 0x22, 0x5e, 0x12, - 0x8c, 0xe3, 0x41, 0xa8, 0x9b, 0x75, 0x4a, 0xb0, 0x7e, 0xb1, 0x00, 0x95, 0xc9, 0x32, 0x14, 0x4b, - 0x6f, 0x52, 0xcc, 0xd9, 0xc1, 0x50, 0x8c, 0xa9, 0x6a, 0x2e, 0xd8, 0x50, 0x0e, 0x75, 0x5e, 0xa8, - 0x4e, 0x5e, 0x97, 0x47, 0x9f, 0xf7, 0x96, 0x9c, 0x4a, 0xa3, 0x58, 0x27, 0xf6, 0xc5, 0x57, 0x1d, - 0x7d, 0x0c, 0x2e, 0x25, 0x3a, 0x93, 0x80, 0x3a, 0x75, 0x9d, 0x2e, 0x28, 0xcc, 0x28, 0x90, 0xac, - 0xfa, 0xb4, 0x15, 0x6a, 0x39, 0x7c, 0x1f, 0x56, 0x34, 0xe6, 0x58, 0x11, 0x2f, 0x8e, 0xd3, 0x56, - 0x34, 0xac, 0x3f, 0x1b, 0x50, 0x99, 0xec, 0x3a, 0xc8, 0x85, 0x2a, 0x8b, 0x73, 0x39, 0xfb, 0xe8, - 0x76, 0xea, 0xfa, 0x9e, 0x5f, 0x9e, 0x67, 0xe2, 0xac, 0x12, 0xb0, 0x37, 0xd9, 0x0c, 0x6a, 0x7d, - 0xfe, 0x29, 0x0d, 0x7d, 0x1d, 0xff, 0x87, 0x53, 0x1a, 0xd6, 0xbb, 0x06, 0xac, 0xe8, 0xec, 0x13, - 0xe1, 0x19, 0x90, 0xe8, 0xd4, 0x27, 0x8e, 0xea, 0x45, 0xf1, 0x33, 0xdf, 0x90, 0xaf, 0x7c, 0xa4, - 0x78, 0xc7, 0x82, 0x15, 0xbf, 0xf0, 0xef, 0xc2, 0xba, 0x96, 0xe0, 0x11, 0x21, 0x3a, 0xa9, 0x54, - 0x9e, 0x96, 0x15, 0xe3, 0x24, 0x22, 0x44, 0xa5, 0xd5, 0x0d, 0x88, 0x13, 0xdb, 0x49, 0x2a, 0xb3, - 0x60, 0xe7, 0xdd, 0xb4, 0x6c, 0x2c, 0x1f, 0x8a, 0x63, 0xfd, 0x74, 0xce, 0xae, 0x31, 0x63, 0xc3, - 0x59, 0x98, 0xb9, 0xe1, 0x8c, 0x4d, 0xdd, 0xdc, 0xc4, 0xd4, 0xb5, 0xbe, 0x04, 0xab, 0xc9, 0x9f, - 0x02, 0x35, 0xd8, 0x88, 0x8d, 0xcb, 0x76, 0x33, 0xd5, 0xa4, 0xd7, 0x35, 0x2b, 0xb3, 0x33, 0xdc, - 0x80, 0x82, 0xea, 0x7d, 0x63, 0x7b, 0x48, 0x5e, 0xd2, 0x74, 0xcb, 0xf3, 0xa1, 0x90, 0xfd, 0xdf, - 0x60, 0x7c, 0x83, 0x30, 0x5e, 0x78, 0x83, 0xb8, 0x06, 0x30, 0xa2, 0x9c, 0x38, 0x9d, 0x4c, 0x2f, - 0x58, 0x13, 0x94, 0x07, 0x82, 0xd0, 0x2c, 0xfc, 0xf2, 0xd9, 0x75, 0xe3, 0x57, 0xcf, 0xae, 0x1b, - 0x7f, 0x79, 0x76, 0xdd, 0x68, 0x2f, 0xcb, 0xff, 0xbf, 0x5f, 0xfd, 0x6f, 0x00, 0x00, 0x00, 0xff, - 0xff, 0xc4, 0x8d, 0x74, 0xea, 0x57, 0x17, 0x00, 0x00, + // 2048 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x58, 0x5b, 0x6f, 0x23, 0x49, + 0x15, 0xa6, 0xe3, 0x5c, 0x8f, 0x9d, 0xd8, 0xa9, 0x4c, 0xe2, 0x66, 0x6e, 0xc9, 0xf4, 0xcc, 0x32, + 0x99, 0x61, 0x37, 0xc1, 0x5e, 0x89, 0x11, 0x0c, 0x2b, 0x91, 0x4c, 0xb2, 0x6c, 0x60, 0x76, 0x37, + 0x6a, 0x87, 0x19, 0x1e, 0x80, 0x56, 0xd9, 0x5d, 0xb6, 0x7b, 0xd2, 0xee, 0x6a, 0x75, 0x95, 0xbd, + 0x13, 0xc4, 0x1f, 0x60, 0x91, 0x78, 0xe3, 0x01, 0xde, 0xb8, 0xfc, 0x09, 0xee, 0xbc, 0x20, 0xf1, + 0xc8, 0x75, 0x11, 0x12, 0x42, 0x68, 0x9e, 0xb9, 0xf3, 0x07, 0x50, 0x5d, 0xba, 0xbb, 0x7c, 0x4b, + 0x66, 0x58, 0x5e, 0xf6, 0xc9, 0xea, 0x73, 0xbe, 0x73, 0xea, 0xd4, 0xa9, 0x53, 0xdf, 0x39, 0x65, + 0xd8, 0x8c, 0x13, 0xca, 0xe9, 0x6e, 0x93, 0xe0, 0x16, 0x8d, 0x76, 0xe3, 0x7a, 0xbc, 0x3b, 0xa8, + 0xed, 0xf2, 0xb3, 0x98, 0xb0, 0x1d, 0xa9, 0x41, 0x1b, 0x84, 0x77, 0x49, 0x42, 0xfa, 0xbd, 0x1d, + 0x85, 0xd9, 0x89, 0xeb, 0xf1, 0xce, 0xa0, 0x76, 0xf9, 0x8a, 0x32, 0x6c, 0xd1, 0x5e, 0x8f, 0x46, + 0xbb, 0x3d, 0xc2, 0x18, 0xee, 0xa4, 0x46, 0xce, 0x7f, 0x96, 0xa1, 0xb8, 0x2f, 0xe1, 0x0d, 0x8e, + 0x39, 0x41, 0xc7, 0x80, 0x06, 0x38, 0x0c, 0x7c, 0xcc, 0x69, 0xe2, 0x25, 0xa4, 0x13, 0x30, 0x9e, + 0x9c, 0xd9, 0xd6, 0x56, 0x61, 0xbb, 0x58, 0xbf, 0xb1, 0x33, 0x79, 0x85, 0x9d, 0x47, 0xa9, 0x85, + 0xbb, 0x9a, 0x19, 0xbb, 0xda, 0x16, 0x1d, 0xc2, 0xe6, 0xb8, 0x47, 0xaf, 0x1f, 0xfb, 0x98, 0x13, + 0x8f, 0xc4, 0xb4, 0xd5, 0xb5, 0x67, 0xb6, 0xac, 0xed, 0x59, 0xf7, 0xea, 0x98, 0xed, 0xe7, 0x25, + 0xe8, 0x50, 0x60, 0xd0, 0x2b, 0x66, 0x60, 0x4d, 0x1c, 0xe2, 0xa8, 0x45, 0x98, 0x5d, 0xd8, 0x2a, + 0x6c, 0xcf, 0x1a, 0xab, 0xee, 0x6b, 0x05, 0xda, 0x85, 0xb5, 0x10, 0x73, 0xc2, 0xb8, 0x97, 0xe0, + 0xc8, 0xc7, 0xd4, 0xeb, 0x05, 0x4f, 0x09, 0xb3, 0xff, 0xba, 0xb0, 0x55, 0xd8, 0x2e, 0xb9, 0xab, + 0x4a, 0xe7, 0x4a, 0xd5, 0x9b, 0x42, 0x83, 0x0e, 0xe0, 0x7a, 0x9c, 0x90, 0x41, 0x40, 0xfb, 0xcc, + 0x63, 0xdd, 0x7e, 0xbb, 0x1d, 0x06, 0x51, 0xc7, 0x63, 0x1c, 0x27, 0xdc, 0x63, 0x5d, 0x9c, 0xf8, + 0xf6, 0xdf, 0x16, 0x64, 0x98, 0x57, 0x52, 0x58, 0x23, 0x45, 0x35, 0x04, 0xa8, 0x21, 0x30, 0x68, + 0x1f, 0xae, 0xb5, 0xfa, 0x49, 0x42, 0x22, 0x3e, 0xc5, 0xc9, 0xdf, 0x95, 0x93, 0xcb, 0x1a, 0x35, + 0xc9, 0xc7, 0x27, 0xc0, 0x9e, 0x10, 0x89, 0xca, 0xd4, 0x3f, 0x94, 0xf9, 0xc6, 0x58, 0x0c, 0x2a, + 0x49, 0xf7, 0xa0, 0x3a, 0xbe, 0xbc, 0xb2, 0xfc, 0xa7, 0xb2, 0x5c, 0x1f, 0x5d, 0x58, 0x19, 0x4e, + 0xd9, 0x3d, 0x21, 0xbe, 0xd7, 0xc5, 0xac, 0xfb, 0x6a, 0xdd, 0xfe, 0x97, 0xb0, 0x2f, 0x4d, 0xda, + 0x3d, 0x21, 0xfe, 0x1b, 0x12, 0x33, 0x65, 0xf7, 0x86, 0x93, 0x7f, 0x2b, 0x27, 0xe3, 0xbb, 0xcf, + 0x7d, 0x98, 0xbb, 0x7f, 0xd2, 0x67, 0x3c, 0x68, 0x07, 0xc4, 0xd7, 0x7b, 0xf8, 0x75, 0x79, 0x78, + 0xf7, 0x9f, 0x4d, 0xf5, 0xd9, 0xee, 0x27, 0x98, 0x26, 0x94, 0x72, 0xfb, 0x37, 0x65, 0xb9, 0xf0, + 0xfa, 0x98, 0xa5, 0x4b, 0x29, 0x47, 0xdb, 0x50, 0x1e, 0x5d, 0xea, 0xb7, 0x6a, 0xa9, 0x95, 0x27, + 0xc3, 0x4b, 0x7c, 0x04, 0x56, 0x46, 0x3c, 0xff, 0x4e, 0x79, 0x5e, 0x7e, 0x32, 0xe4, 0xf1, 0xe3, + 0xb0, 0xa1, 0x05, 0x2d, 0xcc, 0x03, 0x1a, 0x79, 0xcd, 0x80, 0xb7, 0x03, 0x12, 0xfa, 0xf6, 0xef, + 0x95, 0xe3, 0xf5, 0x21, 0xf5, 0xbe, 0xd6, 0x8a, 0x48, 0xda, 0x41, 0x84, 0xc3, 0xe0, 0x2b, 0x59, + 0x24, 0xef, 0xe9, 0x48, 0x32, 0x79, 0x16, 0x49, 0x8e, 0x94, 0x91, 0xfc, 0x41, 0x47, 0x92, 0x89, + 0x65, 0x24, 0x6f, 0x83, 0x2e, 0x76, 0xaf, 0x95, 0x50, 0xc6, 0xc2, 0x20, 0x3a, 0x65, 0xf6, 0x0f, + 0xaa, 0xe7, 0x5f, 0xe8, 0x07, 0x29, 0xd4, 0xad, 0x28, 0xe3, 0x4c, 0xc0, 0xd0, 0x27, 0xe1, 0xc3, + 0xda, 0x61, 0x33, 0xa4, 0xad, 0x53, 0xb9, 0xb6, 0x3e, 0x5f, 0x66, 0xff, 0xa8, 0x2a, 0xef, 0xd7, + 0x86, 0x42, 0xec, 0x0b, 0x80, 0x88, 0x42, 0x9d, 0x2d, 0x43, 0x9f, 0x82, 0xcb, 0x4d, 0xcc, 0x5b, + 0x5d, 0xe2, 0x4f, 0x32, 0xfe, 0xb1, 0x32, 0xae, 0x6a, 0xc8, 0x98, 0xf5, 0x3d, 0xa8, 0xea, 0x95, + 0x59, 0x88, 0x99, 0x74, 0x92, 0xf2, 0xc0, 0x4f, 0xaa, 0x92, 0x08, 0xd6, 0x95, 0xbe, 0xa1, 0xd4, + 0x19, 0x19, 0x7c, 0x31, 0x23, 0x03, 0xcc, 0xc5, 0x8f, 0xcc, 0x39, 0xb3, 0x7f, 0xaa, 0xb2, 0x70, + 0x77, 0x5a, 0x16, 0x8e, 0x49, 0xe4, 0x07, 0x51, 0x67, 0x2f, 0xb7, 0x71, 0x91, 0xf2, 0x63, 0x88, + 0xcc, 0x84, 0x04, 0x91, 0x4f, 0x9e, 0x0e, 0xef, 0xe9, 0x67, 0x43, 0x09, 0x39, 0x12, 0x00, 0x73, + 0x4b, 0x9f, 0x81, 0x92, 0x99, 0x4c, 0xfb, 0xe7, 0xd5, 0x2d, 0x6b, 0xbb, 0x58, 0xbf, 0x39, 0x2d, + 0x24, 0x45, 0xd5, 0x2a, 0x33, 0x45, 0x23, 0xc9, 0xe8, 0x73, 0xa0, 0x4f, 0xca, 0x23, 0xbc, 0x5b, + 0xf3, 0x7c, 0xcc, 0xb1, 0xfd, 0x9d, 0x4d, 0xe9, 0x6c, 0x6b, 0x9a, 0xb3, 0x43, 0xde, 0xad, 0x1d, + 0x60, 0x8e, 0xdd, 0x15, 0x65, 0x9a, 0x7e, 0xa3, 0x37, 0xa1, 0x9c, 0x79, 0xf1, 0x06, 0x94, 0x13, + 0x66, 0x7f, 0x77, 0x53, 0xe6, 0xea, 0xd6, 0x45, 0xbe, 0x1e, 0x51, 0x4e, 0xdc, 0x65, 0x62, 0x7c, + 0x31, 0x74, 0x0b, 0x96, 0x7d, 0x12, 0x53, 0x16, 0xe8, 0x0c, 0xd9, 0xdf, 0xdb, 0x94, 0x25, 0x5d, + 0xd2, 0x52, 0x99, 0x15, 0xe4, 0x40, 0xa9, 0x43, 0x22, 0xc2, 0x02, 0xe6, 0xf1, 0xa0, 0x47, 0xec, + 0xaf, 0xdd, 0x96, 0xa0, 0xa2, 0x16, 0x9e, 0x04, 0x3d, 0x82, 0x6a, 0x30, 0xdb, 0xa6, 0xc9, 0xa9, + 0xfd, 0xee, 0x6d, 0xb9, 0xb3, 0xab, 0xd3, 0xa2, 0x79, 0x9d, 0x26, 0xa7, 0xae, 0x84, 0xa2, 0x35, + 0x98, 0x65, 0x21, 0xe5, 0xf6, 0xd7, 0x95, 0x3b, 0xf9, 0xe1, 0xc4, 0x30, 0x2b, 0x20, 0xe8, 0x0e, + 0x54, 0x32, 0xc6, 0x18, 0x90, 0x84, 0x05, 0x34, 0xb2, 0x2d, 0x89, 0x2b, 0xa7, 0xf2, 0x47, 0x4a, + 0x8c, 0x6e, 0x43, 0x39, 0xe5, 0xb6, 0x14, 0xa9, 0xda, 0xd6, 0x8a, 0x16, 0xa7, 0xc0, 0x4b, 0x30, + 0xa7, 0x2e, 0x6e, 0x41, 0xaa, 0xd5, 0x87, 0xf3, 0x9e, 0x05, 0x68, 0xbc, 0x9e, 0xd0, 0x7d, 0x98, + 0x95, 0x47, 0x65, 0xc9, 0xfd, 0xdc, 0x9e, 0xb6, 0x1f, 0xc3, 0x44, 0x1e, 0x98, 0x34, 0x42, 0x35, + 0xb8, 0x84, 0x3b, 0x9d, 0x84, 0x74, 0x46, 0x28, 0x66, 0x46, 0xf2, 0xc0, 0x9a, 0xa1, 0xcb, 0xf8, + 0xe5, 0x0e, 0x54, 0x5a, 0x7d, 0xc6, 0xa9, 0x7f, 0x96, 0xc3, 0x0b, 0x12, 0x5e, 0xd6, 0xf2, 0x0c, + 0xfa, 0x12, 0xac, 0x04, 0x51, 0x2b, 0xec, 0x8b, 0x4d, 0x79, 0x32, 0x85, 0xb3, 0x72, 0x43, 0xcb, + 0x99, 0xb4, 0x21, 0x52, 0xf9, 0x47, 0x0b, 0x8a, 0x1f, 0x90, 0x1d, 0xed, 0x42, 0xe6, 0x81, 0x78, + 0x2c, 0xe8, 0x44, 0x98, 0xf7, 0x13, 0x22, 0xb7, 0x55, 0x72, 0x51, 0xa6, 0x6a, 0xa4, 0x1a, 0xe7, + 0xfb, 0x05, 0x28, 0x8f, 0x04, 0x8a, 0x90, 0xae, 0x27, 0x2b, 0x2f, 0x27, 0x71, 0xe4, 0xaa, 0xbb, + 0xab, 0x8a, 0x50, 0x1f, 0xe8, 0x1e, 0xd8, 0x6a, 0xcf, 0xe3, 0x5c, 0xa7, 0x23, 0x5c, 0x6f, 0x1a, + 0xd7, 0x39, 0x63, 0x05, 0x74, 0x1f, 0x2e, 0xcb, 0xa2, 0xf1, 0x9a, 0xb4, 0x1f, 0xf9, 0x38, 0x39, + 0x1b, 0x32, 0x55, 0xe1, 0x56, 0x25, 0x62, 0x5f, 0x03, 0x86, 0x8d, 0x33, 0xa2, 0x57, 0x17, 0xd8, + 0x34, 0x9e, 0x53, 0xc6, 0x19, 0x42, 0xe6, 0x3e, 0x37, 0x7e, 0x98, 0xb1, 0x48, 0x86, 0xb0, 0xe7, + 0xe5, 0x41, 0x3e, 0x47, 0xab, 0x28, 0x8f, 0xb4, 0x0a, 0x71, 0x65, 0x46, 0xdb, 0xea, 0xc2, 0xc4, + 0xae, 0xfa, 0x1a, 0x5c, 0xc9, 0x81, 0xe3, 0xc9, 0x5a, 0x94, 0x41, 0xdb, 0x19, 0x64, 0x24, 0x5f, + 0x4e, 0x07, 0x56, 0x8d, 0x53, 0x3a, 0xc1, 0x49, 0x87, 0xf0, 0x89, 0xe7, 0x74, 0x0d, 0x20, 0xf7, + 0xae, 0x8b, 0x6a, 0xa9, 0x99, 0x7a, 0x43, 0x9b, 0x50, 0x8c, 0xb1, 0xbc, 0xe1, 0x52, 0xaf, 0xce, + 0x08, 0x94, 0x48, 0x00, 0x9c, 0xaf, 0xc2, 0xd5, 0x91, 0x72, 0xd8, 0x8b, 0xfc, 0x07, 0x59, 0x95, + 0xbd, 0xbf, 0xda, 0xdf, 0x84, 0xa2, 0x51, 0xc8, 0x32, 0xba, 0x45, 0x17, 0xf2, 0x1a, 0x76, 0xbe, + 0x59, 0x80, 0xa5, 0x6c, 0xd2, 0x46, 0x1b, 0x30, 0x1f, 0xf7, 0x9b, 0xa7, 0xe4, 0x4c, 0xae, 0x56, + 0x72, 0xf5, 0x97, 0x98, 0xc1, 0xde, 0x09, 0x78, 0xd7, 0x4f, 0xf0, 0x3b, 0x38, 0xf4, 0x5a, 0x09, + 0xf1, 0x49, 0xc4, 0x03, 0x1c, 0xb2, 0x34, 0x9b, 0x6a, 0xdb, 0x57, 0x72, 0xd0, 0x83, 0x1c, 0xa3, + 0xcb, 0xe0, 0x0e, 0x54, 0x70, 0x8b, 0x07, 0x03, 0x75, 0x0b, 0xd5, 0xc9, 0xcd, 0x29, 0x5a, 0xcc, + 0xe5, 0xea, 0xe8, 0xae, 0x01, 0x90, 0xa7, 0x01, 0xd7, 0xa0, 0x79, 0x09, 0x5a, 0x12, 0x12, 0xa5, + 0xbe, 0x03, 0x15, 0x23, 0x1a, 0xb3, 0x06, 0xca, 0xb9, 0x5c, 0x41, 0x6f, 0xc2, 0x72, 0xda, 0xd6, + 0x15, 0x6e, 0x51, 0x35, 0x09, 0x2d, 0x54, 0xa0, 0x63, 0x28, 0x89, 0xcc, 0xf5, 0x99, 0xd7, 0x0e, + 0x71, 0x87, 0xd9, 0x4b, 0x5b, 0xd6, 0xf6, 0x4a, 0xfd, 0x95, 0x0b, 0x1f, 0x26, 0x3b, 0x0d, 0x69, + 0xf5, 0xba, 0x30, 0x72, 0x8b, 0x2c, 0xff, 0x70, 0x3e, 0x0d, 0x45, 0x43, 0x87, 0x8a, 0xb0, 0x70, + 0xf4, 0xd6, 0xd1, 0xc9, 0xd1, 0xde, 0xc3, 0xca, 0x87, 0x10, 0x82, 0x15, 0xf5, 0x71, 0x72, 0x78, + 0xe0, 0x1d, 0x7e, 0xe1, 0xe8, 0xa4, 0x62, 0xa1, 0x0a, 0x94, 0x1e, 0x1f, 0x9d, 0xbc, 0x71, 0xe0, + 0xee, 0x3d, 0xde, 0xdb, 0x7f, 0x78, 0x58, 0x99, 0x71, 0x42, 0xa8, 0xca, 0xc1, 0xdd, 0x25, 0x98, + 0x09, 0x56, 0xe9, 0x89, 0x72, 0x21, 0x2d, 0x9a, 0xf8, 0xe2, 0x06, 0xe4, 0x8f, 0x16, 0xd5, 0xfb, + 0x54, 0x3d, 0xae, 0x64, 0x62, 0xd5, 0xfc, 0x26, 0x33, 0x48, 0x5a, 0xc3, 0x05, 0xa3, 0x75, 0x7d, + 0x19, 0x96, 0xf2, 0x1b, 0x96, 0xf5, 0x1a, 0xcb, 0xe8, 0x35, 0x17, 0x50, 0xc0, 0xcc, 0xb9, 0x14, + 0xe0, 0xfc, 0x70, 0x26, 0x7d, 0x10, 0xaa, 0xc1, 0x62, 0xd2, 0x3d, 0x7a, 0x19, 0x90, 0x71, 0x51, + 0x86, 0x1d, 0x57, 0xf2, 0xfb, 0xa2, 0xab, 0xe9, 0x2e, 0xac, 0x8a, 0x84, 0x93, 0x09, 0x04, 0x58, + 0x96, 0x0a, 0x03, 0x7b, 0x13, 0x96, 0xf5, 0x7b, 0x2d, 0x21, 0x03, 0x82, 0x43, 0xcd, 0x76, 0x25, + 0x25, 0x74, 0xa5, 0x0c, 0xbd, 0x06, 0x4b, 0xf9, 0x90, 0x33, 0xf7, 0x9c, 0x33, 0xce, 0x62, 0x3a, + 0x93, 0xa0, 0xab, 0xb0, 0x94, 0x93, 0xff, 0xbc, 0x22, 0x81, 0x4c, 0x20, 0xee, 0x70, 0x93, 0xfa, + 0x67, 0xb2, 0x4a, 0xcf, 0xb9, 0xc3, 0x46, 0x8a, 0xf6, 0xa9, 0x7f, 0xe6, 0x4a, 0x23, 0xe7, 0x5b, + 0x05, 0x28, 0x8f, 0x68, 0xc4, 0x88, 0x37, 0x34, 0x75, 0xaa, 0xb7, 0xf4, 0xcd, 0xe7, 0x20, 0x07, + 0x77, 0xc8, 0x10, 0x3d, 0x06, 0x14, 0x27, 0x34, 0xa6, 0x8c, 0x24, 0x6a, 0x00, 0x0e, 0xa2, 0x0e, + 0xb3, 0x67, 0xa4, 0xbb, 0xed, 0xa9, 0x33, 0xac, 0xb6, 0x68, 0x68, 0x03, 0x77, 0x35, 0x1e, 0x91, + 0x48, 0xc7, 0x6a, 0xa1, 0x21, 0xc7, 0x85, 0xf3, 0x1d, 0xef, 0x69, 0x8b, 0xdc, 0x31, 0x1e, 0x91, + 0x30, 0x74, 0x1f, 0x16, 0xf5, 0x88, 0xc7, 0xec, 0x59, 0xe9, 0x6e, 0x73, 0x9a, 0xbb, 0x03, 0x85, + 0x73, 0x33, 0x03, 0xf4, 0x16, 0x94, 0x07, 0x34, 0xec, 0x47, 0x5c, 0x34, 0x40, 0xc1, 0x28, 0xcc, + 0x9e, 0x93, 0x3e, 0x5e, 0x9a, 0x7a, 0xdb, 0x53, 0xf8, 0xe1, 0xd3, 0x80, 0xbb, 0x2b, 0x03, 0xf3, + 0x93, 0x39, 0xdf, 0xb6, 0xa0, 0x74, 0x90, 0x0e, 0x9c, 0x71, 0x9f, 0x4f, 0x65, 0xd0, 0x1d, 0x58, + 0x8b, 0x13, 0x4a, 0xdb, 0x1e, 0x6d, 0x7b, 0x31, 0x65, 0x8c, 0xb0, 0x6c, 0xda, 0x2b, 0xc9, 0xf4, + 0xd1, 0xf6, 0xdb, 0xed, 0xe3, 0x4c, 0x71, 0x31, 0xe3, 0x16, 0x2e, 0x64, 0x5c, 0xe7, 0x09, 0x20, + 0x75, 0x52, 0x38, 0x14, 0xe3, 0x07, 0xf1, 0x5f, 0x70, 0xd6, 0xb8, 0x0b, 0xab, 0xd3, 0x86, 0x8c, + 0x72, 0x73, 0xa4, 0x5d, 0xfe, 0xc9, 0x82, 0x4b, 0xf2, 0x8c, 0x70, 0x33, 0x24, 0xe6, 0xe8, 0xf6, + 0x51, 0x58, 0x1d, 0x62, 0xab, 0x40, 0xbc, 0xac, 0x2c, 0xf9, 0xb0, 0xaa, 0x98, 0x7c, 0x25, 0xe4, + 0x13, 0xe7, 0xae, 0x99, 0xc9, 0x73, 0x57, 0xda, 0x16, 0x0b, 0xff, 0x4b, 0x5b, 0x7c, 0xe1, 0xa1, + 0xed, 0x1b, 0x16, 0x14, 0xf5, 0x39, 0xcb, 0x24, 0x1e, 0x99, 0xaf, 0x8f, 0xb8, 0xcf, 0x75, 0x77, + 0xbe, 0x75, 0x41, 0x25, 0xca, 0x1a, 0x31, 0x9e, 0x28, 0xba, 0x62, 0x70, 0x8f, 0xf6, 0x23, 0xae, + 0x93, 0xaf, 0xbf, 0x04, 0xa3, 0x88, 0x27, 0x0b, 0xe3, 0xb8, 0x17, 0x6b, 0xb2, 0xce, 0x05, 0xce, + 0x2f, 0x66, 0xa0, 0x32, 0x7a, 0x0d, 0xc5, 0x74, 0x9d, 0x5d, 0x66, 0xb3, 0x31, 0x2c, 0xa7, 0x52, + 0xd5, 0x17, 0x5c, 0x28, 0xc7, 0xba, 0x2e, 0x14, 0x93, 0xd7, 0xe4, 0xd2, 0xe7, 0x3d, 0x5a, 0xc7, + 0xca, 0x28, 0xf5, 0x89, 0x43, 0xf1, 0x55, 0x43, 0x1f, 0x83, 0x4b, 0x99, 0xcf, 0x2c, 0xa1, 0x5e, + 0x4d, 0x97, 0x0b, 0x8a, 0x0d, 0x07, 0x52, 0x55, 0x1b, 0x8f, 0x42, 0x4d, 0xa1, 0xef, 0x23, 0x8a, + 0xfa, 0x94, 0x28, 0xd2, 0x09, 0x75, 0x3c, 0x8a, 0xba, 0xf3, 0x67, 0x0b, 0x2a, 0xa3, 0xac, 0x83, + 0x7c, 0xa8, 0xb2, 0xb4, 0x96, 0xcd, 0xd7, 0xbd, 0x57, 0xd3, 0xe7, 0xfc, 0xf2, 0xb4, 0x10, 0x27, + 0x5d, 0x01, 0x77, 0x9d, 0x4d, 0x90, 0xd6, 0xa6, 0xaf, 0x52, 0xd7, 0xc7, 0xf1, 0x7f, 0x58, 0xa5, + 0xee, 0xbc, 0x6b, 0xc1, 0x82, 0xae, 0x3e, 0x91, 0x9e, 0x1e, 0x49, 0x4e, 0x43, 0xe2, 0x29, 0x2e, + 0x4a, 0xff, 0x4f, 0xb0, 0xe4, 0xdf, 0x09, 0x48, 0xe9, 0x8e, 0x85, 0x2a, 0xfd, 0x2b, 0xe1, 0x2e, + 0xac, 0x6a, 0x0b, 0x9e, 0x10, 0xa2, 0x8b, 0x4a, 0xd5, 0x69, 0x59, 0x29, 0x4e, 0x12, 0x42, 0x54, + 0x59, 0xdd, 0x80, 0xb4, 0xb0, 0xbd, 0xec, 0x66, 0x96, 0xdc, 0xa2, 0x9f, 0x5f, 0x1b, 0x27, 0x84, + 0xe5, 0x21, 0x3e, 0x9d, 0x32, 0x6b, 0x4c, 0x98, 0x70, 0x66, 0x26, 0x4e, 0x38, 0x43, 0x5d, 0xb7, + 0x30, 0xd2, 0x75, 0x9d, 0x2f, 0xc1, 0x62, 0xf6, 0xef, 0xc3, 0x0e, 0xac, 0xa5, 0xc1, 0x99, 0x6c, + 0xa6, 0x48, 0x7a, 0x55, 0xab, 0x8c, 0x99, 0xe1, 0x06, 0x94, 0x14, 0xf7, 0x0d, 0xcd, 0x21, 0x45, + 0x29, 0xd3, 0x94, 0x17, 0x42, 0xc9, 0xfc, 0x83, 0x62, 0x78, 0x82, 0xb0, 0x5e, 0x78, 0x82, 0xb8, + 0x06, 0x30, 0xa0, 0x9c, 0x78, 0x2d, 0x83, 0x0b, 0x96, 0x84, 0xe4, 0x81, 0x10, 0xec, 0x97, 0x7e, + 0xf9, 0xec, 0xba, 0xf5, 0xab, 0x67, 0xd7, 0xad, 0xbf, 0x3c, 0xbb, 0x6e, 0x35, 0xe7, 0xe5, 0x1f, + 0xed, 0xaf, 0xfe, 0x37, 0x00, 0x00, 0xff, 0xff, 0xae, 0x96, 0x55, 0x1f, 0xc0, 0x17, 0x00, 0x00, } func (m *BeaconState) Marshal() (dAtA []byte, err error) { @@ -2405,6 +2471,44 @@ func (m *AttestationData) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *AttestationTarget) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AttestationTarget) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Slot != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintTypes(dAtA, i, uint64(m.Slot)) + } + if len(m.BlockRoot) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintTypes(dAtA, i, uint64(len(m.BlockRoot))) + i += copy(dAtA[i:], m.BlockRoot) + } + if len(m.ParentRoot) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintTypes(dAtA, i, uint64(len(m.ParentRoot))) + i += copy(dAtA[i:], m.ParentRoot) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func (m *AttestationDataAndCustodyBit) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -3413,6 +3517,29 @@ func (m *AttestationData) Size() (n int) { return n } +func (m *AttestationTarget) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Slot != 0 { + n += 1 + sovTypes(uint64(m.Slot)) + } + l = len(m.BlockRoot) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + l = len(m.ParentRoot) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *AttestationDataAndCustodyBit) Size() (n int) { if m == nil { return 0 @@ -5576,6 +5703,147 @@ func (m *AttestationData) Unmarshal(dAtA []byte) error { } return nil } +func (m *AttestationTarget) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: AttestationTarget: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: AttestationTarget: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slot", wireType) + } + m.Slot = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slot |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BlockRoot", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BlockRoot = append(m.BlockRoot[:0], dAtA[iNdEx:postIndex]...) + if m.BlockRoot == nil { + m.BlockRoot = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ParentRoot", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ParentRoot = append(m.ParentRoot[:0], dAtA[iNdEx:postIndex]...) + if m.ParentRoot == nil { + m.ParentRoot = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *AttestationDataAndCustodyBit) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/proto/beacon/p2p/v1/types.proto b/proto/beacon/p2p/v1/types.proto index 22c19c09958c..6c9b29cb0a56 100644 --- a/proto/beacon/p2p/v1/types.proto +++ b/proto/beacon/p2p/v1/types.proto @@ -80,6 +80,12 @@ message AttestationData { bytes justified_block_root_hash32 = 8; } +message AttestationTarget { + uint64 slot = 1; + bytes block_root = 2; + bytes parent_root = 3; +} + message AttestationDataAndCustodyBit { AttestationData data = 1; bool custody_bit = 2; From 41987b20353ce26ffba21ca7ffba504ce321b1ba Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 22 Apr 2019 19:06:24 -0700 Subject: [PATCH 03/54] Renovate Mega Updates (#2332) * Update com_github_atlassian_bazel_tools commit hash to 31382b2 * Update io_bazel_rules_k8s commit hash to e7b408f * Update libp2p * fixed nat * nat --- WORKSPACE | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 18530ee25233..bf00f7b77236 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -21,9 +21,9 @@ http_archive( http_archive( name = "com_github_atlassian_bazel_tools", - sha256 = "f4d370dec7316e668c015be86b05593a0a71fa3047108c445b6643cb954193f7", - strip_prefix = "bazel-tools-e31d79dc72434fe064b7df4efb51875b2c201e84", - urls = ["https://github.com/atlassian/bazel-tools/archive/e31d79dc72434fe064b7df4efb51875b2c201e84.tar.gz"], + sha256 = "af4908ea16b43e0c88007742a5c54997ee068fc393f9850ade2d59b1d5f49f55", + strip_prefix = "bazel-tools-31382b2267b0bc3d9771085e3503473a061593e1", + urls = ["https://github.com/atlassian/bazel-tools/archive/31382b2267b0bc3d9771085e3503473a061593e1.tar.gz"], ) http_archive( @@ -41,9 +41,9 @@ http_archive( http_archive( name = "io_bazel_rules_k8s", - sha256 = "f0a66b29bb6705da233899b61723d3fcc1b68d1e6414189a6f9c82795d407f27", - strip_prefix = "rules_k8s-50fb3c2f868fcd0b5c3f223ac19ba72db1f6eb7d", - url = "https://github.com/bazelbuild/rules_k8s/archive/50fb3c2f868fcd0b5c3f223ac19ba72db1f6eb7d.tar.gz", + sha256 = "b4aabc3f931a67b4970076ae9490afc18caa1690360c952fba9681225739f6d5", + strip_prefix = "rules_k8s-e7b408f07785e3b660ee7e99173b0e4328c7b65e", + url = "https://github.com/bazelbuild/rules_k8s/archive/e7b408f07785e3b660ee7e99173b0e4328c7b65e.tar.gz", ) load( @@ -182,7 +182,7 @@ go_repository( go_repository( name = "com_github_libp2p_go_libp2p", - commit = "f106bf7f74f573d33505b7173ef2f7f0596b5d67", # v0.0.13 + commit = "5bce62bbecc0c07087af4cae2490b764afe1c594", # v0.0.15 importpath = "github.com/libp2p/go-libp2p", ) @@ -213,7 +213,7 @@ go_repository( go_repository( name = "com_github_multiformats_go_multihash", - commit = "6b439b7c6e3c44c112171540500be697ba235235", # v0.0.1 + commit = "922cbd7915f407488e3f4413ada76400a1b29b49", # v0.0.3 importpath = "github.com/multiformats/go-multihash", ) @@ -750,13 +750,13 @@ go_repository( go_repository( name = "com_github_libp2p_go_libp2p_kad_dht", build_file_proto_mode = "disable_global", - commit = "b99a6ee931a8331ccfb8292bee6d3e5c03edf5e1", # v0.0.8 + commit = "b220ac23e82c0df788178e806b1fefa92e936766", # v0.0.9 importpath = "github.com/libp2p/go-libp2p-kad-dht", ) go_repository( name = "com_github_ipfs_go_datastore", - commit = "80940bb93587a19603a74ea07f9e5c921bae3b07", # v0.0.4 + commit = "f8bd98feaffb64c335ac5e74f0a3d3aedbfd13ba", # v0.0.5 importpath = "github.com/ipfs/go-datastore", ) @@ -829,7 +829,7 @@ go_repository( go_repository( name = "com_github_libp2p_go_libp2p_discovery", - commit = "d4ef632c52ff995d94eaed926529eede8faa3965", # v0.0.1 + commit = "4cb4193d603389a75bccd07336de74d54bea6b00", # v0.0.2 importpath = "github.com/libp2p/go-libp2p-discovery", ) From 7a27d9a227533bfe418834ab09e7fa50ede2a808 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Mon, 22 Apr 2019 21:30:37 -0500 Subject: [PATCH 04/54] Move Inclusion Maps to Epoch Processing (#2347) * fix broken initial sync * tests pass --- beacon-chain/core/balances/BUILD.bazel | 1 - .../core/balances/rewards_penalties.go | 31 ++++---- .../core/balances/rewards_penalties_test.go | 34 +++++++-- beacon-chain/core/blocks/BUILD.bazel | 1 - beacon-chain/core/blocks/block_operations.go | 74 ------------------- beacon-chain/core/state/transition.go | 17 ++++- 6 files changed, 57 insertions(+), 101 deletions(-) diff --git a/beacon-chain/core/balances/BUILD.bazel b/beacon-chain/core/balances/BUILD.bazel index c73235f08aa2..4f9f9ea39a65 100644 --- a/beacon-chain/core/balances/BUILD.bazel +++ b/beacon-chain/core/balances/BUILD.bazel @@ -6,7 +6,6 @@ go_library( importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/balances", visibility = ["//beacon-chain:__subpackages__"], deps = [ - "//beacon-chain/core/blocks:go_default_library", "//beacon-chain/core/epoch:go_default_library", "//beacon-chain/core/helpers:go_default_library", "//proto/beacon/p2p/v1:go_default_library", diff --git a/beacon-chain/core/balances/rewards_penalties.go b/beacon-chain/core/balances/rewards_penalties.go index 91d5a856f283..40703e29f3f1 100644 --- a/beacon-chain/core/balances/rewards_penalties.go +++ b/beacon-chain/core/balances/rewards_penalties.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" - "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks" "github.com/prysmaticlabs/prysm/beacon-chain/core/epoch" "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" @@ -134,14 +133,15 @@ func ExpectedBeaconChainHead( func InclusionDistance( state *pb.BeaconState, attesterIndices []uint64, - totalBalance uint64) (*pb.BeaconState, error) { + totalBalance uint64, + inclusionDistanceByAttester map[uint64]uint64) (*pb.BeaconState, error) { baseRewardQuotient := helpers.BaseRewardQuotient(totalBalance) for _, index := range attesterIndices { - inclusionDistance, err := blocks.AttsInclusionDistance(index) - if err != nil { - return nil, err + inclusionDistance, ok := inclusionDistanceByAttester[index] + if !ok { + return nil, fmt.Errorf("could not get inclusion distance for attester: %d", index) } if inclusionDistance == 0 { return nil, errors.New("could not process inclusion distance: 0") @@ -261,14 +261,14 @@ func InactivityExitedPenalties( func InactivityInclusionDistance( state *pb.BeaconState, attesterIndices []uint64, - totalBalance uint64) (*pb.BeaconState, error) { - + totalBalance uint64, + inclusionDistanceByAttester map[uint64]uint64) (*pb.BeaconState, error) { baseRewardQuotient := helpers.BaseRewardQuotient(totalBalance) for _, index := range attesterIndices { - inclusionDistance, err := blocks.AttsInclusionDistance(index) - if err != nil { - return nil, err + inclusionDistance, ok := inclusionDistanceByAttester[index] + if !ok { + return nil, fmt.Errorf("could not get inclusion distance for attester: %d", index) } baseReward := helpers.BaseReward(state, index, baseRewardQuotient) state.ValidatorBalances[index] -= baseReward - @@ -290,18 +290,19 @@ func InactivityInclusionDistance( func AttestationInclusion( state *pb.BeaconState, totalBalance uint64, - prevEpochAttesterIndices []uint64) (*pb.BeaconState, error) { + prevEpochAttesterIndices []uint64, + inclusionSlotByAttester map[uint64]uint64) (*pb.BeaconState, error) { baseRewardQuotient := helpers.BaseRewardQuotient(totalBalance) for _, index := range prevEpochAttesterIndices { // Get the attestation's inclusion slot using the attestor's index. - slot, err := blocks.AttsInclusionSlot(index) - if err != nil { - return nil, fmt.Errorf("could not get inclusion slot: %v", err) + slot, ok := inclusionSlotByAttester[index] + if !ok { + return nil, fmt.Errorf("could not get inclusion slot for attester: %d", index) } proposerIndex, err := helpers.BeaconProposerIndex(state, slot) if err != nil { - return nil, fmt.Errorf("could not get propoer index: %v", err) + return nil, fmt.Errorf("could not get proposer index: %v", err) } state.ValidatorBalances[proposerIndex] += helpers.BaseReward(state, proposerIndex, baseRewardQuotient) / diff --git a/beacon-chain/core/balances/rewards_penalties_test.go b/beacon-chain/core/balances/rewards_penalties_test.go index 74a1f735e6f5..0d3d93b6131b 100644 --- a/beacon-chain/core/balances/rewards_penalties_test.go +++ b/beacon-chain/core/balances/rewards_penalties_test.go @@ -188,10 +188,15 @@ func TestInclusionDistRewards_AccurateRewards(t *testing.T) { if _, err := blocks.ProcessBlockAttestations(state, block, false /* verify sig */); err != nil { t.Fatal(err) } + inclusionMap := make(map[uint64]uint64) + for _, voted := range tt.voted { + inclusionMap[voted] = state.Slot + } state, err := InclusionDistance( state, tt.voted, - uint64(len(validatorBalances))*params.BeaconConfig().MaxDepositAmount) + uint64(len(validatorBalances))*params.BeaconConfig().MaxDepositAmount, + inclusionMap) if err != nil { t.Fatalf("could not execute InclusionDistRewards:%v", err) } @@ -231,7 +236,8 @@ func TestInclusionDistRewards_OutOfBounds(t *testing.T) { ValidatorRegistry: validators, LatestAttestations: attestation, } - _, err := InclusionDistance(state, tt.voted, 0) + inclusionMap := make(map[uint64]uint64) + _, err := InclusionDistance(state, tt.voted, 0, inclusionMap) if err == nil { t.Fatal("InclusionDistRewards should have failed") } @@ -422,10 +428,15 @@ func TestInactivityInclusionPenalty_AccuratePenalties(t *testing.T) { ValidatorBalances: validatorBalances, LatestAttestations: attestation, } + inclusionMap := make(map[uint64]uint64) + for _, voted := range tt.voted { + inclusionMap[voted] = state.Slot + 1 + } state, err := InactivityInclusionDistance( state, tt.voted, - uint64(len(validatorBalances))*params.BeaconConfig().MaxDepositAmount) + uint64(len(validatorBalances))*params.BeaconConfig().MaxDepositAmount, + inclusionMap) for _, i := range tt.voted { validatorBalances[i] = 32000055555 @@ -464,7 +475,8 @@ func TestInactivityInclusionPenalty_OutOfBounds(t *testing.T) { ValidatorRegistry: validators, LatestAttestations: attestation, } - _, err := InactivityInclusionDistance(state, tt.voted, 0) + inclusionMap := make(map[uint64]uint64) + _, err := InactivityInclusionDistance(state, tt.voted, 0, inclusionMap) if err == nil { t.Fatal("InclusionDistRewards should have failed") } @@ -522,10 +534,16 @@ func TestAttestationInclusionRewards_AccurateRewards(t *testing.T) { if err != nil { t.Fatal(err) } + inclusionMap := make(map[uint64]uint64) + for _, voted := range tt.voted { + inclusionMap[voted] = state.Slot + } + state, err = AttestationInclusion( state, uint64(len(validatorBalances))*params.BeaconConfig().MaxDepositAmount, - tt.voted) + tt.voted, + inclusionMap) for _, i := range tt.voted { validatorBalances[i] = 32000008680 @@ -564,7 +582,8 @@ func TestAttestationInclusionRewards_NoInclusionSlot(t *testing.T) { ValidatorRegistry: validators, ValidatorBalances: validatorBalances, } - if _, err := AttestationInclusion(state, 0, tt.voted); err == nil { + inclusionMap := make(map[uint64]uint64) + if _, err := AttestationInclusion(state, 0, tt.voted, inclusionMap); err == nil { t.Fatal("AttestationInclusionRewards should have failed with no inclusion slot") } } @@ -600,7 +619,8 @@ func TestAttestationInclusionRewards_NoProposerIndex(t *testing.T) { ValidatorBalances: validatorBalances, LatestAttestations: attestation, } - if _, err := AttestationInclusion(state, 0, tt.voted); err == nil { + inclusionMap := make(map[uint64]uint64) + if _, err := AttestationInclusion(state, 0, tt.voted, inclusionMap); err == nil { t.Fatal("AttestationInclusionRewards should have failed with no proposer index") } } diff --git a/beacon-chain/core/blocks/BUILD.bazel b/beacon-chain/core/blocks/BUILD.bazel index 4a26cdf229bd..20cde46643e5 100644 --- a/beacon-chain/core/blocks/BUILD.bazel +++ b/beacon-chain/core/blocks/BUILD.bazel @@ -10,7 +10,6 @@ go_library( importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks", visibility = ["//beacon-chain:__subpackages__"], deps = [ - "//beacon-chain/cache:go_default_library", "//beacon-chain/core/helpers:go_default_library", "//beacon-chain/core/state/stateutils:go_default_library", "//beacon-chain/core/validators:go_default_library", diff --git a/beacon-chain/core/blocks/block_operations.go b/beacon-chain/core/blocks/block_operations.go index b5ac5aaee1d3..416cb91a0c0d 100644 --- a/beacon-chain/core/blocks/block_operations.go +++ b/beacon-chain/core/blocks/block_operations.go @@ -9,10 +9,8 @@ import ( "errors" "fmt" "reflect" - "sync" "github.com/gogo/protobuf/proto" - "github.com/prysmaticlabs/prysm/beacon-chain/cache" "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" "github.com/prysmaticlabs/prysm/beacon-chain/core/state/stateutils" v "github.com/prysmaticlabs/prysm/beacon-chain/core/validators" @@ -25,24 +23,6 @@ import ( "github.com/sirupsen/logrus" ) -var committeeCache = cache.NewCommitteesCache() - -type attesterInclusionStore struct { - slotLock sync.RWMutex - distLock sync.RWMutex - // attesterInclusion is a mapping that tracks when an attester's attestation - // last get included in beacon chain. - attesterInclusionSlot map[uint64]uint64 - // attesterInclusion is a mapping that tracks the difference in slot number - // of when attestation gets submitted and when it gets included. - attesterInclusionDist map[uint64]uint64 -} - -var attsInclStore = attesterInclusionStore{ - attesterInclusionSlot: make(map[uint64]uint64), - attesterInclusionDist: make(map[uint64]uint64), -} - // VerifyProposerSignature uses BLS signature verification to ensure // the correct proposer created an incoming beacon block during state // transition processing. @@ -433,38 +413,6 @@ func ProcessBlockAttestations( return nil, fmt.Errorf("could not verify attestation at index %d in block: %v", idx, err) } - var IndicesInCommittee []uint64 - // get the validator indices from the attestation using committees info cache. - cachedCommittees, err := committeeCache.CommitteesInfoBySlot(attestation.Data.Slot) - if err != nil { - return nil, err - } - if cachedCommittees == nil { - crosslinkCommittees, err := helpers.CrosslinkCommitteesAtSlot(beaconState, attestation.Data.Slot, false /* registryChange */) - if err != nil { - return nil, err - } - cachedCommittees = helpers.ToCommitteeCache(attestation.Data.Slot, crosslinkCommittees) - if err := committeeCache.AddCommittees(cachedCommittees); err != nil { - return nil, err - } - } - for _, v := range cachedCommittees.Committees { - if v.Shard == attestation.Data.Shard { - IndicesInCommittee = v.Committee - break - } - } - - attsInclStore.slotLock.Lock() - attsInclStore.distLock.Lock() - defer attsInclStore.slotLock.Unlock() - defer attsInclStore.distLock.Unlock() - for _, index := range IndicesInCommittee { - attsInclStore.attesterInclusionSlot[index] = beaconState.Slot - attsInclStore.attesterInclusionDist[index] = beaconState.Slot - attestation.Data.Slot - } - beaconState.LatestAttestations = append(beaconState.LatestAttestations, &pb.PendingAttestation{ Data: attestation.Data, AggregationBitfield: attestation.AggregationBitfield, @@ -754,25 +702,3 @@ func verifyExit(beaconState *pb.BeaconState, exit *pb.VoluntaryExit, verifySigna } return nil } - -// AttsInclusionSlot returns the slot of when an attestator's attestation last gets included -// in the beacon chain by a proposer. -func AttsInclusionSlot(index uint64) (uint64, error) { - attsInclStore.slotLock.RLock() - attsInclStore.slotLock.RUnlock() - if slot, ok := attsInclStore.attesterInclusionSlot[index]; ok { - return slot, nil - } - return 0, fmt.Errorf("no inclusion slot for attestor %d", index) -} - -// AttsInclusionDistance returns diff in slot of when an attestator's attestation gets submitted -// and included. -func AttsInclusionDistance(index uint64) (uint64, error) { - attsInclStore.distLock.RLock() - attsInclStore.distLock.RUnlock() - if slot, ok := attsInclStore.attesterInclusionDist[index]; ok { - return slot, nil - } - return 0, fmt.Errorf("no inclusion distance for attestor %d", index) -} diff --git a/beacon-chain/core/state/transition.go b/beacon-chain/core/state/transition.go index 0e79c05bd507..523d170d38d1 100644 --- a/beacon-chain/core/state/transition.go +++ b/beacon-chain/core/state/transition.go @@ -223,6 +223,9 @@ func ProcessEpoch(ctx context.Context, state *pb.BeaconState, block *pb.BeaconBl prevEpochHeadAttestations := []*pb.PendingAttestation{} prevEpochHeadAttesterIndices := []uint64{} + inclusionSlotByAttester := make(map[uint64]uint64) + inclusionDistanceByAttester := make(map[uint64]uint64) + for _, attestation := range state.LatestAttestations { // We determine the attestation participants. @@ -234,6 +237,11 @@ func ProcessEpoch(ctx context.Context, state *pb.BeaconState, block *pb.BeaconBl return nil, err } + for _, participant := range attesterIndices { + inclusionDistanceByAttester[participant] = state.Slot - attestation.Data.Slot + inclusionSlotByAttester[participant] = attestation.InclusionSlot + } + // We extract the attestations from the current epoch. if currentEpoch == helpers.SlotToEpoch(attestation.Data.Slot) { currentEpochAttestations = append(currentEpochAttestations, attestation) @@ -358,7 +366,8 @@ func ProcessEpoch(ctx context.Context, state *pb.BeaconState, block *pb.BeaconBl state, err = bal.InclusionDistance( state, prevEpochAttesterIndices, - totalBalance) + totalBalance, + inclusionDistanceByAttester) if err != nil { return nil, fmt.Errorf("could not calculate inclusion dist rewards: %v", err) } @@ -397,7 +406,8 @@ func ProcessEpoch(ctx context.Context, state *pb.BeaconState, block *pb.BeaconBl state, err = bal.InactivityInclusionDistance( state, prevEpochAttesterIndices, - totalBalance) + totalBalance, + inclusionDistanceByAttester) if err != nil { return nil, fmt.Errorf("could not calculate inclusion penalties: %v", err) } @@ -407,7 +417,8 @@ func ProcessEpoch(ctx context.Context, state *pb.BeaconState, block *pb.BeaconBl state, err = bal.AttestationInclusion( state, totalBalance, - prevEpochAttesterIndices) + prevEpochAttesterIndices, + inclusionSlotByAttester) if err != nil { return nil, fmt.Errorf("could not process attestation inclusion rewards: %v", err) } From a8ce61b3879661e0cb4873afed6ea3c406962e00 Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Tue, 23 Apr 2019 13:04:21 +0800 Subject: [PATCH 05/54] Remove Proto Clone from State Reads (#2349) * add cached serialized state to reduce use of proto clone * review comments * review comments * comments on cache --- beacon-chain/db/db.go | 11 +++++---- beacon-chain/db/state.go | 50 ++++++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/beacon-chain/db/db.go b/beacon-chain/db/db.go index 519bdad95856..2e13d7445251 100644 --- a/beacon-chain/db/db.go +++ b/beacon-chain/db/db.go @@ -20,11 +20,12 @@ var log = logrus.WithField("prefix", "beacondb") // For example, instead of defining get, put, remove // This defines methods such as getBlock, saveBlocksAndAttestations, etc. type BeaconDB struct { - stateLock sync.RWMutex - currentState *pb.BeaconState - stateHash [32]byte - db *bolt.DB - DatabasePath string + stateLock sync.RWMutex + currentState *pb.BeaconState + serializedState []byte + stateHash [32]byte + db *bolt.DB + DatabasePath string // Beacon block info in memory. highestBlockSlot uint64 diff --git a/beacon-chain/db/state.go b/beacon-chain/db/state.go index 48fd2d3f2596..67bbee13a85b 100644 --- a/beacon-chain/db/state.go +++ b/beacon-chain/db/state.go @@ -1,6 +1,7 @@ package db import ( + "bytes" "context" "encoding/binary" "errors" @@ -47,6 +48,7 @@ func (db *BeaconDB) InitializeState(ctx context.Context, genesisTime uint64, dep zeroBinary := encodeSlotNumber(0) db.currentState = beaconState + db.serializedState = stateEnc db.stateHash = stateHash if err := db.SaveState(ctx, beaconState); err != nil { @@ -101,10 +103,20 @@ func (db *BeaconDB) HeadState(ctx context.Context) (*pb.BeaconState, error) { // Return in-memory cached state, if available. if db.currentState != nil { - _, span := trace.StartSpan(ctx, "proto.Clone") + _, span := trace.StartSpan(ctx, "proto.Marshal") defer span.End() - cachedState := proto.Clone(db.currentState).(*pb.BeaconState) - return cachedState, nil + // For each READ, we compare the cached state with a serialized copy of the state + // to determine if the cached state has been corrupted. If the marshaled version + // of the cached state is not equal to the serialized copy, we then retrieve the + // state from disk and log an error. + enc, err := proto.Marshal(db.currentState) + if err != nil { + return nil, err + } + if bytes.Equal(enc, db.serializedState) { + return db.currentState, nil + } + log.Error("Cached state has been mutated, so retrieving state from disk") } var beaconState *pb.BeaconState @@ -117,9 +129,13 @@ func (db *BeaconDB) HeadState(ctx context.Context) (*pb.BeaconState, error) { var err error beaconState, err = createState(enc) + if beaconState != nil && beaconState.Slot > db.highestBlockSlot { db.highestBlockSlot = beaconState.Slot } + db.serializedState = enc + db.currentState = beaconState + return err }) @@ -141,21 +157,18 @@ func (db *BeaconDB) SaveState(ctx context.Context, beaconState *pb.BeaconState) defer db.stateLock.Unlock() lockSpan.End() - // Clone to prevent mutations of the cached copy - ctx, cloneSpan := trace.StartSpan(ctx, "proto.Clone") - currentState, ok := proto.Clone(beaconState).(*pb.BeaconState) - if !ok { - cloneSpan.End() - return errors.New("could not clone beacon state") - } - db.currentState = currentState - cloneSpan.End() + db.currentState = beaconState - stateHash, err := hashutil.HashProto(beaconState) + // For each WRITE of the state, we serialize the inputted state and save it in memory, + // then the inputted state is assigned our cached state. Further on we write the state to disk then. + // This removes the need to clone the state, as we handle mutated caches in each READ. + enc, err := proto.Marshal(beaconState) if err != nil { return err } + stateHash := hashutil.Hash(enc) + db.serializedState = enc db.stateHash = stateHash if err := db.SaveHistoricalState(ctx, beaconState); err != nil { @@ -165,16 +178,9 @@ func (db *BeaconDB) SaveState(ctx context.Context, beaconState *pb.BeaconState) return db.update(func(tx *bolt.Tx) error { chainInfo := tx.Bucket(chainInfoBucket) - _, marshalSpan := trace.StartSpan(ctx, "proto.Marshal") - beaconStateEnc, err := proto.Marshal(beaconState) - if err != nil { - return err - } - marshalSpan.End() - - stateBytes.Set(float64(len(beaconStateEnc))) + stateBytes.Set(float64(len(enc))) reportStateMetrics(beaconState) - return chainInfo.Put(stateLookupKey, beaconStateEnc) + return chainInfo.Put(stateLookupKey, enc) }) } From 4ed86566c119ff7e7bdc07e29530563a2847c49a Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Tue, 23 Apr 2019 23:57:10 +0800 Subject: [PATCH 06/54] Disable RPC while Syncing (#2353) * handle nil block * blocking rpc till synced * remove unused field * remove log * fix tests --- beacon-chain/node/node.go | 6 ++++++ beacon-chain/rpc/service.go | 10 ++++++++++ beacon-chain/rpc/service_test.go | 20 +++++++++++++++----- beacon-chain/sync/service.go | 24 ++++++++++++++++++------ shared/params/config.go | 17 ++++++++++------- 5 files changed, 59 insertions(+), 18 deletions(-) diff --git a/beacon-chain/node/node.go b/beacon-chain/node/node.go index 4f82e864f1aa..0ca0848aad89 100644 --- a/beacon-chain/node/node.go +++ b/beacon-chain/node/node.go @@ -344,6 +344,11 @@ func (b *BeaconNode) registerRPCService(ctx *cli.Context) error { return err } + var syncService *rbcsync.Service + if err := b.services.FetchService(&syncService); err != nil { + return err + } + port := ctx.GlobalString(utils.RPCPort.Name) cert := ctx.GlobalString(utils.CertFlag.Name) key := ctx.GlobalString(utils.KeyFlag.Name) @@ -355,6 +360,7 @@ func (b *BeaconNode) registerRPCService(ctx *cli.Context) error { ChainService: chainService, OperationService: operationService, POWChainService: web3Service, + SyncService: syncService, }) return b.services.RegisterService(rpcService) diff --git a/beacon-chain/rpc/service.go b/beacon-chain/rpc/service.go index a67176a0336d..135260b1c75f 100644 --- a/beacon-chain/rpc/service.go +++ b/beacon-chain/rpc/service.go @@ -55,6 +55,10 @@ type powChainService interface { ChainStartDeposits() [][]byte } +type syncService interface { + Status() error +} + // Service defining an RPC server for a beacon node. type Service struct { ctx context.Context @@ -63,6 +67,7 @@ type Service struct { chainService chainService powChainService powChainService operationService operationService + syncService syncService port string listener net.Listener withCert string @@ -82,6 +87,7 @@ type Config struct { ChainService chainService POWChainService powChainService OperationService operationService + SyncService syncService } // NewRPCService creates a new instance of a struct implementing the BeaconServiceServer @@ -95,6 +101,7 @@ func NewRPCService(ctx context.Context, cfg *Config) *Service { chainService: cfg.ChainService, powChainService: cfg.POWChainService, operationService: cfg.OperationService, + syncService: cfg.SyncService, port: cfg.Port, withCert: cfg.CertFlag, withKey: cfg.KeyFlag, @@ -174,6 +181,9 @@ func (s *Service) Start() { reflection.Register(s.grpcServer) go func() { + for s.syncService.Status() != nil { + time.Sleep(time.Second * params.BeaconConfig().RPCSyncCheck) + } if s.listener != nil { if err := s.grpcServer.Serve(s.listener); err != nil { log.Errorf("Could not serve gRPC: %v", err) diff --git a/beacon-chain/rpc/service_test.go b/beacon-chain/rpc/service_test.go index 7cd09f2ff0d1..722b56f1ad6c 100644 --- a/beacon-chain/rpc/service_test.go +++ b/beacon-chain/rpc/service_test.go @@ -121,12 +121,20 @@ func newMockChainService() *mockChainService { } } +type mockSyncService struct { +} + +func (ms *mockSyncService) Status() error { + return nil +} + func TestLifecycle_OK(t *testing.T) { hook := logTest.NewGlobal() rpcService := NewRPCService(context.Background(), &Config{ - Port: "7348", - CertFlag: "alice.crt", - KeyFlag: "alice.key", + Port: "7348", + CertFlag: "alice.crt", + KeyFlag: "alice.key", + SyncService: &mockSyncService{}, }) rpcService.Start() @@ -150,7 +158,8 @@ func TestRPC_BadEndpoint(t *testing.T) { hook := logTest.NewLocal(fl.Logger) rpcService := NewRPCService(context.Background(), &Config{ - Port: "ralph merkle!!!", + Port: "ralph merkle!!!", + SyncService: &mockSyncService{}, }) if val, ok := log.(*TestLogger).testMap["error"]; ok { @@ -179,7 +188,8 @@ func TestStatus_CredentialError(t *testing.T) { func TestRPC_InsecureEndpoint(t *testing.T) { hook := logTest.NewGlobal() rpcService := NewRPCService(context.Background(), &Config{ - Port: "7777", + Port: "7777", + SyncService: &mockSyncService{}, }) rpcService.Start() diff --git a/beacon-chain/sync/service.go b/beacon-chain/sync/service.go index cf452b9fb484..14c8d5529a18 100644 --- a/beacon-chain/sync/service.go +++ b/beacon-chain/sync/service.go @@ -2,6 +2,7 @@ package sync import ( "context" + "errors" "fmt" "github.com/prysmaticlabs/prysm/beacon-chain/db" @@ -16,9 +17,10 @@ var slog = logrus.WithField("prefix", "sync") // Service defines the main routines used in the sync service. type Service struct { - RegularSync *RegularSync - InitialSync *initialsync.InitialSync - Querier *Querier + RegularSync *RegularSync + InitialSync *initialsync.InitialSync + Querier *Querier + querierFinished bool } // Config defines the configured services required for sync to work. @@ -60,9 +62,10 @@ func NewSyncService(ctx context.Context, cfg *Config) *Service { is := initialsync.NewInitialSyncService(ctx, isCfg) return &Service{ - RegularSync: rs, - InitialSync: is, - Querier: sq, + RegularSync: rs, + InitialSync: is, + Querier: sq, + querierFinished: false, } } @@ -91,9 +94,14 @@ func (ss *Service) Stop() error { // Status checks the status of the node. It returns nil if it's synced // with the rest of the network and no errors occurred. Otherwise, it returns an error. func (ss *Service) Status() error { + if !ss.querierFinished { + return errors.New("querier is still running") + } + if !ss.Querier.chainStarted { return nil } + if ss.Querier.atGenesis { return nil } @@ -102,6 +110,9 @@ func (ss *Service) Status() error { if err != nil { return fmt.Errorf("could not retrieve chain head %v", err) } + if blk == nil { + return errors.New("no chain head exists in db") + } if blk.Slot < ss.InitialSync.HighestObservedSlot() { return fmt.Errorf("node is not synced as the current chain head is at slot %d", blk.Slot-params.BeaconConfig().GenesisSlot) } @@ -114,6 +125,7 @@ func (ss *Service) run() { if err != nil { slog.Fatalf("Unable to retrieve result from sync querier %v", err) } + ss.querierFinished = true // Sets the highest observed slot from querier. ss.InitialSync.InitializeObservedSlot(ss.Querier.currentHeadSlot) diff --git a/shared/params/config.go b/shared/params/config.go index 1d735f0d0ae4..ad799394821a 100644 --- a/shared/params/config.go +++ b/shared/params/config.go @@ -4,6 +4,7 @@ package params import ( "math/big" + "time" ) // BeaconChainConfig contains constant configs for node to participate in beacon chain. @@ -77,13 +78,14 @@ type BeaconChainConfig struct { MaxAttesterSlashings uint64 // MaxAttesterSlashings defines the maximum number of casper FFG slashings possible in a block. // Prysm constants. - DepositsForChainStart uint64 // DepositsForChainStart defines how many validator deposits needed to kick off beacon chain. - RandBytes uint64 // RandBytes is the number of bytes used as entropy to shuffle validators. - SyncPollingInterval int64 // SyncPollingInterval queries network nodes for sync status. - BatchBlockLimit uint64 // BatchBlockLimit is maximum number of blocks that can be requested for initial sync. - SyncEpochLimit uint64 // SyncEpochLimit is the number of epochs the current node can be behind before it requests for the latest state. - MaxNumLog2Validators uint64 // MaxNumLog2Validators is the Max number of validators in Log2 exists given total ETH supply. - LogBlockDelay int64 // Number of blocks to wait from the current head before processing logs from the deposit contract. + DepositsForChainStart uint64 // DepositsForChainStart defines how many validator deposits needed to kick off beacon chain. + RandBytes uint64 // RandBytes is the number of bytes used as entropy to shuffle validators. + SyncPollingInterval int64 // SyncPollingInterval queries network nodes for sync status. + BatchBlockLimit uint64 // BatchBlockLimit is maximum number of blocks that can be requested for initial sync. + SyncEpochLimit uint64 // SyncEpochLimit is the number of epochs the current node can be behind before it requests for the latest state. + MaxNumLog2Validators uint64 // MaxNumLog2Validators is the Max number of validators in Log2 exists given total ETH supply. + LogBlockDelay int64 // Number of blocks to wait from the current head before processing logs from the deposit contract. + RPCSyncCheck time.Duration // Number of seconds to query the sync service, to find out if the node is synced or not. } // DepositContractConfig contains the deposits for @@ -172,6 +174,7 @@ var defaultBeaconConfig = &BeaconChainConfig{ BatchBlockLimit: 64 * 4, // Process blocks in batches of 4 epochs of blocks (threshold before casper penalties). MaxNumLog2Validators: 24, LogBlockDelay: 2, // + RPCSyncCheck: 1, } var defaultShardConfig = &ShardChainConfig{ From e36714805a7f09aa1398c17129949d0f9056cd5d Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 23 Apr 2019 13:15:58 -0500 Subject: [PATCH 07/54] Enable Ancestor Cache By Default (#2358) * remove ancestor cache flag * ancestor cache on by default --- beacon-chain/blockchain/fork_choice.go | 15 +++------------ beacon-chain/blockchain/fork_choice_test.go | 9 --------- shared/featureconfig/config.go | 5 ----- shared/featureconfig/flags.go | 7 ------- 4 files changed, 3 insertions(+), 33 deletions(-) diff --git a/beacon-chain/blockchain/fork_choice.go b/beacon-chain/blockchain/fork_choice.go index a90bb4af3529..412a155c0789 100644 --- a/beacon-chain/blockchain/fork_choice.go +++ b/beacon-chain/blockchain/fork_choice.go @@ -12,7 +12,6 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/db" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/bytesutil" - "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/params" "github.com/sirupsen/logrus" @@ -329,17 +328,9 @@ func VoteCount(block *pb.BeaconBlock, state *pb.BeaconState, targets map[uint64] } for validatorIndex, target := range targets { - if featureconfig.FeatureConfig().EnableBlockAncestorCache { - ancestorRoot, err = cachedAncestor(target, block.Slot, beaconDB) - if err != nil { - return 0, err - } - } else { - // if block ancestor cache was not enabled, retrieve the ancestor recursively. - ancestorRoot, err = BlockAncestor(target, block.Slot, beaconDB) - if err != nil { - return 0, err - } + ancestorRoot, err = cachedAncestor(target, block.Slot, beaconDB) + if err != nil { + return 0, err } // This covers the following case, we start at B5, and want to process B6 and B7 // B6 can be processed, B7 can not be processed because it's pointed to the diff --git a/beacon-chain/blockchain/fork_choice_test.go b/beacon-chain/blockchain/fork_choice_test.go index 9d448b23d55d..2878062294d7 100644 --- a/beacon-chain/blockchain/fork_choice_test.go +++ b/beacon-chain/blockchain/fork_choice_test.go @@ -20,7 +20,6 @@ import ( pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/bls" "github.com/prysmaticlabs/prysm/shared/bytesutil" - "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/forkutil" "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/params" @@ -1443,10 +1442,6 @@ func setupFFGTest(t *testing.T) ([32]byte, *pb.BeaconBlock, *pb.BeaconState, []* } func TestVoteCount_CacheEnabledAndMiss(t *testing.T) { - featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ - EnableBlockAncestorCache: true, - }) - beaconDB := internal.SetupDB(t) defer internal.TeardownDB(t, beaconDB) genesisBlock := b.NewGenesisBlock([]byte("stateroot")) @@ -1514,10 +1509,6 @@ func TestVoteCount_CacheEnabledAndMiss(t *testing.T) { } func TestVoteCount_CacheEnabledAndHit(t *testing.T) { - featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ - EnableBlockAncestorCache: true, - }) - genesisBlock := b.NewGenesisBlock([]byte("stateroot")) genesisRoot, err := hashutil.HashBeaconBlock(genesisBlock) if err != nil { diff --git a/shared/featureconfig/config.go b/shared/featureconfig/config.go index b6d30588d31f..34c13f9133bc 100644 --- a/shared/featureconfig/config.go +++ b/shared/featureconfig/config.go @@ -30,7 +30,6 @@ type FeatureFlagConfig struct { EnableCrosslinks bool // EnableCrosslinks in epoch processing. EnableCheckBlockStateRoot bool // EnableCheckBlockStateRoot in block processing. EnableHistoricalStatePruning bool // EnableHistoricalStatePruning when updating finalized states. - EnableBlockAncestorCache bool //EnableBlockAncestorCache for fork choice optimization. DisableGossipSub bool // DisableGossipSub in p2p messaging. } @@ -73,10 +72,6 @@ func ConfigureBeaconFeatures(ctx *cli.Context) { log.Info("Enabled historical state pruning") cfg.EnableHistoricalStatePruning = true } - if ctx.GlobalBool(EnableBlockAncestorCacheFlag.Name) { - log.Info("Enabled block ancestor cache") - cfg.EnableBlockAncestorCache = true - } if ctx.GlobalBool(DisableGossipSubFlag.Name) { log.Info("Disabled gossipsub, using floodsub") cfg.DisableGossipSub = true diff --git a/shared/featureconfig/flags.go b/shared/featureconfig/flags.go index 1726d725522a..9d180526d35c 100644 --- a/shared/featureconfig/flags.go +++ b/shared/featureconfig/flags.go @@ -25,12 +25,6 @@ var ( Name: "enable-crosslinks", Usage: "Enable crosslinks in epoch processing, default is disabled.", } - // EnableBlockAncestorCacheFlag enables block ancestor cache for LMD GHOST fork choice optimization. I - // it is disabled by default. - EnableBlockAncestorCacheFlag = cli.BoolFlag{ - Name: "enable-block-ancestor-cache", - Usage: "Enable block ancestor cache for fork choice optimization, default is disabled.", - } // EnableCheckBlockStateRootFlag check block state root in block processing. It is disabled by default. EnableCheckBlockStateRootFlag = cli.BoolFlag{ Name: "enable-check-block-state-root", @@ -57,6 +51,5 @@ var BeaconChainFlags = []cli.Flag{ EnableCrosslinksFlag, EnableCheckBlockStateRootFlag, EnableHistoricalStatePruningFlag, - EnableBlockAncestorCacheFlag, DisableGossipSubFlag, } From e4bbf54f9f215c35def2c9e8d0d6e0766874dd6c Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 23 Apr 2019 15:30:59 -0500 Subject: [PATCH 08/54] Hardcoded Bootstrap + Relay Nodes (#2359) * default bootstrap node and local network flag * default relay node value * Update shared/p2p/service.go Co-Authored-By: rauljordan * use no discovery --- beacon-chain/node/p2p_config.go | 1 + beacon-chain/usage.go | 1 + shared/cmd/flags.go | 8 ++++++++ shared/p2p/service.go | 12 ++++++++---- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/beacon-chain/node/p2p_config.go b/beacon-chain/node/p2p_config.go index 6c5d84f3d1fb..b021b63ace70 100644 --- a/beacon-chain/node/p2p_config.go +++ b/beacon-chain/node/p2p_config.go @@ -28,6 +28,7 @@ var topicMappings = map[pb.Topic]proto.Message{ func configureP2P(ctx *cli.Context) (*p2p.Server, error) { s, err := p2p.NewServer(&p2p.ServerConfig{ + NoDiscovery: ctx.GlobalBool(cmd.NoDiscovery.Name), BootstrapNodeAddr: ctx.GlobalString(cmd.BootstrapNode.Name), RelayNodeAddr: ctx.GlobalString(cmd.RelayNode.Name), Port: ctx.GlobalInt(cmd.P2PPort.Name), diff --git a/beacon-chain/usage.go b/beacon-chain/usage.go index 8ad8a0de92a8..cc68a9cf9f2f 100644 --- a/beacon-chain/usage.go +++ b/beacon-chain/usage.go @@ -44,6 +44,7 @@ var appHelpFlagGroups = []flagGroup{ { Name: "cmd", Flags: []cli.Flag{ + cmd.NoDiscovery, cmd.BootstrapNode, cmd.RelayNode, cmd.P2PPort, diff --git a/shared/cmd/flags.go b/shared/cmd/flags.go index 848854a0beff..6aad14ee780b 100644 --- a/shared/cmd/flags.go +++ b/shared/cmd/flags.go @@ -47,16 +47,24 @@ var ( Usage: "Port used to listening and respond metrics for prometheus.", Value: 8080, } + // NoDiscovery specifies whether we are running a local network and have no need for connecting + // to the bootstrap nodes in the cloud + NoDiscovery = cli.BoolFlag{ + Name: "no-discovery", + Usage: "Enable only local network p2p and do not connect to cloud bootstrap nodes.", + } // BootstrapNode tells the beacon node which bootstrap node to connect to BootstrapNode = cli.StringFlag{ Name: "bootstrap-node", Usage: "The address of bootstrap node. Beacon node will connect for peer discovery via DHT", + Value: "/ip4/35.224.249.2/tcp/30001/p2p/QmQEe7o6hKJdGdSkJRh7WJzS6xrex5f4w2SPR6oWbJNriw", } // RelayNode tells the beacon node which relay node to connect to. RelayNode = cli.StringFlag{ Name: "relay-node", Usage: "The address of relay node. The beacon node will connect to the " + "relay node and advertise their address via the relay node to other peers", + Value: "/ip4/35.224.249.2/tcp/30000/p2p/QmfAgkmjiZNZhr2wFN9TwaRgHouMTBT6HELyzE5A3BT2wK", } // P2PPort defines the port to be used by libp2p. P2PPort = cli.IntFlag{ diff --git a/shared/p2p/service.go b/shared/p2p/service.go index af0b01c48ee9..16743e00a588 100644 --- a/shared/p2p/service.go +++ b/shared/p2p/service.go @@ -55,10 +55,12 @@ type Server struct { topicMapping map[reflect.Type]string bootstrapNode string relayNodeAddr string + noDiscovery bool } // ServerConfig for peer to peer networking. type ServerConfig struct { + NoDiscovery bool BootstrapNodeAddr string RelayNodeAddr string Port int @@ -112,6 +114,7 @@ func NewServer(cfg *ServerConfig) (*Server, error) { topicMapping: make(map[reflect.Type]string), bootstrapNode: cfg.BootstrapNodeAddr, relayNodeAddr: cfg.RelayNodeAddr, + noDiscovery: cfg.NoDiscovery, }, nil } @@ -139,7 +142,7 @@ func (s *Server) Start() { defer span.End() log.Info("Starting service") - if s.bootstrapNode != "" { + if !s.noDiscovery && s.bootstrapNode != "" { if err := startDHTDiscovery(ctx, s.host, s.bootstrapNode); err != nil { log.Errorf("Could not start peer discovery via DHT: %v", err) } @@ -149,8 +152,7 @@ func (s *Server) Start() { log.Errorf("Failed to bootstrap DHT: %v", err) } } - - if s.relayNodeAddr != "" { + if !s.noDiscovery && s.relayNodeAddr != "" { if err := dialRelayNode(ctx, s.host, s.relayNodeAddr); err != nil { log.Errorf("Could not dial relay node: %v", err) } @@ -161,7 +163,9 @@ func (s *Server) Start() { return } - startPeerWatcher(ctx, s.host, s.bootstrapNode, s.relayNodeAddr) + if !s.noDiscovery { + startPeerWatcher(ctx, s.host, s.bootstrapNode, s.relayNodeAddr) + } } // Stop the main p2p loop. From 44bc1c1bc624a036de32300ec1d9f2ae189540fa Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 23 Apr 2019 15:52:52 -0500 Subject: [PATCH 09/54] Suggest to ClearDB if Contract Mismatches Cluster (#2360) * clear db by default * revert changes * comment * run with clear db --- beacon-chain/db/verify_contract.go | 2 +- beacon-chain/main.go | 2 +- beacon-chain/node/node.go | 2 +- shared/cmd/flags.go | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/beacon-chain/db/verify_contract.go b/beacon-chain/db/verify_contract.go index c45e7c909e37..742269a7d864 100644 --- a/beacon-chain/db/verify_contract.go +++ b/beacon-chain/db/verify_contract.go @@ -29,7 +29,7 @@ func (db *BeaconDB) VerifyContractAddress(ctx context.Context, addr common.Addre } if !bytes.Equal(expectedAddress, addr.Bytes()) { - return fmt.Errorf("invalid deposit contract address, expected %#x", expectedAddress) + return fmt.Errorf("invalid deposit contract address, expected %#x - try running with --clear-db", expectedAddress) } return nil diff --git a/beacon-chain/main.go b/beacon-chain/main.go index 37b81e615545..3e7292867e1c 100644 --- a/beacon-chain/main.go +++ b/beacon-chain/main.go @@ -63,7 +63,7 @@ func main() { cmd.TraceSampleFractionFlag, cmd.MonitoringPortFlag, cmd.DisableMonitoringFlag, - cmd.ClearDBFlag, + cmd.ClearDB, debug.PProfFlag, debug.PProfAddrFlag, debug.PProfPortFlag, diff --git a/beacon-chain/node/node.go b/beacon-chain/node/node.go index 0ca0848aad89..4b529450c228 100644 --- a/beacon-chain/node/node.go +++ b/beacon-chain/node/node.go @@ -168,7 +168,7 @@ func (b *BeaconNode) Close() { func (b *BeaconNode) startDB(ctx *cli.Context) error { baseDir := ctx.GlobalString(cmd.DataDirFlag.Name) dbPath := path.Join(baseDir, beaconChainDBName) - if b.ctx.GlobalBool(cmd.ClearDBFlag.Name) { + if b.ctx.GlobalBool(cmd.ClearDB.Name) { if err := db.ClearDB(dbPath); err != nil { return err } diff --git a/shared/cmd/flags.go b/shared/cmd/flags.go index 6aad14ee780b..25aab174c275 100644 --- a/shared/cmd/flags.go +++ b/shared/cmd/flags.go @@ -72,8 +72,8 @@ var ( Usage: "The port used by libp2p.", Value: 12000, } - // ClearDBFlag tells the beacon node to remove any previously stored data at the data directory. - ClearDBFlag = cli.BoolFlag{ + // ClearDB tells the beacon node to remove any previously stored data at the data directory. + ClearDB = cli.BoolFlag{ Name: "clear-db", Usage: "Clears any previously stored data at the data directory", } From 376aacdad6947f87bb0e80c013c00beb15273445 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Tue, 23 Apr 2019 17:17:02 -0400 Subject: [PATCH 10/54] Use infura for http provider as well (#2363) --- k8s/beacon-chain/beacon-chain.deploy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/beacon-chain/beacon-chain.deploy.yaml b/k8s/beacon-chain/beacon-chain.deploy.yaml index c22d76514a95..6ef55ffeb48a 100644 --- a/k8s/beacon-chain/beacon-chain.deploy.yaml +++ b/k8s/beacon-chain/beacon-chain.deploy.yaml @@ -44,6 +44,7 @@ spec: image: gcr.io/prysmaticlabs/prysm/beacon-chain:latest args: - --web3provider=wss://goerli.infura.io/ws/v3/be3fb7ed377c418087602876a40affa1 + - --http-web3provider=https://goerli.infura.io/v3/be3fb7ed377c418087602876a40affa1 #- --verbosity=debug - --deposit-contract=$(DEPOSIT_CONTRACT_ADDRESS) - --rpc-port=4000 From 64e7db6241984f308f1cd06acdb43de4df5a1e31 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 23 Apr 2019 16:33:47 -0500 Subject: [PATCH 11/54] Resolve Contract Address Automatically via HTTP (#2361) * automatically resolve contract address * Update beacon-chain/node/node.go Co-Authored-By: rauljordan * testnet contract endpoint * fmt * use config in msg --- beacon-chain/node/node.go | 19 ++++++++++++++++++- shared/params/config.go | 20 ++++++++++++-------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/beacon-chain/node/node.go b/beacon-chain/node/node.go index 4b529450c228..056f86b3d499 100644 --- a/beacon-chain/node/node.go +++ b/beacon-chain/node/node.go @@ -4,6 +4,8 @@ package node import ( "context" "fmt" + "io/ioutil" + "net/http" "os" "os/signal" "path" @@ -246,7 +248,22 @@ func (b *BeaconNode) registerPOWChainService(cliCtx *cli.Context) error { depAddress := cliCtx.GlobalString(utils.DepositContractFlag.Name) if depAddress == "" { - log.Fatal("No deposit contract specified. Add --deposit-contract with a valid deposit contract address to start.") + log.Infof("Fetching testnet cluster address from %s...", params.BeaconConfig().TestnetContractEndpoint) + resp, err := http.Get(params.BeaconConfig().TestnetContractEndpoint) + if err != nil { + log.Fatalf("Could not get latest deposit contract address: %v", err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Fatal(err) + } + }() + + contractResponse, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatal(err) + } + depAddress = string(contractResponse) } if !common.IsHexAddress(depAddress) { diff --git a/shared/params/config.go b/shared/params/config.go index ad799394821a..d019a4ded447 100644 --- a/shared/params/config.go +++ b/shared/params/config.go @@ -78,14 +78,15 @@ type BeaconChainConfig struct { MaxAttesterSlashings uint64 // MaxAttesterSlashings defines the maximum number of casper FFG slashings possible in a block. // Prysm constants. - DepositsForChainStart uint64 // DepositsForChainStart defines how many validator deposits needed to kick off beacon chain. - RandBytes uint64 // RandBytes is the number of bytes used as entropy to shuffle validators. - SyncPollingInterval int64 // SyncPollingInterval queries network nodes for sync status. - BatchBlockLimit uint64 // BatchBlockLimit is maximum number of blocks that can be requested for initial sync. - SyncEpochLimit uint64 // SyncEpochLimit is the number of epochs the current node can be behind before it requests for the latest state. - MaxNumLog2Validators uint64 // MaxNumLog2Validators is the Max number of validators in Log2 exists given total ETH supply. - LogBlockDelay int64 // Number of blocks to wait from the current head before processing logs from the deposit contract. - RPCSyncCheck time.Duration // Number of seconds to query the sync service, to find out if the node is synced or not. + DepositsForChainStart uint64 // DepositsForChainStart defines how many validator deposits needed to kick off beacon chain. + RandBytes uint64 // RandBytes is the number of bytes used as entropy to shuffle validators. + SyncPollingInterval int64 // SyncPollingInterval queries network nodes for sync status. + BatchBlockLimit uint64 // BatchBlockLimit is maximum number of blocks that can be requested for initial sync. + SyncEpochLimit uint64 // SyncEpochLimit is the number of epochs the current node can be behind before it requests for the latest state. + MaxNumLog2Validators uint64 // MaxNumLog2Validators is the Max number of validators in Log2 exists given total ETH supply. + LogBlockDelay int64 // Number of blocks to wait from the current head before processing logs from the deposit contract. + RPCSyncCheck time.Duration // Number of seconds to query the sync service, to find out if the node is synced or not. + TestnetContractEndpoint string // TestnetContractEndpoint to fetch the contract address of the Prysmatic Labs testnet. } // DepositContractConfig contains the deposits for @@ -175,6 +176,9 @@ var defaultBeaconConfig = &BeaconChainConfig{ MaxNumLog2Validators: 24, LogBlockDelay: 2, // RPCSyncCheck: 1, + + // Testnet misc values. + TestnetContractEndpoint: "https://beta.prylabs.net/contract", // defines an http endpoint to fetch the testnet contract addr. } var defaultShardConfig = &ShardChainConfig{ From e7ec153390f88d80c57351a404b6081282327fed Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 23 Apr 2019 23:29:06 -0500 Subject: [PATCH 12/54] Interactive Validator Client Onboarding (#2362) --- validator/BUILD.bazel | 2 ++ validator/accounts/account.go | 22 ++++++++++++ validator/main.go | 67 ++++++++++++++++++++++++++++++----- validator/node/node.go | 9 +++-- validator/node/node_test.go | 2 +- validator/types/BUILD.bazel | 5 ++- validator/types/flags.go | 35 +++++++++++++++++- 7 files changed, 126 insertions(+), 16 deletions(-) diff --git a/validator/BUILD.bazel b/validator/BUILD.bazel index afc9aea28588..f0d35d329bd1 100644 --- a/validator/BUILD.bazel +++ b/validator/BUILD.bazel @@ -21,6 +21,7 @@ go_library( "@com_github_sirupsen_logrus//:go_default_library", "@com_github_urfave_cli//:go_default_library", "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library", + "@org_golang_x_crypto//ssh/terminal:go_default_library", ], ) @@ -49,6 +50,7 @@ go_image( "@com_github_sirupsen_logrus//:go_default_library", "@com_github_urfave_cli//:go_default_library", "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library", + "@org_golang_x_crypto//ssh/terminal:go_default_library", ], ) diff --git a/validator/accounts/account.go b/validator/accounts/account.go index a1974c3c4892..241e1ce828e7 100644 --- a/validator/accounts/account.go +++ b/validator/accounts/account.go @@ -6,6 +6,8 @@ import ( "encoding/hex" "errors" "fmt" + "io" + "os" "github.com/prysmaticlabs/prysm/shared/keystore" "github.com/prysmaticlabs/prysm/shared/params" @@ -87,3 +89,23 @@ func NewValidatorAccount(directory string, password string) error { `, serializedData) return nil } + +// Exists checks if a validator account at a given keystore path exists. +func Exists(keystorePath string) (bool, error) { + /* #nosec */ + f, err := os.Open(keystorePath) + if err != nil { + return false, nil + } + defer func() { + if err := f.Close(); err != nil { + log.Fatal(err) + } + }() + + _, err = f.Readdirnames(1) // Or f.Readdir(1) + if err == io.EOF { + return false, nil + } + return true, err +} diff --git a/validator/main.go b/validator/main.go index 1cc6a54743da..cacc0d4ca7f9 100644 --- a/validator/main.go +++ b/validator/main.go @@ -1,10 +1,12 @@ package main import ( - "errors" + "bufio" "fmt" "os" "runtime" + "strings" + "syscall" "github.com/prysmaticlabs/prysm/shared/cmd" "github.com/prysmaticlabs/prysm/shared/debug" @@ -16,13 +18,37 @@ import ( "github.com/sirupsen/logrus" "github.com/urfave/cli" prefixed "github.com/x-cray/logrus-prefixed-formatter" + "golang.org/x/crypto/ssh/terminal" ) func startNode(ctx *cli.Context) error { keystoreDirectory := ctx.String(types.KeystorePathFlag.Name) keystorePassword := ctx.String(types.PasswordFlag.Name) - if err := accounts.VerifyAccountNotExists(keystoreDirectory, keystorePassword); err == nil { - return errors.New("no account found, use `validator accounts create` to generate a new keystore") + + exists, err := accounts.Exists(keystoreDirectory) + if err != nil { + logrus.Fatal(err) + } + if !exists { + // If an account does not exist, we create a new one and start the node. + keystoreDirectory, keystorePassword, err = createValidatorAccount(ctx) + if err != nil { + logrus.Fatalf("Could not create validator account: %v", err) + } + } else { + if keystorePassword == "" { + logrus.Info("Enter your validator account password:") + bytePassword, err := terminal.ReadPassword(int(syscall.Stdin)) + if err != nil { + logrus.Fatalf("Could not read account password: %v", err) + } + text := string(bytePassword) + keystorePassword = strings.Replace(text, "\n", "", -1) + } + + if err := accounts.VerifyAccountNotExists(keystoreDirectory, keystorePassword); err == nil { + logrus.Info("No account found, creating new validator account...") + } } verbosity := ctx.GlobalString(cmd.VerbosityFlag.Name) @@ -32,7 +58,7 @@ func startNode(ctx *cli.Context) error { } logrus.SetLevel(level) - validatorClient, err := node.NewValidatorClient(ctx) + validatorClient, err := node.NewValidatorClient(ctx, keystorePassword) if err != nil { return err } @@ -41,13 +67,34 @@ func startNode(ctx *cli.Context) error { return nil } -func createValidatorAccount(ctx *cli.Context) error { +func createValidatorAccount(ctx *cli.Context) (string, string, error) { keystoreDirectory := ctx.String(types.KeystorePathFlag.Name) keystorePassword := ctx.String(types.PasswordFlag.Name) + if keystorePassword == "" { + reader := bufio.NewReader(os.Stdin) + logrus.Info("Create a new validator account for eth2") + logrus.Info("Enter a password:") + bytePassword, err := terminal.ReadPassword(int(syscall.Stdin)) + if err != nil { + logrus.Fatalf("Could not read account password: %v", err) + } + text := string(bytePassword) + keystorePassword = strings.Replace(text, "\n", "", -1) + logrus.Infof("Keystore path to save your private keys (leave blank for default %s):", keystoreDirectory) + text, err = reader.ReadString('\n') + if err != nil { + logrus.Fatal(err) + } + text = strings.Replace(text, "\n", "", -1) + if text != "" { + keystoreDirectory = text + } + } + if err := accounts.NewValidatorAccount(keystoreDirectory, keystorePassword); err != nil { - return fmt.Errorf("could not initialize validator account: %v", err) + return "", "", fmt.Errorf("could not initialize validator account: %v", err) } - return nil + return keystoreDirectory, keystorePassword, nil } func main() { @@ -77,7 +124,11 @@ contract in order to activate the validator client`, types.KeystorePathFlag, types.PasswordFlag, }, - Action: createValidatorAccount, + Action: func(ctx *cli.Context) { + if keystoreDir, _, err := createValidatorAccount(ctx); err != nil { + logrus.Fatalf("Could not create validator at path: %s", keystoreDir) + } + }, }, }, }, diff --git a/validator/node/node.go b/validator/node/node.go index 8aec596689e5..249df05b5392 100644 --- a/validator/node/node.go +++ b/validator/node/node.go @@ -37,7 +37,7 @@ type ValidatorClient struct { } // NewValidatorClient creates a new, Ethereum Serenity validator client. -func NewValidatorClient(ctx *cli.Context) (*ValidatorClient, error) { +func NewValidatorClient(ctx *cli.Context, password string) (*ValidatorClient, error) { if err := tracing.Setup( "validator", // service name ctx.GlobalString(cmd.TracingEndpointFlag.Name), @@ -65,7 +65,7 @@ func NewValidatorClient(ctx *cli.Context) (*ValidatorClient, error) { return nil, err } - if err := ValidatorClient.registerClientService(ctx); err != nil { + if err := ValidatorClient.registerClientService(ctx, password); err != nil { return nil, err } @@ -126,14 +126,13 @@ func (s *ValidatorClient) registerPrometheusService(ctx *cli.Context) error { return s.services.RegisterService(service) } -func (s *ValidatorClient) registerClientService(ctx *cli.Context) error { +func (s *ValidatorClient) registerClientService(ctx *cli.Context, password string) error { endpoint := ctx.GlobalString(types.BeaconRPCProviderFlag.Name) keystoreDirectory := ctx.GlobalString(types.KeystorePathFlag.Name) - keystorePassword := ctx.String(types.PasswordFlag.Name) v, err := client.NewValidatorService(context.Background(), &client.Config{ Endpoint: endpoint, KeystorePath: keystoreDirectory, - Password: keystorePassword, + Password: password, }) if err != nil { return fmt.Errorf("could not initialize client service: %v", err) diff --git a/validator/node/node_test.go b/validator/node/node_test.go index 216498c6f2d3..09eb05c599da 100644 --- a/validator/node/node_test.go +++ b/validator/node/node_test.go @@ -25,7 +25,7 @@ func TestNode_Builds(t *testing.T) { if err := accounts.NewValidatorAccount(dir, "1234"); err != nil { t.Fatalf("Could not create validator account: %v", err) } - _, err := NewValidatorClient(context) + _, err := NewValidatorClient(context, "1234") if err != nil { t.Fatalf("Failed to create ValidatorClient: %v", err) } diff --git a/validator/types/BUILD.bazel b/validator/types/BUILD.bazel index d8a250b6f835..633f530604aa 100644 --- a/validator/types/BUILD.bazel +++ b/validator/types/BUILD.bazel @@ -5,5 +5,8 @@ go_library( srcs = ["flags.go"], importpath = "github.com/prysmaticlabs/prysm/validator/types", visibility = ["//validator:__subpackages__"], - deps = ["@com_github_urfave_cli//:go_default_library"], + deps = [ + "//shared/cmd:go_default_library", + "@com_github_urfave_cli//:go_default_library", + ], ) diff --git a/validator/types/flags.go b/validator/types/flags.go index 7f7f3c737bab..676304122749 100644 --- a/validator/types/flags.go +++ b/validator/types/flags.go @@ -1,6 +1,12 @@ package types import ( + "os" + "os/user" + "path/filepath" + "runtime" + + "github.com/prysmaticlabs/prysm/shared/cmd" "github.com/urfave/cli" ) @@ -22,9 +28,10 @@ var ( Usage: "Certificate for secure gRPC. Pass this and the tls-key flag in order to use gRPC securely.", } // KeystorePathFlag defines the location of the keystore directory for a validator's account. - KeystorePathFlag = cli.StringFlag{ + KeystorePathFlag = cmd.DirectoryFlag{ Name: "keystore-path", Usage: "path to the desired keystore directory", + Value: cmd.DirectoryString{Value: defaultValidatorDir()}, } // PasswordFlag defines the password value for storing and retrieving validator private keys from the keystore. PasswordFlag = cli.StringFlag{ @@ -32,3 +39,29 @@ var ( Usage: "string value of the password for your validator private keys", } ) + +func homeDir() string { + if home := os.Getenv("HOME"); home != "" { + return home + } + if usr, err := user.Current(); err == nil { + return usr.HomeDir + } + return "" +} + +func defaultValidatorDir() string { + // Try to place the data folder in the user's home dir + home := homeDir() + if home != "" { + if runtime.GOOS == "darwin" { + return filepath.Join(home, "Library", "Eth2Validators") + } else if runtime.GOOS == "windows" { + return filepath.Join(home, "AppData", "Roaming", "Eth2Validators") + } else { + return filepath.Join(home, ".eth2validators") + } + } + // As we cannot guess a stable location, return empty and handle later + return "" +} From 6c1740eefc34440d4643f89efec2c52528565041 Mon Sep 17 00:00:00 2001 From: shayzluf Date: Wed, 24 Apr 2019 11:09:02 +0530 Subject: [PATCH 13/54] Add Caching to Tree Hashing Algorithm (#1929) * added todo to hash file in ssz * params and copy of block cache * start hash cache * Hash cache implementation * fixed some comments * fixed promatheus duplicate counter name * removed TODO * change to use special expiration cache * function name fixes junk object generator * naming changes * gazzle fix * added pruning last read data test * fixed gometallinter errors * fix benchmarks and no int64 not serializable * move struct from test * add feature flag * fix merge issues * add featureflag to beacon and validator * featureflag init for tests * added feature flag to all ssz dependent tests * remove setter func * replace k8s tweaked expiration cache to https://github.com/karlseguin/ccache * remove else * change request by preston * added init featureflags to genesis_test * Update shared/ssz/hash_cache.go add dot Co-Authored-By: shayzluf * Update shared/ssz/hash_cache.go Co-Authored-By: shayzluf * Update shared/ssz/hash_cache.go remove extra space Co-Authored-By: shayzluf * Update shared/params/config.go add dot Co-Authored-By: shayzluf * Update shared/featureconfig/config.go remove dot Co-Authored-By: shayzluf * Update shared/featureconfig/config.go remove dot Co-Authored-By: shayzluf * remove powchain from prometheus hash cache name * fixes fron change requests * fix change requests * remove faulty merge test * gazelle fix * fix fmt.sprintf * remove debug binary * fix gazelle --- WORKSPACE | 6 + .../stategenerator/state_generator_test.go | 5 +- beacon-chain/core/blocks/BUILD.bazel | 1 + .../core/blocks/block_operations_test.go | 7 + beacon-chain/core/helpers/BUILD.bazel | 1 + beacon-chain/core/helpers/deposits_test.go | 7 + beacon-chain/core/state/state_test.go | 10 +- beacon-chain/db/block_test.go | 7 + beacon-chain/powchain/BUILD.bazel | 1 + beacon-chain/powchain/log_processing_test.go | 7 + beacon-chain/sync/regular_sync_test.go | 4 + shared/featureconfig/config.go | 10 + shared/featureconfig/flags.go | 10 +- shared/keystore/BUILD.bazel | 1 + shared/keystore/deposit_input_test.go | 7 + shared/params/config.go | 2 + shared/sliceutil/BUILD.bazel | 12 +- shared/sliceutil/slice_generic_test.go | 7 + shared/ssz/BUILD.bazel | 12 + shared/ssz/decode_test.go | 8 + shared/ssz/hash.go | 10 + shared/ssz/hash_cache.go | 234 ++++++++++++++++++ shared/ssz/hash_cache_test.go | 148 +++++++++++ shared/ssz/ssz_utils_cache.go | 1 + validator/accounts/BUILD.bazel | 1 + validator/accounts/account_test.go | 7 + validator/client/BUILD.bazel | 1 + validator/client/service_test.go | 7 + validator/node/BUILD.bazel | 1 + validator/node/node_test.go | 7 + 30 files changed, 529 insertions(+), 13 deletions(-) create mode 100644 shared/ssz/hash_cache.go create mode 100644 shared/ssz/hash_cache_test.go diff --git a/WORKSPACE b/WORKSPACE index bf00f7b77236..bcf5834843c0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1064,3 +1064,9 @@ go_repository( commit = "502116f1a0a0c1140aab04fd3787489209b357d3", # v1.2.0 importpath = "github.com/grpc-ecosystem/go-grpc-prometheus", ) + +go_repository( + name = "com_github_karlseguin_ccache", + commit = "ec06cd93a07565b373789b0078ba88fe697fddd9", + importpath = "github.com/karlseguin/ccache", +) diff --git a/beacon-chain/blockchain/stategenerator/state_generator_test.go b/beacon-chain/blockchain/stategenerator/state_generator_test.go index bd4dd3ef76ef..d147de97e469 100644 --- a/beacon-chain/blockchain/stategenerator/state_generator_test.go +++ b/beacon-chain/blockchain/stategenerator/state_generator_test.go @@ -15,9 +15,10 @@ import ( ) func init() { - featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{}) + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) } - func TestGenerateState_OK(t *testing.T) { b, err := backend.NewSimulatedBackend() if err != nil { diff --git a/beacon-chain/core/blocks/BUILD.bazel b/beacon-chain/core/blocks/BUILD.bazel index 20cde46643e5..e2b9f0513682 100644 --- a/beacon-chain/core/blocks/BUILD.bazel +++ b/beacon-chain/core/blocks/BUILD.bazel @@ -41,6 +41,7 @@ go_test( "//beacon-chain/core/state:go_default_library", "//proto/beacon/p2p/v1:go_default_library", "//shared/bls:go_default_library", + "//shared/featureconfig:go_default_library", "//shared/forkutil:go_default_library", "//shared/hashutil:go_default_library", "//shared/params:go_default_library", diff --git a/beacon-chain/core/blocks/block_operations_test.go b/beacon-chain/core/blocks/block_operations_test.go index 5898fe23678f..afc7da53e726 100644 --- a/beacon-chain/core/blocks/block_operations_test.go +++ b/beacon-chain/core/blocks/block_operations_test.go @@ -15,12 +15,19 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/core/state" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/bls" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/forkutil" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/ssz" "github.com/prysmaticlabs/prysm/shared/trieutil" ) +func init() { + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) +} + func setupInitialDeposits(t *testing.T, numDeposits int) ([]*pb.Deposit, []*bls.SecretKey) { privKeys := make([]*bls.SecretKey, numDeposits) deposits := make([]*pb.Deposit, numDeposits) diff --git a/beacon-chain/core/helpers/BUILD.bazel b/beacon-chain/core/helpers/BUILD.bazel index bede86d6e0be..5fef0bed7efc 100644 --- a/beacon-chain/core/helpers/BUILD.bazel +++ b/beacon-chain/core/helpers/BUILD.bazel @@ -41,6 +41,7 @@ go_test( deps = [ "//beacon-chain/cache:go_default_library", "//proto/beacon/p2p/v1:go_default_library", + "//shared/featureconfig:go_default_library", "//shared/params:go_default_library", "@com_github_gogo_protobuf//proto:go_default_library", "@org_golang_google_grpc//codes:go_default_library", diff --git a/beacon-chain/core/helpers/deposits_test.go b/beacon-chain/core/helpers/deposits_test.go index 2001c2977eb6..c41acd2d634b 100644 --- a/beacon-chain/core/helpers/deposits_test.go +++ b/beacon-chain/core/helpers/deposits_test.go @@ -6,9 +6,16 @@ import ( "github.com/gogo/protobuf/proto" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/params" ) +func init() { + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) +} + func TestEncodeDecodeDepositInput_Ok(t *testing.T) { input := &pb.DepositInput{ Pubkey: []byte("key"), diff --git a/beacon-chain/core/state/state_test.go b/beacon-chain/core/state/state_test.go index 737207ca9371..63b472752029 100644 --- a/beacon-chain/core/state/state_test.go +++ b/beacon-chain/core/state/state_test.go @@ -8,14 +8,20 @@ import ( "testing" "time" - "github.com/prysmaticlabs/prysm/shared/hashutil" - "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" "github.com/prysmaticlabs/prysm/beacon-chain/core/state" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/featureconfig" + "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/params" ) +func init() { + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) +} + func TestGenesisBeaconState_OK(t *testing.T) { if params.BeaconConfig().SlotsPerEpoch != 64 { t.Errorf("SlotsPerEpoch should be 64 for these tests to pass") diff --git a/beacon-chain/db/block_test.go b/beacon-chain/db/block_test.go index 22fe1d7c864d..285193da5ab6 100644 --- a/beacon-chain/db/block_test.go +++ b/beacon-chain/db/block_test.go @@ -8,10 +8,17 @@ import ( "github.com/gogo/protobuf/proto" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/params" ) +func init() { + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) +} + func TestNilDB_OK(t *testing.T) { db := setupDB(t) defer teardownDB(t, db) diff --git a/beacon-chain/powchain/BUILD.bazel b/beacon-chain/powchain/BUILD.bazel index 969b62750665..c9e728f613d9 100644 --- a/beacon-chain/powchain/BUILD.bazel +++ b/beacon-chain/powchain/BUILD.bazel @@ -46,6 +46,7 @@ go_test( "//contracts/deposit-contract:go_default_library", "//proto/beacon/p2p/v1:go_default_library", "//shared/event:go_default_library", + "//shared/featureconfig:go_default_library", "//shared/ssz:go_default_library", "//shared/testutil:go_default_library", "@com_github_ethereum_go_ethereum//:go_default_library", diff --git a/beacon-chain/powchain/log_processing_test.go b/beacon-chain/powchain/log_processing_test.go index 9946af328f1c..2ebd064fb194 100644 --- a/beacon-chain/powchain/log_processing_test.go +++ b/beacon-chain/powchain/log_processing_test.go @@ -14,11 +14,18 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/db" contracts "github.com/prysmaticlabs/prysm/contracts/deposit-contract" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/ssz" "github.com/prysmaticlabs/prysm/shared/testutil" logTest "github.com/sirupsen/logrus/hooks/test" ) +func init() { + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) +} + func TestProcessDepositLog_OK(t *testing.T) { hook := logTest.NewGlobal() endpoint := "ws://127.0.0.1" diff --git a/beacon-chain/sync/regular_sync_test.go b/beacon-chain/sync/regular_sync_test.go index 47a8fa3c2cac..5d54d8dcfc49 100644 --- a/beacon-chain/sync/regular_sync_test.go +++ b/beacon-chain/sync/regular_sync_test.go @@ -14,6 +14,7 @@ import ( pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/bytesutil" "github.com/prysmaticlabs/prysm/shared/event" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/p2p" "github.com/prysmaticlabs/prysm/shared/params" @@ -25,6 +26,9 @@ import ( func init() { logrus.SetLevel(logrus.DebugLevel) logrus.SetOutput(ioutil.Discard) + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) } type mockP2P struct { diff --git a/shared/featureconfig/config.go b/shared/featureconfig/config.go index 34c13f9133bc..4e13ceec125e 100644 --- a/shared/featureconfig/config.go +++ b/shared/featureconfig/config.go @@ -31,6 +31,8 @@ type FeatureFlagConfig struct { EnableCheckBlockStateRoot bool // EnableCheckBlockStateRoot in block processing. EnableHistoricalStatePruning bool // EnableHistoricalStatePruning when updating finalized states. DisableGossipSub bool // DisableGossipSub in p2p messaging. + EnableCommitteesCache bool // EnableCommitteesCache for state transition. + CacheTreeHash bool // CacheTreeHash determent whether tree hashes will be cached. } var featureConfig *FeatureFlagConfig @@ -68,6 +70,10 @@ func ConfigureBeaconFeatures(ctx *cli.Context) { log.Info("Enabled check block state root") cfg.EnableCheckBlockStateRoot = true } + if ctx.GlobalBool(CacheTreeHashFlag.Name) { + log.Info("Cache tree hashes for ssz") + cfg.CacheTreeHash = true + } if ctx.GlobalBool(EnableHistoricalStatePruningFlag.Name) { log.Info("Enabled historical state pruning") cfg.EnableHistoricalStatePruning = true @@ -88,6 +94,10 @@ func ConfigureValidatorFeatures(ctx *cli.Context) { log.Info("Verifying signatures for attestations") cfg.VerifyAttestationSigs = true } + if ctx.GlobalBool(CacheTreeHashFlag.Name) { + log.Info("Cache tree hashes for ssz") + cfg.CacheTreeHash = true + } InitFeatureConfig(cfg) } diff --git a/shared/featureconfig/flags.go b/shared/featureconfig/flags.go index 9d180526d35c..a1345abe97fd 100644 --- a/shared/featureconfig/flags.go +++ b/shared/featureconfig/flags.go @@ -5,6 +5,11 @@ import ( ) var ( + // CacheTreeHashFlag determines whether to cache tree hashes for ssz. + CacheTreeHashFlag = cli.BoolFlag{ + Name: "enable-cache-tree-hash", + Usage: "Cache tree hashes for ssz", + } // VerifyAttestationSigsFlag determines whether to verify signatures for attestations. VerifyAttestationSigsFlag = cli.BoolFlag{ Name: "enable-attestation-signature-verification", @@ -43,7 +48,9 @@ var ( ) // ValidatorFlags contains a list of all the feature flags that apply to the validator client. -var ValidatorFlags = []cli.Flag{} +var ValidatorFlags = []cli.Flag{ + CacheTreeHashFlag, +} // BeaconChainFlags contains a list of all the feature flags that apply to the beacon-chain client. var BeaconChainFlags = []cli.Flag{ @@ -52,4 +59,5 @@ var BeaconChainFlags = []cli.Flag{ EnableCheckBlockStateRootFlag, EnableHistoricalStatePruningFlag, DisableGossipSubFlag, + CacheTreeHashFlag, } diff --git a/shared/keystore/BUILD.bazel b/shared/keystore/BUILD.bazel index d23af2daf820..9882d8a5ca92 100644 --- a/shared/keystore/BUILD.bazel +++ b/shared/keystore/BUILD.bazel @@ -35,6 +35,7 @@ go_test( "//proto/beacon/p2p/v1:go_default_library", "//shared/bls:go_default_library", "//shared/bytesutil:go_default_library", + "//shared/featureconfig:go_default_library", "//shared/params:go_default_library", "//shared/ssz:go_default_library", "//shared/testutil:go_default_library", diff --git a/shared/keystore/deposit_input_test.go b/shared/keystore/deposit_input_test.go index 67adc2eff24b..5113a84c01f7 100644 --- a/shared/keystore/deposit_input_test.go +++ b/shared/keystore/deposit_input_test.go @@ -8,11 +8,18 @@ import ( "github.com/gogo/protobuf/proto" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/bls" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/keystore" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/ssz" ) +func init() { + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) +} + func TestDepositInput_GeneratesPb(t *testing.T) { k1, err := keystore.NewKey(rand.Reader) if err != nil { diff --git a/shared/params/config.go b/shared/params/config.go index d019a4ded447..c1e3bfdfe2bc 100644 --- a/shared/params/config.go +++ b/shared/params/config.go @@ -24,6 +24,7 @@ type BeaconChainConfig struct { WithdrawalPrivkeyFileName string // WithdrawalPrivKeyFileName specifies the string name of a withdrawal private key file. BLSPubkeyLength int // BLSPubkeyLength defines the expected length of BLS public keys in bytes. DefaultBufferSize int // DefaultBufferSize for channels across the Prysm repository. + HashCacheSize int64 // HashCacheSize defines the size of object hashes that are cached. // BLS domain values. DomainDeposit uint64 // DomainDeposit defines the BLS signature domain for deposit verification. @@ -118,6 +119,7 @@ var defaultBeaconConfig = &BeaconChainConfig{ WithdrawalPrivkeyFileName: "/shardwithdrawalkey", BLSPubkeyLength: 96, DefaultBufferSize: 10000, + HashCacheSize: 100000, // BLS domain values. DomainDeposit: 0, diff --git a/shared/sliceutil/BUILD.bazel b/shared/sliceutil/BUILD.bazel index 6192e886e743..200b4117e7fb 100644 --- a/shared/sliceutil/BUILD.bazel +++ b/shared/sliceutil/BUILD.bazel @@ -18,12 +18,8 @@ go_test( "slice_test.go", ], embed = [":go_default_library"], - deps = ["//shared/ssz:go_default_library"], -) - -go_test( - name = "go_generic_test", - srcs = ["slice_generic_test.go"], - embed = [":go_default_library"], - deps = ["//shared/ssz:go_default_library"], + deps = [ + "//shared/featureconfig:go_default_library", + "//shared/ssz:go_default_library", + ], ) diff --git a/shared/sliceutil/slice_generic_test.go b/shared/sliceutil/slice_generic_test.go index 586232248132..642bd436616b 100644 --- a/shared/sliceutil/slice_generic_test.go +++ b/shared/sliceutil/slice_generic_test.go @@ -5,9 +5,16 @@ import ( "reflect" "testing" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/ssz" ) +func init() { + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) +} + func TestGenericIntersection(t *testing.T) { testCases := []struct { setA []uint64 diff --git a/shared/ssz/BUILD.bazel b/shared/ssz/BUILD.bazel index 6f9b5d6a4704..ab5c484ef097 100644 --- a/shared/ssz/BUILD.bazel +++ b/shared/ssz/BUILD.bazel @@ -7,13 +7,20 @@ go_library( "doc.go", "encode.go", "hash.go", + "hash_cache.go", "ssz_utils_cache.go", ], importpath = "github.com/prysmaticlabs/prysm/shared/ssz", visibility = ["//visibility:public"], deps = [ "//shared/bytesutil:go_default_library", + "//shared/featureconfig:go_default_library", "//shared/hashutil:go_default_library", + "//shared/params:go_default_library", + "@com_github_ethereum_go_ethereum//common:go_default_library", + "@com_github_karlseguin_ccache//:go_default_library", + "@com_github_prometheus_client_golang//prometheus:go_default_library", + "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", ], ) @@ -24,7 +31,12 @@ go_test( "encode_test.go", "example_and_test.go", "example_encode_test.go", + "hash_cache_test.go", "hash_test.go", ], embed = [":go_default_library"], + deps = [ + "//shared/bytesutil:go_default_library", + "//shared/featureconfig:go_default_library", + ], ) diff --git a/shared/ssz/decode_test.go b/shared/ssz/decode_test.go index 34cf7fb51549..6f4f1466fb3c 100644 --- a/shared/ssz/decode_test.go +++ b/shared/ssz/decode_test.go @@ -6,6 +6,8 @@ import ( "fmt" "reflect" "testing" + + "github.com/prysmaticlabs/prysm/shared/featureconfig" ) type decodeTest struct { @@ -184,6 +186,12 @@ var decodeTests = []decodeTest{ {input: "04000000 0200 01 01", ptr: new(simpleStruct), error: "decode error: input is too long for output type ssz.simpleStruct"}, } +func init() { + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) +} + func runTests(t *testing.T, decode func([]byte, interface{}) error) { for i, test := range decodeTests { input, err := hex.DecodeString(stripSpace(test.input)) diff --git a/shared/ssz/hash.go b/shared/ssz/hash.go index ddfa9a87d8fb..d4a2b816c397 100644 --- a/shared/ssz/hash.go +++ b/shared/ssz/hash.go @@ -7,12 +7,15 @@ import ( "reflect" "github.com/prysmaticlabs/prysm/shared/bytesutil" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/hashutil" ) const hashLengthBytes = 32 const sszChunkSize = 128 +var useCache bool + // Hashable defines the interface for supporting tree-hash function. type Hashable interface { TreeHashSSZ() ([32]byte, error) @@ -51,6 +54,7 @@ func newHashError(msg string, typ reflect.Type) *hashError { } func makeHasher(typ reflect.Type) (hasher, error) { + useCache = featureconfig.FeatureConfig().CacheTreeHash kind := typ.Kind() switch { case kind == reflect.Bool || @@ -64,8 +68,14 @@ func makeHasher(typ reflect.Type) (hasher, error) { kind == reflect.Array && typ.Elem().Kind() == reflect.Uint8: return hashedEncoding, nil case kind == reflect.Slice || kind == reflect.Array: + if useCache { + return makeSliceHasherCache(typ) + } return makeSliceHasher(typ) case kind == reflect.Struct: + if useCache { + return makeStructHasherCache(typ) + } return makeStructHasher(typ) case kind == reflect.Ptr: return makePtrHasher(typ) diff --git a/shared/ssz/hash_cache.go b/shared/ssz/hash_cache.go new file mode 100644 index 000000000000..22320054a71d --- /dev/null +++ b/shared/ssz/hash_cache.go @@ -0,0 +1,234 @@ +package ssz + +import ( + "errors" + "fmt" + "reflect" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/karlseguin/ccache" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prysmaticlabs/prysm/shared/bytesutil" + "github.com/prysmaticlabs/prysm/shared/hashutil" + "github.com/prysmaticlabs/prysm/shared/params" +) + +var ( + // ErrNotMerkleRoot will be returned when a cache object is not a merkle root. + ErrNotMerkleRoot = errors.New("object is not a merkle root") + // maxCacheSize is 2x of the follow distance for additional cache padding. + // Requests should be only accessing blocks within recent blocks within the + // Eth1FollowDistance. + maxCacheSize = params.BeaconConfig().HashCacheSize + // Metrics + hashCacheMiss = promauto.NewCounter(prometheus.CounterOpts{ + Name: "hash_cache_miss", + Help: "The number of hash requests that aren't present in the cache.", + }) + hashCacheHit = promauto.NewCounter(prometheus.CounterOpts{ + Name: "hash_cache_hit", + Help: "The number of hash requests that are present in the cache.", + }) + hashCacheSize = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "hash_cache_size", + Help: "The number of hashes in the block cache", + }) +) + +// hashCacheS struct with one queue for looking up by hash. +type hashCacheS struct { + hashCache *ccache.Cache +} + +// root specifies the hash of data in a struct +type root struct { + Hash common.Hash + MerkleRoot []byte +} + +// newHashCache creates a new hash cache for storing/accessing root hashes from +// memory. +func newHashCache() *hashCacheS { + return &hashCacheS{ + hashCache: ccache.New(ccache.Configure().MaxSize(maxCacheSize)), + } +} + +// RootByEncodedHash fetches Root by the encoded hash of the object. Returns true with a +// reference to the root if exists. Otherwise returns false, nil. +func (b *hashCacheS) RootByEncodedHash(h common.Hash) (bool, *root, error) { + item := b.hashCache.Get(h.Hex()) + if item == nil { + hashCacheMiss.Inc() + return false, nil, nil + } + hashCacheHit.Inc() + hInfo, ok := item.Value().(*root) + if !ok { + return false, nil, ErrNotMerkleRoot + } + + return true, hInfo, nil +} + +// TrieRootCached computes a trie root and add it to the cache. +// if the encoded hash of the object is in cache, it will be retrieved from cache. +// This method also trims the least recently added root info. if the cache size +// has reached the max cache size limit. +func (b *hashCacheS) TrieRootCached(val interface{}) ([32]byte, error) { + if val == nil { + return [32]byte{}, newHashError("untyped nil is not supported", nil) + } + rval := reflect.ValueOf(val) + hs, err := hashedEncoding(rval) + if err != nil { + return [32]byte{}, newHashError(error.Error(err), rval.Type()) + } + exists, fetchedInfo, err := b.RootByEncodedHash(bytesutil.ToBytes32(hs)) + if err != nil { + return [32]byte{}, newHashError(fmt.Sprint(err), rval.Type()) + } + var paddedOutput [32]byte + if exists { + paddedOutput = bytesutil.ToBytes32(fetchedInfo.MerkleRoot) + } else { + sszUtils, err := cachedSSZUtils(rval.Type()) + if err != nil { + return [32]byte{}, newHashError(fmt.Sprint(err), rval.Type()) + } + output, err := sszUtils.hasher(rval) + if err != nil { + return [32]byte{}, newHashError(fmt.Sprint(err), rval.Type()) + } + // Right-pad with 0 to make 32 bytes long, if necessary. + paddedOutput = bytesutil.ToBytes32(output) + err = b.AddRoot(bytesutil.ToBytes32(hs), paddedOutput[:]) + if err != nil { + return [32]byte{}, newHashError(fmt.Sprint(err), rval.Type()) + } + } + return paddedOutput, nil +} + +// MerkleHashCached adds a merkle object to the cache. This method also trims the +// least recently added root info if the cache size has reached the max cache +// size limit. +func (b *hashCacheS) MerkleHashCached(byteSlice [][]byte) ([]byte, error) { + mh := []byte{} + hs, err := hashedEncoding(reflect.ValueOf(byteSlice)) + if err != nil { + return mh, newHashError(fmt.Sprint(err), reflect.TypeOf(byteSlice)) + } + exists, fetchedInfo, err := b.RootByEncodedHash(bytesutil.ToBytes32(hs)) + if err != nil { + return mh, newHashError(fmt.Sprint(err), reflect.TypeOf(byteSlice)) + } + if exists { + mh = fetchedInfo.MerkleRoot + } else { + mh, err = merkleHash(byteSlice) + if err != nil { + return nil, err + } + mr := &root{ + Hash: bytesutil.ToBytes32(hs), + MerkleRoot: mh, + } + b.hashCache.Set(mr.Hash.Hex(), mr, time.Hour) + hashCacheSize.Set(float64(b.hashCache.ItemCount())) + } + + return mh, nil +} + +// AddRoot adds an encodedhash of the object as key and a rootHash object to the cache. +// This method also trims the +// least recently added root info if the cache size has reached the max cache +// size limit. +func (b *hashCacheS) AddRoot(h common.Hash, rootB []byte) error { + mr := &root{ + Hash: h, + MerkleRoot: rootB, + } + b.hashCache.Set(mr.Hash.Hex(), mr, time.Hour) + return nil +} + +// MakeSliceHasherCache add caching mechanism to slice hasher. +func makeSliceHasherCache(typ reflect.Type) (hasher, error) { + elemSSZUtils, err := cachedSSZUtilsNoAcquireLock(typ.Elem()) + if err != nil { + return nil, fmt.Errorf("failed to get ssz utils: %v", err) + } + hasher := func(val reflect.Value) ([]byte, error) { + hs, err := hashedEncoding(val) + if err != nil { + return nil, fmt.Errorf("failed to encode element of slice/array: %v", err) + } + exists, fetchedInfo, err := hashCache.RootByEncodedHash(bytesutil.ToBytes32(hs)) + if err != nil { + return nil, fmt.Errorf("failed to encode element of slice/array: %v", err) + } + var output []byte + if exists { + output = fetchedInfo.MerkleRoot + } else { + var elemHashList [][]byte + for i := 0; i < val.Len(); i++ { + elemHash, err := elemSSZUtils.hasher(val.Index(i)) + if err != nil { + return nil, fmt.Errorf("failed to hash element of slice/array: %v", err) + } + elemHashList = append(elemHashList, elemHash) + } + output, err = hashCache.MerkleHashCached(elemHashList) + if err != nil { + return nil, fmt.Errorf("failed to calculate merkle hash of element hash list: %v", err) + } + err := hashCache.AddRoot(bytesutil.ToBytes32(hs), output) + if err != nil { + return nil, fmt.Errorf("failed to add root to cache: %v", err) + } + hashCacheSize.Set(float64(hashCache.hashCache.ItemCount())) + + } + + return output, nil + } + return hasher, nil +} + +func makeStructHasherCache(typ reflect.Type) (hasher, error) { + fields, err := structFields(typ) + if err != nil { + return nil, err + } + hasher := func(val reflect.Value) ([]byte, error) { + hs, err := hashedEncoding(val) + if err != nil { + return nil, fmt.Errorf("failed to encode element of slice/array: %v", err) + } + exists, fetchedInfo, err := hashCache.RootByEncodedHash(bytesutil.ToBytes32(hs)) + if err != nil { + return nil, fmt.Errorf("failed to encode element of slice/array: %v", err) + } + var result [32]byte + if exists { + result = bytesutil.ToBytes32(fetchedInfo.MerkleRoot) + return result[:], nil + } + concatElemHash := make([]byte, 0) + for _, f := range fields { + elemHash, err := f.sszUtils.hasher(val.Field(f.index)) + if err != nil { + return nil, fmt.Errorf("failed to hash field of struct: %v", err) + } + concatElemHash = append(concatElemHash, elemHash...) + } + result = hashutil.Hash(concatElemHash) + return result[:], nil + } + return hasher, nil +} diff --git a/shared/ssz/hash_cache_test.go b/shared/ssz/hash_cache_test.go new file mode 100644 index 000000000000..bdfcf6fda758 --- /dev/null +++ b/shared/ssz/hash_cache_test.go @@ -0,0 +1,148 @@ +package ssz + +import ( + "bytes" + "log" + "reflect" + "testing" + "time" + + "github.com/prysmaticlabs/prysm/shared/bytesutil" + "github.com/prysmaticlabs/prysm/shared/featureconfig" +) + +type junkObject struct { + D2Int64Slice [][]uint64 + Uint uint64 + Int64Slice []uint64 +} + +type tree struct { + First []*junkObject + Second []*junkObject +} + +func generateJunkObject(size uint64) []*junkObject { + object := make([]*junkObject, size) + for i := uint64(0); i < uint64(len(object)); i++ { + d2Int64Slice := make([][]uint64, size) + is := make([]uint64, size) + uInt := uint64(time.Now().UnixNano()) + is[i] = i + d2Int64Slice[i] = make([]uint64, size) + for j := uint64(0); j < uint64(len(object)); j++ { + d2Int64Slice[i][j] = i + j + } + object[i] = &junkObject{ + D2Int64Slice: d2Int64Slice, + Uint: uInt, + Int64Slice: is, + } + + } + return object +} + +func TestObjCache_byHash(t *testing.T) { + cache := newHashCache() + byteSl := [][]byte{{0, 0}, {1, 1}} + mr, err := merkleHash(byteSl) + if err != nil { + t.Fatal(err) + } + hs, err := hashedEncoding(reflect.ValueOf(byteSl)) + if err != nil { + t.Fatal(err) + } + exists, _, err := cache.RootByEncodedHash(bytesutil.ToBytes32(hs)) + if err != nil { + t.Fatal(err) + } + if exists { + t.Error("Expected block info not to exist in empty cache") + } + if _, err := cache.MerkleHashCached(byteSl); err != nil { + t.Fatal(err) + } + exists, fetchedInfo, err := cache.RootByEncodedHash(bytesutil.ToBytes32(hs)) + if err != nil { + t.Fatal(err) + } + if !exists { + t.Error("Expected blockInfo to exist") + } + if !bytes.Equal(mr, fetchedInfo.MerkleRoot) { + t.Errorf( + "Expected fetched info number to be %v, got %v", + mr, + fetchedInfo.MerkleRoot, + ) + } + if fetchedInfo.Hash != bytesutil.ToBytes32(hs) { + t.Errorf( + "Expected fetched info hash to be %v, got %v", + hs, + fetchedInfo.Hash, + ) + } +} + +func TestMerkleHashWithCache(t *testing.T) { + cache := newHashCache() + for i := 0; i < 200; i++ { + runMerkleHashTests(t, func(val [][]byte) ([]byte, error) { + return merkleHash(val) + }) + } + for i := 0; i < 200; i++ { + runMerkleHashTests(t, func(val [][]byte) ([]byte, error) { + return cache.MerkleHashCached(val) + }) + } +} + +func BenchmarkHashWithoutCache(b *testing.B) { + featureconfig.FeatureConfig().CacheTreeHash = false + First := generateJunkObject(100) + TreeHash(&tree{First: First, Second: First}) + for n := 0; n < b.N; n++ { + TreeHash(&tree{First: First, Second: First}) + } +} + +func BenchmarkHashWithCache(b *testing.B) { + featureconfig.FeatureConfig().CacheTreeHash = true + First := generateJunkObject(100) + type tree struct { + First []*junkObject + Second []*junkObject + } + TreeHash(&tree{First: First, Second: First}) + for n := 0; n < b.N; n++ { + TreeHash(&tree{First: First, Second: First}) + } +} + +func TestBlockCache_maxSize(t *testing.T) { + maxCacheSize = 10000 + cache := newHashCache() + for i := uint64(0); i < uint64(maxCacheSize+1025); i++ { + + if err := cache.AddRoot(bytesutil.ToBytes32(bytesutil.Bytes4(i)), []byte{1}); err != nil { + t.Fatal(err) + } + } + log.Printf( + "hash cache key size is %d, itemcount is %d", + maxCacheSize, + cache.hashCache.ItemCount(), + ) + time.Sleep(1 * time.Second) + if int64(cache.hashCache.ItemCount()) > maxCacheSize { + t.Errorf( + "Expected hash cache key size to be %d, got %d", + maxCacheSize, + cache.hashCache.ItemCount(), + ) + } +} diff --git a/shared/ssz/ssz_utils_cache.go b/shared/ssz/ssz_utils_cache.go index e2ee369d91e1..e1309713d86b 100644 --- a/shared/ssz/ssz_utils_cache.go +++ b/shared/ssz/ssz_utils_cache.go @@ -29,6 +29,7 @@ type sszUtils struct { var ( sszUtilsCacheMutex sync.RWMutex sszUtilsCache = make(map[reflect.Type]*sszUtils) + hashCache = newHashCache() ) // Get cached encoder, encodeSizer and decoder implementation for a specified type. diff --git a/validator/accounts/BUILD.bazel b/validator/accounts/BUILD.bazel index 500c66703dcb..3dcf121a3c94 100644 --- a/validator/accounts/BUILD.bazel +++ b/validator/accounts/BUILD.bazel @@ -18,6 +18,7 @@ go_test( srcs = ["account_test.go"], embed = [":go_default_library"], deps = [ + "//shared/featureconfig:go_default_library", "//shared/keystore:go_default_library", "//shared/params:go_default_library", "//shared/testutil:go_default_library", diff --git a/validator/accounts/account_test.go b/validator/accounts/account_test.go index 72f35fd7a19d..c2656a5b4023 100644 --- a/validator/accounts/account_test.go +++ b/validator/accounts/account_test.go @@ -6,11 +6,18 @@ import ( "os" "testing" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/keystore" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/testutil" ) +func init() { + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) +} + func TestNewValidatorAccount_AccountExists(t *testing.T) { directory := testutil.TempDir() + "/testkeystore" defer os.RemoveAll(directory) diff --git a/validator/client/BUILD.bazel b/validator/client/BUILD.bazel index 97902f6cfa06..801d622d11fd 100644 --- a/validator/client/BUILD.bazel +++ b/validator/client/BUILD.bazel @@ -51,6 +51,7 @@ go_test( "//proto/beacon/rpc/v1:go_default_library", "//shared:go_default_library", "//shared/bitutil:go_default_library", + "//shared/featureconfig:go_default_library", "//shared/keystore:go_default_library", "//shared/mathutil:go_default_library", "//shared/params:go_default_library", diff --git a/validator/client/service_test.go b/validator/client/service_test.go index 69cbc17d1d9d..201ef3d23bf2 100644 --- a/validator/client/service_test.go +++ b/validator/client/service_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/prysmaticlabs/prysm/shared" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/keystore" "github.com/prysmaticlabs/prysm/shared/testutil" "github.com/prysmaticlabs/prysm/validator/accounts" @@ -34,6 +35,12 @@ func keySetup() { } } +func init() { + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) +} + func TestMain(m *testing.M) { dir := testutil.TempDir() + "/keystore1" defer os.RemoveAll(dir) diff --git a/validator/node/BUILD.bazel b/validator/node/BUILD.bazel index 4d261ba691eb..0c9818a8a96b 100644 --- a/validator/node/BUILD.bazel +++ b/validator/node/BUILD.bazel @@ -5,6 +5,7 @@ go_test( srcs = ["node_test.go"], embed = [":go_default_library"], deps = [ + "//shared/featureconfig:go_default_library", "//shared/testutil:go_default_library", "//validator/accounts:go_default_library", "@com_github_urfave_cli//:go_default_library", diff --git a/validator/node/node_test.go b/validator/node/node_test.go index 09eb05c599da..703181952551 100644 --- a/validator/node/node_test.go +++ b/validator/node/node_test.go @@ -5,11 +5,18 @@ import ( "os" "testing" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/testutil" "github.com/prysmaticlabs/prysm/validator/accounts" "github.com/urfave/cli" ) +func init() { + featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ + CacheTreeHash: false, + }) +} + // Test that the sharding node can build with default flag values. func TestNode_Builds(t *testing.T) { app := cli.NewApp() From 89941eec979e6402edd00099bb1d623547d39f45 Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Wed, 24 Apr 2019 19:03:55 +0800 Subject: [PATCH 14/54] Fix State Reads (#2368) * fix state reads * add method to retrieve validator from state * lint * gosec --- beacon-chain/attestation/service.go | 9 +-- beacon-chain/db/db.go | 14 +++-- beacon-chain/db/state.go | 90 ++++++++++++++++++++--------- beacon-chain/db/state_test.go | 5 +- 4 files changed, 76 insertions(+), 42 deletions(-) diff --git a/beacon-chain/attestation/service.go b/beacon-chain/attestation/service.go index 8040c7318f28..5b94a2d1b3f8 100644 --- a/beacon-chain/attestation/service.go +++ b/beacon-chain/attestation/service.go @@ -94,17 +94,12 @@ func (a *Service) IncomingAttestationFeed() *event.Feed { // Attestation` be the attestation with the highest slot number in `store` // from the validator with the given `validator_index` func (a *Service) LatestAttestation(ctx context.Context, index uint64) (*pb.Attestation, error) { - validatorRegistry, err := a.beaconDB.ValidatorRegistry(ctx) + validator, err := a.beaconDB.ValidatorFromState(ctx, index) if err != nil { return nil, err } - // return error if it's an invalid validator index. - if index >= uint64(len(validatorRegistry)) { - return nil, fmt.Errorf("invalid validator index %d", index) - } - pubKey := bytesutil.ToBytes48(validatorRegistry[index].Pubkey) - + pubKey := bytesutil.ToBytes48(validator.Pubkey) a.store.RLock() defer a.store.RUnlock() if _, exists := a.store.m[pubKey]; !exists { diff --git a/beacon-chain/db/db.go b/beacon-chain/db/db.go index 2e13d7445251..7b998576dfac 100644 --- a/beacon-chain/db/db.go +++ b/beacon-chain/db/db.go @@ -20,12 +20,14 @@ var log = logrus.WithField("prefix", "beacondb") // For example, instead of defining get, put, remove // This defines methods such as getBlock, saveBlocksAndAttestations, etc. type BeaconDB struct { - stateLock sync.RWMutex - currentState *pb.BeaconState - serializedState []byte - stateHash [32]byte - db *bolt.DB - DatabasePath string + // state objects and caches + stateLock sync.RWMutex + serializedState []byte + stateHash [32]byte + validatorRegistry []*pb.Validator + validatorBalances []uint64 + db *bolt.DB + DatabasePath string // Beacon block info in memory. highestBlockSlot uint64 diff --git a/beacon-chain/db/state.go b/beacon-chain/db/state.go index 67bbee13a85b..ccce45ae57fe 100644 --- a/beacon-chain/db/state.go +++ b/beacon-chain/db/state.go @@ -1,7 +1,6 @@ package db import ( - "bytes" "context" "encoding/binary" "errors" @@ -47,7 +46,6 @@ func (db *BeaconDB) InitializeState(ctx context.Context, genesisTime uint64, dep blockEnc, _ := proto.Marshal(genesisBlock) zeroBinary := encodeSlotNumber(0) - db.currentState = beaconState db.serializedState = stateEnc db.stateHash = stateHash @@ -102,21 +100,15 @@ func (db *BeaconDB) HeadState(ctx context.Context) (*pb.BeaconState, error) { lockSpan.End() // Return in-memory cached state, if available. - if db.currentState != nil { + if db.serializedState != nil { _, span := trace.StartSpan(ctx, "proto.Marshal") defer span.End() - // For each READ, we compare the cached state with a serialized copy of the state - // to determine if the cached state has been corrupted. If the marshaled version - // of the cached state is not equal to the serialized copy, we then retrieve the - // state from disk and log an error. - enc, err := proto.Marshal(db.currentState) - if err != nil { + newState := &pb.BeaconState{} + // For each READ we unmarshal the serialized state into a new state struct and return that. + if err := proto.Unmarshal(db.serializedState, newState); err != nil { return nil, err } - if bytes.Equal(enc, db.serializedState) { - return db.currentState, nil - } - log.Error("Cached state has been mutated, so retrieving state from disk") + return newState, nil } var beaconState *pb.BeaconState @@ -134,7 +126,7 @@ func (db *BeaconDB) HeadState(ctx context.Context) (*pb.BeaconState, error) { db.highestBlockSlot = beaconState.Slot } db.serializedState = enc - db.currentState = beaconState + db.stateHash = hashutil.Hash(enc) return err }) @@ -157,17 +149,18 @@ func (db *BeaconDB) SaveState(ctx context.Context, beaconState *pb.BeaconState) defer db.stateLock.Unlock() lockSpan.End() - db.currentState = beaconState - // For each WRITE of the state, we serialize the inputted state and save it in memory, - // then the inputted state is assigned our cached state. Further on we write the state to disk then. - // This removes the need to clone the state, as we handle mutated caches in each READ. + // and then the state is saved to disk. enc, err := proto.Marshal(beaconState) if err != nil { return err } stateHash := hashutil.Hash(enc) + tempState := &pb.BeaconState{} + tempState.ValidatorRegistry = beaconState.ValidatorRegistry + copy(db.validatorBalances, beaconState.ValidatorBalances) + db.validatorRegistry = proto.Clone(tempState).(*pb.BeaconState).ValidatorRegistry db.serializedState = enc db.stateHash = stateHash @@ -336,14 +329,14 @@ func (db *BeaconDB) ValidatorRegistry(ctx context.Context) ([]*pb.Validator, err defer db.stateLock.RUnlock() // Return in-memory cached state, if available. - if db.currentState != nil { + if db.validatorRegistry != nil { _, span := trace.StartSpan(ctx, "proto.Clone.ValidatorRegistry") defer span.End() - newRegistry := make([]*pb.Validator, len(db.currentState.ValidatorRegistry)) - for i, v := range db.currentState.ValidatorRegistry { - newRegistry[i] = proto.Clone(v).(*pb.Validator) + tempState := &pb.BeaconState{ + ValidatorRegistry: db.validatorRegistry, } - return newRegistry, nil + newState := proto.Clone(tempState).(*pb.BeaconState) + return newState.ValidatorRegistry, nil } var beaconState *pb.BeaconState @@ -365,20 +358,61 @@ func (db *BeaconDB) ValidatorRegistry(ctx context.Context) ([]*pb.Validator, err return beaconState.ValidatorRegistry, err } +// ValidatorFromState fetches the validator with the desired index from the cached registry. +func (db *BeaconDB) ValidatorFromState(ctx context.Context, index uint64) (*pb.Validator, error) { + ctx, span := trace.StartSpan(ctx, "BeaconDB.ValidatorFromState") + defer span.End() + + db.stateLock.RLock() + defer db.stateLock.RUnlock() + + if db.validatorRegistry != nil { + // return error if it's an invalid validator index. + if index >= uint64(len(db.validatorRegistry)) { + return nil, fmt.Errorf("invalid validator index %d", index) + } + validator := proto.Clone(db.validatorRegistry[index]).(*pb.Validator) + return validator, nil + } + + var beaconState *pb.BeaconState + err := db.view(func(tx *bolt.Tx) error { + chainInfo := tx.Bucket(chainInfoBucket) + enc := chainInfo.Get(stateLookupKey) + if enc == nil { + return nil + } + + var err error + beaconState, err = createState(enc) + if beaconState != nil && beaconState.Slot > db.highestBlockSlot { + db.highestBlockSlot = beaconState.Slot + } + return err + }) + + // return error if it's an invalid validator index. + if index >= uint64(len(db.validatorRegistry)) { + return nil, fmt.Errorf("invalid validator index %d", index) + } + + return beaconState.ValidatorRegistry[index], err +} + // ValidatorBalances fetches the current validator balances stored in state. func (db *BeaconDB) ValidatorBalances(ctx context.Context) ([]uint64, error) { - ctx, span := trace.StartSpan(ctx, "BeaconDB.ValidatorRegistry") + ctx, span := trace.StartSpan(ctx, "BeaconDB.ValidatorBalances") defer span.End() db.stateLock.RLock() defer db.stateLock.RUnlock() // Return in-memory cached state, if available. - if db.currentState != nil { - _, span := trace.StartSpan(ctx, "proto.Clone.ValidatorRegistry") + if db.validatorBalances != nil { + _, span := trace.StartSpan(ctx, "BeaconDB.Copy.Balances") defer span.End() - newBalances := make([]uint64, len(db.currentState.ValidatorBalances)) - copy(newBalances, db.currentState.ValidatorBalances) + newBalances := make([]uint64, len(db.validatorBalances)) + copy(newBalances, db.validatorBalances) return newBalances, nil } diff --git a/beacon-chain/db/state_test.go b/beacon-chain/db/state_test.go index 5ac1f598ed34..91099117d5d6 100644 --- a/beacon-chain/db/state_test.go +++ b/beacon-chain/db/state_test.go @@ -138,7 +138,10 @@ func BenchmarkState_ReadingFromCache(b *testing.B) { b.Fatalf("Could not save beacon state to cache from DB: %v", err) } - if db.currentState.Slot != params.BeaconConfig().GenesisSlot+1 { + savedState := &pb.BeaconState{} + savedState.Unmarshal(db.serializedState) + + if savedState.Slot != params.BeaconConfig().GenesisSlot+1 { b.Fatal("cache should be prepared on state after saving to DB") } From 54f6fffb5849dd3c17ad9279c3d7286a0f520b1a Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Wed, 24 Apr 2019 22:28:52 +0800 Subject: [PATCH 15/54] Batch Attestations in Attestation Service (#2370) * fix state reads * add method to retrieve validator from state * lint * batch update attestation * handle nil case * add test * lint * batch update attestation --- beacon-chain/attestation/service.go | 67 ++++++++++++++++++--- beacon-chain/attestation/service_test.go | 44 ++++++++++++++ beacon-chain/blockchain/block_processing.go | 7 +-- 3 files changed, 104 insertions(+), 14 deletions(-) diff --git a/beacon-chain/attestation/service.go b/beacon-chain/attestation/service.go index 5b94a2d1b3f8..2e66b83f0500 100644 --- a/beacon-chain/attestation/service.go +++ b/beacon-chain/attestation/service.go @@ -4,6 +4,7 @@ package attestation import ( "context" "fmt" + "sort" "sync" "github.com/gogo/protobuf/proto" @@ -150,8 +151,8 @@ func (a *Service) attestationPool() { log.Debug("Attestation pool closed, exiting goroutine") return // Listen for a newly received incoming attestation from the sync service. - case attestation := <-a.incomingChan: - handler.SafelyHandleMessage(a.ctx, a.handleAttestation, attestation) + case attestations := <-a.incomingChan: + handler.SafelyHandleMessage(a.ctx, a.handleAttestation, attestations) } } } @@ -185,10 +186,58 @@ func (a *Service) UpdateLatestAttestation(ctx context.Context, attestation *pb.A if err != nil { return err } + return a.updateAttestation(ctx, headRoot, beaconState, attestation) +} + +// BatchUpdateLatestAttestation updates multiple attestations and adds them into the attestation store +// if they are valid. +func (a *Service) BatchUpdateLatestAttestation(ctx context.Context, attestations []*pb.Attestation) error { + + if attestations == nil { + return nil + } + // Potential improvement, instead of getting the state, + // we could get a mapping of validator index to public key. + beaconState, err := a.beaconDB.HeadState(ctx) + if err != nil { + return err + } + head, err := a.beaconDB.ChainHead() + if err != nil { + return err + } + headRoot, err := hashutil.HashBeaconBlock(head) + if err != nil { + return err + } + + attestations = a.sortAttestations(attestations) + + for _, attestation := range attestations { + if err := a.updateAttestation(ctx, headRoot, beaconState, attestation); err != nil { + return err + } + } + return nil +} + +// InsertAttestationIntoStore locks the store, inserts the attestation, then +// unlocks the store again. This method may be used by external services +// in testing to populate the attestation store. +func (a *Service) InsertAttestationIntoStore(pubkey [48]byte, att *pb.Attestation) { + a.store.Lock() + defer a.store.Unlock() + a.store.m[pubkey] = att +} + +func (a *Service) updateAttestation(ctx context.Context, headRoot [32]byte, beaconState *pb.BeaconState, + attestation *pb.Attestation) error { + totalAttestationSeen.Inc() slot := attestation.Data.Slot var committee []uint64 var cachedCommittees *cache.CommitteesInSlot + var err error for beaconState.Slot < slot { beaconState, err = state.ExecuteStateTransition( @@ -279,11 +328,11 @@ func (a *Service) UpdateLatestAttestation(ctx context.Context, attestation *pb.A return nil } -// InsertAttestationIntoStore locks the store, inserts the attestation, then -// unlocks the store again. This method may be used by external services -// in testing to populate the attestation store. -func (a *Service) InsertAttestationIntoStore(pubkey [48]byte, att *pb.Attestation) { - a.store.Lock() - defer a.store.Unlock() - a.store.m[pubkey] = att +// sortAttestations sorts attestations by their slot number in ascending order. +func (a *Service) sortAttestations(attestations []*pb.Attestation) []*pb.Attestation { + sort.SliceStable(attestations, func(i, j int) bool { + return attestations[i].Data.Slot < attestations[j].Data.Slot + }) + + return attestations } diff --git a/beacon-chain/attestation/service_test.go b/beacon-chain/attestation/service_test.go index f42aca9c8159..165a9c90c812 100644 --- a/beacon-chain/attestation/service_test.go +++ b/beacon-chain/attestation/service_test.go @@ -403,3 +403,47 @@ func TestUpdateLatestAttestation_CacheEnabledAndHit(t *testing.T) { attestation.Data.Slot, service.store.m[pubkey].Data.Slot) } } + +func TestUpdateLatestAttestation_BatchUpdate(t *testing.T) { + beaconDB := internal.SetupDB(t) + defer internal.TeardownDB(t, beaconDB) + ctx := context.Background() + + var validators []*pb.Validator + for i := 0; i < 64; i++ { + validators = append(validators, &pb.Validator{ + Pubkey: []byte{byte(i)}, + ActivationEpoch: params.BeaconConfig().GenesisEpoch, + ExitEpoch: params.BeaconConfig().GenesisEpoch + 10, + }) + } + + beaconState := &pb.BeaconState{ + Slot: params.BeaconConfig().GenesisSlot + 1, + ValidatorRegistry: validators, + } + block := &pb.BeaconBlock{ + Slot: params.BeaconConfig().GenesisSlot + 1, + } + if err := beaconDB.SaveBlock(block); err != nil { + t.Fatal(err) + } + if err := beaconDB.UpdateChainHead(ctx, block, beaconState); err != nil { + t.Fatal(err) + } + service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB}) + attestations := make([]*pb.Attestation, 0) + for i := 0; i < 10; i++ { + attestations = append(attestations, &pb.Attestation{ + AggregationBitfield: []byte{0x80}, + Data: &pb.AttestationData{ + Slot: params.BeaconConfig().GenesisSlot + 1, + Shard: 1, + }, + }) + } + + if err := service.BatchUpdateLatestAttestation(ctx, attestations); err != nil { + t.Fatalf("could not update latest attestation: %v", err) + } +} diff --git a/beacon-chain/blockchain/block_processing.go b/beacon-chain/blockchain/block_processing.go index 29f4d5a5887c..456d4d7f903f 100644 --- a/beacon-chain/blockchain/block_processing.go +++ b/beacon-chain/blockchain/block_processing.go @@ -233,11 +233,8 @@ func (c *ChainService) CleanupBlockOperations(ctx context.Context, block *pb.Bea log.Error("Sent processed block to no subscribers") } - // Update attestation store with latest attestation target. - for _, att := range block.Body.Attestations { - if err := c.attsService.UpdateLatestAttestation(ctx, att); err != nil { - return fmt.Errorf("failed to update latest attestation for store: %v", err) - } + if err := c.attsService.BatchUpdateLatestAttestation(ctx, block.Body.Attestations); err != nil { + return fmt.Errorf("failed to update latest attestation for store: %v", err) } // Remove pending deposits from the deposit queue. From 941810ee7b16a88dfcaa64c8b87c1b6d9fd92435 Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Wed, 24 Apr 2019 23:46:06 +0800 Subject: [PATCH 16/54] Use State Instead of Cached Registry (#2372) * fix state reads * add method to retrieve validator from state * lint * batch update attestation * handle nil case * add test * lint * batch update attestation * use state instead * fix test --- beacon-chain/attestation/service.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beacon-chain/attestation/service.go b/beacon-chain/attestation/service.go index 2e66b83f0500..fa32a81fbdc8 100644 --- a/beacon-chain/attestation/service.go +++ b/beacon-chain/attestation/service.go @@ -95,12 +95,17 @@ func (a *Service) IncomingAttestationFeed() *event.Feed { // Attestation` be the attestation with the highest slot number in `store` // from the validator with the given `validator_index` func (a *Service) LatestAttestation(ctx context.Context, index uint64) (*pb.Attestation, error) { - validator, err := a.beaconDB.ValidatorFromState(ctx, index) + bState, err := a.beaconDB.HeadState(ctx) if err != nil { return nil, err } - pubKey := bytesutil.ToBytes48(validator.Pubkey) + // return error if it's an invalid validator index. + if index >= uint64(len(bState.ValidatorRegistry)) { + return nil, fmt.Errorf("invalid validator index %d", index) + } + + pubKey := bytesutil.ToBytes48(bState.ValidatorRegistry[index].Pubkey) a.store.RLock() defer a.store.RUnlock() if _, exists := a.store.m[pubKey]; !exists { From b81eeec3747222d843e0ea1d4e9a00aa18bf7b9d Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 24 Apr 2019 10:21:00 -0700 Subject: [PATCH 17/54] Optimize block fetch with attestation target fetch (#2373) * 1st attemp to optimize blk fetch w/ att tgt fetch * test test test test --- beacon-chain/attestation/service.go | 13 ++--- beacon-chain/attestation/service_test.go | 8 +++ beacon-chain/blockchain/block_processing.go | 7 +++ beacon-chain/blockchain/fork_choice_test.go | 7 +++ beacon-chain/blockchain/service.go | 12 +++++ beacon-chain/db/attestation.go | 50 +++++++++++++++++-- beacon-chain/db/db.go | 4 +- beacon-chain/db/schema.go | 15 +++--- beacon-chain/sync/initial-sync/service.go | 13 +++++ beacon-chain/sync/initial-sync/sync_blocks.go | 7 +++ beacon-chain/sync/initial-sync/sync_state.go | 16 ++++++ shared/ssz/hash_cache.go | 2 +- 12 files changed, 131 insertions(+), 23 deletions(-) diff --git a/beacon-chain/attestation/service.go b/beacon-chain/attestation/service.go index fa32a81fbdc8..6994b39465d3 100644 --- a/beacon-chain/attestation/service.go +++ b/beacon-chain/attestation/service.go @@ -131,18 +131,11 @@ func (a *Service) LatestAttestationTarget(ctx context.Context, index uint64) (*p return nil, nil } targetRoot := bytesutil.ToBytes32(attestation.Data.BeaconBlockRootHash32) - targetBlock, err := a.beaconDB.Block(targetRoot) - if err != nil { - return nil, fmt.Errorf("could not get target block: %v", err) - } - if targetBlock == nil { + if !a.beaconDB.HasBlock(targetRoot) { return nil, nil } - return &pb.AttestationTarget{ - Slot: targetBlock.Slot, - BlockRoot: targetRoot[:], - ParentRoot: targetBlock.ParentRootHash32, - }, nil + + return a.beaconDB.AttestationTarget(targetRoot) } // attestationPool takes an newly received attestation from sync service diff --git a/beacon-chain/attestation/service_test.go b/beacon-chain/attestation/service_test.go index 165a9c90c812..53c53f6caa4b 100644 --- a/beacon-chain/attestation/service_test.go +++ b/beacon-chain/attestation/service_test.go @@ -235,6 +235,13 @@ func TestLatestAttestationTarget_ReturnsLatestAttestedBlock(t *testing.T) { if err != nil { log.Fatalf("could not hash block: %v", err) } + if err := beaconDB.SaveAttestationTarget(ctx, &pb.AttestationTarget{ + Slot: block.Slot, + BlockRoot: blockRoot[:], + ParentRoot: []byte{}, + }); err != nil { + log.Fatalf("could not save att target: %v", err) + } service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB}) @@ -249,6 +256,7 @@ func TestLatestAttestationTarget_ReturnsLatestAttestedBlock(t *testing.T) { if err != nil { t.Fatalf("Could not get latest attestation: %v", err) } + if !bytes.Equal(blockRoot[:], latestAttestedTarget.BlockRoot) { t.Errorf("Wanted: %v, got: %v", blockRoot[:], latestAttestedTarget.BlockRoot) } diff --git a/beacon-chain/blockchain/block_processing.go b/beacon-chain/blockchain/block_processing.go index 456d4d7f903f..c1245d3705bd 100644 --- a/beacon-chain/blockchain/block_processing.go +++ b/beacon-chain/blockchain/block_processing.go @@ -215,6 +215,13 @@ func (c *ChainService) SaveAndBroadcastBlock(ctx context.Context, block *pb.Beac if err := c.beaconDB.SaveBlock(block); err != nil { return fmt.Errorf("failed to save block: %v", err) } + if err := c.beaconDB.SaveAttestationTarget(ctx, &pb.AttestationTarget{ + Slot: block.Slot, + BlockRoot: blockRoot[:], + ParentRoot: block.ParentRootHash32, + }); err != nil { + return fmt.Errorf("failed to save attestation target: %v", err) + } // Announce the new block to the network. c.p2p.Broadcast(ctx, &pb.BeaconBlockAnnounce{ Hash: blockRoot[:], diff --git a/beacon-chain/blockchain/fork_choice_test.go b/beacon-chain/blockchain/fork_choice_test.go index 2878062294d7..3eca3c01ff24 100644 --- a/beacon-chain/blockchain/fork_choice_test.go +++ b/beacon-chain/blockchain/fork_choice_test.go @@ -240,6 +240,13 @@ func TestAttestationTargets_RetrieveWorks(t *testing.T) { if err != nil { log.Fatalf("could not hash block: %v", err) } + if err := beaconDB.SaveAttestationTarget(ctx, &pb.AttestationTarget{ + Slot: block.Slot, + BlockRoot: blockRoot[:], + ParentRoot: []byte{}, + }); err != nil { + log.Fatalf("could not save att tgt: %v", err) + } attsService := attestation.NewAttestationService( context.Background(), diff --git a/beacon-chain/blockchain/service.go b/beacon-chain/blockchain/service.go index 4cab666ebcae..c37d9fe18eae 100644 --- a/beacon-chain/blockchain/service.go +++ b/beacon-chain/blockchain/service.go @@ -147,12 +147,24 @@ func (c *ChainService) initializeBeaconChain(genesisTime time.Time, deposits []* return nil, fmt.Errorf("could not hash beacon state: %v", err) } genBlock := b.NewGenesisBlock(stateRoot[:]) + genBlockRoot, err := hashutil.HashBeaconBlock(genBlock) + if err != nil { + return nil, fmt.Errorf("could not hash beacon block: %v", err) + } + // TODO(#2011): Remove this in state caching. beaconState.LatestBlock = genBlock if err := c.beaconDB.SaveBlock(genBlock); err != nil { return nil, fmt.Errorf("could not save genesis block to disk: %v", err) } + if err := c.beaconDB.SaveAttestationTarget(ctx, &pb.AttestationTarget{ + Slot: genBlock.Slot, + BlockRoot: genBlockRoot[:], + ParentRoot: genBlock.ParentRootHash32, + }); err != nil { + return nil, fmt.Errorf("failed to save attestation target: %v", err) + } if err := c.beaconDB.UpdateChainHead(ctx, genBlock, beaconState); err != nil { return nil, fmt.Errorf("could not set chain head, %v", err) } diff --git a/beacon-chain/db/attestation.go b/beacon-chain/db/attestation.go index 7974a086c696..a63dc4388982 100644 --- a/beacon-chain/db/attestation.go +++ b/beacon-chain/db/attestation.go @@ -16,16 +16,33 @@ func (db *BeaconDB) SaveAttestation(ctx context.Context, attestation *pb.Attesta ctx, span := trace.StartSpan(ctx, "beaconDB.SaveAttestation") defer span.End() - encodedState, err := proto.Marshal(attestation) + encodedAtt, err := proto.Marshal(attestation) if err != nil { return err } - hash := hashutil.Hash(encodedState) + hash := hashutil.Hash(encodedAtt) return db.update(func(tx *bolt.Tx) error { a := tx.Bucket(attestationBucket) - return a.Put(hash[:], encodedState) + return a.Put(hash[:], encodedAtt) + }) +} + +// SaveAttestationTarget puts the attestation target record into the beacon chain db. +func (db *BeaconDB) SaveAttestationTarget(ctx context.Context, attTarget *pb.AttestationTarget) error { + ctx, span := trace.StartSpan(ctx, "beaconDB.SaveAttestationTarget") + defer span.End() + + encodedAttTgt, err := proto.Marshal(attTarget) + if err != nil { + return err + } + + return db.update(func(tx *bolt.Tx) error { + a := tx.Bucket(attestationTargetBucket) + + return a.Put(attTarget.BlockRoot, encodedAttTgt) }) } @@ -85,6 +102,25 @@ func (db *BeaconDB) Attestations() ([]*pb.Attestation, error) { return attestations, err } +// AttestationTarget retrieves an attestation target record from the db using its hash. +func (db *BeaconDB) AttestationTarget(hash [32]byte) (*pb.AttestationTarget, error) { + var attTgt *pb.AttestationTarget + err := db.view(func(tx *bolt.Tx) error { + a := tx.Bucket(attestationTargetBucket) + + enc := a.Get(hash[:]) + if enc == nil { + return nil + } + + var err error + attTgt, err = createAttestationTarget(enc) + return err + }) + + return attTgt, err +} + // HasAttestation checks if the attestation exists. func (db *BeaconDB) HasAttestation(hash [32]byte) bool { exists := false @@ -105,3 +141,11 @@ func createAttestation(enc []byte) (*pb.Attestation, error) { } return protoAttestation, nil } + +func createAttestationTarget(enc []byte) (*pb.AttestationTarget, error) { + protoAttTgt := &pb.AttestationTarget{} + if err := proto.Unmarshal(enc, protoAttTgt); err != nil { + return nil, fmt.Errorf("failed to unmarshal encoding: %v", err) + } + return protoAttTgt, nil +} diff --git a/beacon-chain/db/db.go b/beacon-chain/db/db.go index 7b998576dfac..101d8da3d876 100644 --- a/beacon-chain/db/db.go +++ b/beacon-chain/db/db.go @@ -85,8 +85,8 @@ func NewDB(dirPath string) (*BeaconDB, error) { db := &BeaconDB{db: boltDB, DatabasePath: dirPath} if err := db.update(func(tx *bolt.Tx) error { - return createBuckets(tx, blockBucket, attestationBucket, mainChainBucket, histStateBucket, - chainInfoBucket, cleanupHistoryBucket, blockOperationsBucket, validatorBucket) + return createBuckets(tx, blockBucket, attestationBucket, attestationTargetBucket, mainChainBucket, + histStateBucket, chainInfoBucket, cleanupHistoryBucket, blockOperationsBucket, validatorBucket) }); err != nil { return nil, err } diff --git a/beacon-chain/db/schema.go b/beacon-chain/db/schema.go index fcdb8d07e2f1..33d0790934e7 100644 --- a/beacon-chain/db/schema.go +++ b/beacon-chain/db/schema.go @@ -15,13 +15,14 @@ import ( // The fields below define the suffix of keys in the db. var ( - attestationBucket = []byte("attestation-bucket") - blockOperationsBucket = []byte("block-operations-bucket") - blockBucket = []byte("block-bucket") - mainChainBucket = []byte("main-chain-bucket") - histStateBucket = []byte("historical-state-bucket") - chainInfoBucket = []byte("chain-info") - validatorBucket = []byte("validator") + attestationBucket = []byte("attestation-bucket") + attestationTargetBucket = []byte("attestation-target-bucket") + blockOperationsBucket = []byte("block-operations-bucket") + blockBucket = []byte("block-bucket") + mainChainBucket = []byte("main-chain-bucket") + histStateBucket = []byte("historical-state-bucket") + chainInfoBucket = []byte("chain-info") + validatorBucket = []byte("validator") mainChainHeightKey = []byte("chain-height") stateLookupKey = []byte("state") diff --git a/beacon-chain/sync/initial-sync/service.go b/beacon-chain/sync/initial-sync/service.go index 1989476b8552..a528cc6e1e53 100644 --- a/beacon-chain/sync/initial-sync/service.go +++ b/beacon-chain/sync/initial-sync/service.go @@ -17,6 +17,8 @@ import ( "sync" "time" + "github.com/prysmaticlabs/prysm/shared/hashutil" + "github.com/ethereum/go-ethereum/common" "github.com/gogo/protobuf/proto" "github.com/prysmaticlabs/prysm/beacon-chain/blockchain" @@ -203,6 +205,17 @@ func (s *InitialSync) exitInitialSync(ctx context.Context, block *pb.BeaconBlock if err := s.db.SaveBlock(block); err != nil { return err } + root, err := hashutil.HashBeaconBlock(block) + if err != nil { + return fmt.Errorf("failed to tree hash block: %v", err) + } + if err := s.db.SaveAttestationTarget(ctx, &pb.AttestationTarget{ + Slot: block.Slot, + BlockRoot: root[:], + ParentRoot: block.ParentRootHash32, + }); err != nil { + return fmt.Errorf("failed to save attestation target: %v", err) + } state, err = s.chainService.ApplyBlockStateTransition(ctx, block, state) if err != nil { switch err.(type) { diff --git a/beacon-chain/sync/initial-sync/sync_blocks.go b/beacon-chain/sync/initial-sync/sync_blocks.go index aa567a0aba8d..84b5eeb97c6d 100644 --- a/beacon-chain/sync/initial-sync/sync_blocks.go +++ b/beacon-chain/sync/initial-sync/sync_blocks.go @@ -165,6 +165,13 @@ func (s *InitialSync) validateAndSaveNextBlock(ctx context.Context, block *pb.Be if err := s.db.SaveBlock(block); err != nil { return err } + if err := s.db.SaveAttestationTarget(ctx, &pb.AttestationTarget{ + Slot: block.Slot, + BlockRoot: root[:], + ParentRoot: block.ParentRootHash32, + }); err != nil { + return fmt.Errorf("could not to save attestation target: %v", err) + } state, err = s.chainService.ApplyBlockStateTransition(ctx, block, state) if err != nil { return fmt.Errorf("could not apply block state transition: %v", err) diff --git a/beacon-chain/sync/initial-sync/sync_state.go b/beacon-chain/sync/initial-sync/sync_state.go index 1aa0fa994c81..9e82102134fa 100644 --- a/beacon-chain/sync/initial-sync/sync_state.go +++ b/beacon-chain/sync/initial-sync/sync_state.go @@ -3,6 +3,8 @@ package initialsync import ( "context" + "github.com/prysmaticlabs/prysm/shared/hashutil" + "github.com/prysmaticlabs/prysm/beacon-chain/core/validators" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/bytesutil" @@ -38,6 +40,20 @@ func (s *InitialSync) processState(msg p2p.Message) { return } + root, err := hashutil.HashBeaconBlock(finalizedState.LatestBlock) + if err != nil { + log.Errorf("Could not hash finalized block %v", err) + return + } + if err := s.db.SaveAttestationTarget(ctx, &pb.AttestationTarget{ + Slot: finalizedState.LatestBlock.Slot, + BlockRoot: root[:], + ParentRoot: finalizedState.LatestBlock.ParentRootHash32, + }); err != nil { + log.Errorf("Could not to save attestation target: %v", err) + return + } + if err := s.db.SaveJustifiedState(finalizedState); err != nil { log.Errorf("Could not set beacon state for initial sync %v", err) return diff --git a/shared/ssz/hash_cache.go b/shared/ssz/hash_cache.go index 22320054a71d..75c54a8c13c4 100644 --- a/shared/ssz/hash_cache.go +++ b/shared/ssz/hash_cache.go @@ -84,7 +84,7 @@ func (b *hashCacheS) TrieRootCached(val interface{}) ([32]byte, error) { rval := reflect.ValueOf(val) hs, err := hashedEncoding(rval) if err != nil { - return [32]byte{}, newHashError(error.Error(err), rval.Type()) + return [32]byte{}, newHashError(fmt.Sprint(err), rval.Type()) } exists, fetchedInfo, err := b.RootByEncodedHash(bytesutil.ToBytes32(hs)) if err != nil { From 66f3964c1b06a6e62e2425e92c5b9f8500d9d193 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Wed, 24 Apr 2019 16:57:06 -0400 Subject: [PATCH 18/54] allow CORS from anywhere (#2375) --- k8s/beacon-chain/contract-address.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/k8s/beacon-chain/contract-address.yaml b/k8s/beacon-chain/contract-address.yaml index 55d98427fa66..fc5a6d2c15d7 100644 --- a/k8s/beacon-chain/contract-address.yaml +++ b/k8s/beacon-chain/contract-address.yaml @@ -72,3 +72,9 @@ spec: port: number: 8080 host: contract-address-http.beacon-chain.svc.cluster.local + corsPolicy: + allowOrigin: + - '*' + allowMethods: + - GET + maxAge: '24h' From 8abd2e51bdf3006ab52589ffdc40da1f33d3d36e Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Thu, 25 Apr 2019 07:59:12 -0400 Subject: [PATCH 19/54] Update flags.go (#2376) --- shared/cmd/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/cmd/flags.go b/shared/cmd/flags.go index 25aab174c275..56e7837c33b9 100644 --- a/shared/cmd/flags.go +++ b/shared/cmd/flags.go @@ -64,7 +64,7 @@ var ( Name: "relay-node", Usage: "The address of relay node. The beacon node will connect to the " + "relay node and advertise their address via the relay node to other peers", - Value: "/ip4/35.224.249.2/tcp/30000/p2p/QmfAgkmjiZNZhr2wFN9TwaRgHouMTBT6HELyzE5A3BT2wK", + Value: "", } // P2PPort defines the port to be used by libp2p. P2PPort = cli.IntFlag{ From 75be7be64605468517826488b7f3ec726121cf17 Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Thu, 25 Apr 2019 21:11:07 +0800 Subject: [PATCH 20/54] Handle Panics In Attestation and Disable Logging (#2380) * Revert "Use State Instead of Cached Registry (#2372)" This reverts commit 941810ee7b16a88dfcaa64c8b87c1b6d9fd92435. * handle panic * fix logging * add test * gazelle --- beacon-chain/attestation/BUILD.bazel | 2 ++ beacon-chain/attestation/service.go | 18 ++++++---- beacon-chain/attestation/service_test.go | 45 ++++++++++++++++++++++++ beacon-chain/core/state/transition.go | 4 ++- 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/beacon-chain/attestation/BUILD.bazel b/beacon-chain/attestation/BUILD.bazel index 75e4df87fc28..e0e370dd9cef 100644 --- a/beacon-chain/attestation/BUILD.bazel +++ b/beacon-chain/attestation/BUILD.bazel @@ -38,6 +38,8 @@ go_test( "//shared/bytesutil:go_default_library", "//shared/hashutil:go_default_library", "//shared/params:go_default_library", + "//shared/testutil:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", + "@com_github_sirupsen_logrus//hooks/test:go_default_library", ], ) diff --git a/beacon-chain/attestation/service.go b/beacon-chain/attestation/service.go index 6994b39465d3..9bff9e51ec0c 100644 --- a/beacon-chain/attestation/service.go +++ b/beacon-chain/attestation/service.go @@ -95,17 +95,12 @@ func (a *Service) IncomingAttestationFeed() *event.Feed { // Attestation` be the attestation with the highest slot number in `store` // from the validator with the given `validator_index` func (a *Service) LatestAttestation(ctx context.Context, index uint64) (*pb.Attestation, error) { - bState, err := a.beaconDB.HeadState(ctx) + validator, err := a.beaconDB.ValidatorFromState(ctx, index) if err != nil { return nil, err } - // return error if it's an invalid validator index. - if index >= uint64(len(bState.ValidatorRegistry)) { - return nil, fmt.Errorf("invalid validator index %d", index) - } - - pubKey := bytesutil.ToBytes48(bState.ValidatorRegistry[index].Pubkey) + pubKey := bytesutil.ToBytes48(validator.Pubkey) a.store.RLock() defer a.store.RUnlock() if _, exists := a.store.m[pubKey]; !exists { @@ -294,6 +289,15 @@ func (a *Service) updateAttestation(ctx context.Context, headRoot [32]byte, beac continue } + if i >= len(committee) { + log.Errorf("Bitfield points to an invalid index in the committee: bitfield %08b", bitfield) + continue + } + + if int(committee[i]) >= len(beaconState.ValidatorRegistry) { + log.Errorf("Index doesn't exist in validator registry: index %d", committee[i]) + } + // If the attestation came from this attester. We use the slot committee to find the // validator's actual index. pubkey := bytesutil.ToBytes48(beaconState.ValidatorRegistry[committee[i]].Pubkey) diff --git a/beacon-chain/attestation/service_test.go b/beacon-chain/attestation/service_test.go index 53c53f6caa4b..a376e4bedbeb 100644 --- a/beacon-chain/attestation/service_test.go +++ b/beacon-chain/attestation/service_test.go @@ -14,7 +14,9 @@ import ( "github.com/prysmaticlabs/prysm/shared/bytesutil" "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/params" + "github.com/prysmaticlabs/prysm/shared/testutil" "github.com/sirupsen/logrus" + logTest "github.com/sirupsen/logrus/hooks/test" ) func init() { @@ -412,6 +414,49 @@ func TestUpdateLatestAttestation_CacheEnabledAndHit(t *testing.T) { } } +func TestUpdateLatestAttestation_InvalidIndex(t *testing.T) { + beaconDB := internal.SetupDB(t) + hook := logTest.NewGlobal() + defer internal.TeardownDB(t, beaconDB) + ctx := context.Background() + + var validators []*pb.Validator + for i := 0; i < 64; i++ { + validators = append(validators, &pb.Validator{ + Pubkey: []byte{byte(i)}, + ActivationEpoch: params.BeaconConfig().GenesisEpoch, + ExitEpoch: params.BeaconConfig().GenesisEpoch + 10, + }) + } + + beaconState := &pb.BeaconState{ + Slot: params.BeaconConfig().GenesisSlot + 1, + ValidatorRegistry: validators, + } + block := &pb.BeaconBlock{ + Slot: params.BeaconConfig().GenesisSlot + 1, + } + if err := beaconDB.SaveBlock(block); err != nil { + t.Fatal(err) + } + if err := beaconDB.UpdateChainHead(ctx, block, beaconState); err != nil { + t.Fatal(err) + } + service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB}) + attestation := &pb.Attestation{ + AggregationBitfield: []byte{0xC0}, + Data: &pb.AttestationData{ + Slot: params.BeaconConfig().GenesisSlot + 1, + Shard: 1, + }, + } + + if err := service.UpdateLatestAttestation(ctx, attestation); err != nil { + t.Fatalf("could not update latest attestation: %v", err) + } + testutil.AssertLogsContain(t, hook, "Bitfield points to an invalid index in the committee") +} + func TestUpdateLatestAttestation_BatchUpdate(t *testing.T) { beaconDB := internal.SetupDB(t) defer internal.TeardownDB(t, beaconDB) diff --git a/beacon-chain/core/state/transition.go b/beacon-chain/core/state/transition.go index 523d170d38d1..18145bc3af78 100644 --- a/beacon-chain/core/state/transition.go +++ b/beacon-chain/core/state/transition.go @@ -376,7 +376,9 @@ func ProcessEpoch(ctx context.Context, state *pb.BeaconState, block *pb.BeaconBl } case epochsSinceFinality > 4: - log.WithField("epochSinceFinality", epochsSinceFinality).Info("Applying quadratic leak penalties") + if config.Logging { + log.WithField("epochSinceFinality", epochsSinceFinality).Info("Applying quadratic leak penalties") + } // Apply penalties for long inactive FFG source participants. state = bal.InactivityFFGSource( state, From a84fd21cd7171eb8304ea6966d6f73b77d8c3d02 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 25 Apr 2019 08:10:58 -0700 Subject: [PATCH 21/54] Minor Cosmetic Updates to Validator Logging (#2377) * update validator logs * ops, revert mid point * do it before the wait * delta --- validator/client/validator.go | 2 +- validator/client/validator_attest.go | 2 +- validator/client/validator_metrics.go | 11 +++++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/validator/client/validator.go b/validator/client/validator.go index 59cd9f62855f..54de53308a74 100644 --- a/validator/client/validator.go +++ b/validator/client/validator.go @@ -67,7 +67,7 @@ func (v *validator) WaitForChainStart(ctx context.Context) error { // Once the ChainStart log is received, we update the genesis time of the validator client // and begin a slot ticker used to track the current slot the beacon node is in. v.ticker = slotutil.GetSlotTicker(time.Unix(int64(v.genesisTime), 0), params.BeaconConfig().SecondsPerSlot) - log.Infof("Beacon chain initialized at unix time: %v", time.Unix(int64(v.genesisTime), 0)) + log.WithField("genesisTime", time.Unix(int64(v.genesisTime), 0)).Info("Beacon chain initialized") return nil } diff --git a/validator/client/validator_attest.go b/validator/client/validator_attest.go index a3b30d896015..f1553e2e299f 100644 --- a/validator/client/validator_attest.go +++ b/validator/client/validator_attest.go @@ -31,7 +31,7 @@ func (v *validator) AttestToBlockHead(ctx context.Context, slot uint64, idx stri if len(idx) > 12 { truncatedPk = idx[:12] } - log.Infof("%v Performing a beacon block attestation...", truncatedPk) + log.WithField("validator", truncatedPk).Info("Performing a beacon block attestation...") v.waitToSlotMidpoint(ctx, slot) // First the validator should construct attestation_data, an AttestationData diff --git a/validator/client/validator_metrics.go b/validator/client/validator_metrics.go index b6e1ef606af7..a88e1426816c 100644 --- a/validator/client/validator_metrics.go +++ b/validator/client/validator_metrics.go @@ -56,18 +56,17 @@ func (v *validator) LogValidatorGainsAndLosses(ctx context.Context, slot uint64) reported = true } newBalance := float64(resp.Balance) / float64(params.BeaconConfig().GweiPerEth) - log.WithFields(logrus.Fields{ - "ethBalance": newBalance, - }).Infof("%v New validator balance", tpk) if v.prevBalance > 0 { prevBalance := float64(v.prevBalance) / float64(params.BeaconConfig().GweiPerEth) percentNet := (newBalance - prevBalance) / prevBalance - log.WithField("prevEthBalance", prevBalance).Infof("%v Previous validator balance", tpk) log.WithFields(logrus.Fields{ - "eth": fmt.Sprintf("%f", newBalance-prevBalance), + "prevBalance": prevBalance, + "newBalance": newBalance, + "delta": fmt.Sprintf("%f", newBalance-prevBalance), "percentChange": fmt.Sprintf("%.2f%%", percentNet*100), - }).Infof("%v Net gains/losses in eth", tpk) + "pubKey": tpk, + }).Infof("Net gains/losses in eth") } totalPrevBalance += resp.Balance } From 706aa9b82fcfce3a7d2661b3ef0b13963b5162cc Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Thu, 25 Apr 2019 10:59:18 -0500 Subject: [PATCH 22/54] Update Jaeger Export Tracing (#2382) * added exporter * fixed jaeger export * lint * imports --- WORKSPACE | 8 ++++++++ shared/tracing/BUILD.bazel | 2 +- shared/tracing/tracer.go | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index bcf5834843c0..8bce741d1f43 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -632,6 +632,14 @@ go_repository( importpath = "go.opencensus.io", ) +go_repository( + name = "io_opencensus_go_contrib_exporter_jaeger", + commit = "5b8293c22f362562285c2acbc52f4a1870a47a33", + importpath = "contrib.go.opencensus.io/exporter/jaeger", + remote = "http://github.com/census-ecosystem/opencensus-go-exporter-jaeger", + vcs = "git", +) + go_repository( name = "org_golang_google_api", commit = "0cbcb99a9ea0c8023c794b2693cbe1def82ed4d7", # v0.3.2 diff --git a/shared/tracing/BUILD.bazel b/shared/tracing/BUILD.bazel index 858b22d85e85..567f0b9a356f 100644 --- a/shared/tracing/BUILD.bazel +++ b/shared/tracing/BUILD.bazel @@ -7,7 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "@com_github_sirupsen_logrus//:go_default_library", - "@io_opencensus_go//exporter/jaeger:go_default_library", "@io_opencensus_go//trace:go_default_library", + "@io_opencensus_go_contrib_exporter_jaeger//:go_default_library", ], ) diff --git a/shared/tracing/tracer.go b/shared/tracing/tracer.go index 8780be138c08..524bea819eb5 100644 --- a/shared/tracing/tracer.go +++ b/shared/tracing/tracer.go @@ -3,8 +3,8 @@ package tracing import ( "errors" + "contrib.go.opencensus.io/exporter/jaeger" "github.com/sirupsen/logrus" - "go.opencensus.io/exporter/jaeger" "go.opencensus.io/trace" ) From 538d7ec07cc909d2b698f8db809592446dec6716 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Thu, 25 Apr 2019 11:44:35 -0500 Subject: [PATCH 23/54] Prevent Fetching Head State Many Times in Requesting Latest Attestation (#2374) --- beacon-chain/attestation/service.go | 34 +++------ beacon-chain/attestation/service_test.go | 80 +++------------------ beacon-chain/blockchain/fork_choice.go | 6 +- beacon-chain/blockchain/fork_choice_test.go | 2 +- 4 files changed, 25 insertions(+), 97 deletions(-) diff --git a/beacon-chain/attestation/service.go b/beacon-chain/attestation/service.go index 9bff9e51ec0c..f7d003a7bc60 100644 --- a/beacon-chain/attestation/service.go +++ b/beacon-chain/attestation/service.go @@ -87,18 +87,18 @@ func (a *Service) IncomingAttestationFeed() *event.Feed { return a.incomingFeed } -// LatestAttestation returns the latest attestation from validator index, the highest -// slotNumber attestation from the attestation pool gets returned. +// LatestAttestationTarget returns the target block that the validator index attested to, +// the highest slotNumber attestation in attestation pool gets returned. // // Spec pseudocode definition: -// Let `get_latest_attestation(store: Store, validator_index: ValidatorIndex) -> -// Attestation` be the attestation with the highest slot number in `store` -// from the validator with the given `validator_index` -func (a *Service) LatestAttestation(ctx context.Context, index uint64) (*pb.Attestation, error) { - validator, err := a.beaconDB.ValidatorFromState(ctx, index) - if err != nil { - return nil, err +// Let `get_latest_attestation_target(store: Store, validator_index: ValidatorIndex) -> +// BeaconBlock` be the target block in the attestation +// `get_latest_attestation(store, validator_index)`. +func (a *Service) LatestAttestationTarget(beaconState *pb.BeaconState, index uint64) (*pb.AttestationTarget, error) { + if index >= uint64(len(beaconState.ValidatorRegistry)) { + return nil, fmt.Errorf("invalid validator index %d", index) } + validator := beaconState.ValidatorRegistry[index] pubKey := bytesutil.ToBytes48(validator.Pubkey) a.store.RLock() @@ -107,21 +107,7 @@ func (a *Service) LatestAttestation(ctx context.Context, index uint64) (*pb.Atte return nil, nil } - return a.store.m[pubKey], nil -} - -// LatestAttestationTarget returns the target block the validator index attested to, -// the highest slotNumber attestation in attestation pool gets returned. -// -// Spec pseudocode definition: -// Let `get_latest_attestation_target(store: Store, validator_index: ValidatorIndex) -> -// BeaconBlock` be the target block in the attestation -// `get_latest_attestation(store, validator_index)`. -func (a *Service) LatestAttestationTarget(ctx context.Context, index uint64) (*pb.AttestationTarget, error) { - attestation, err := a.LatestAttestation(ctx, index) - if err != nil { - return nil, fmt.Errorf("could not get attestation: %v", err) - } + attestation := a.store.m[pubKey] if attestation == nil { return nil, nil } diff --git a/beacon-chain/attestation/service_test.go b/beacon-chain/attestation/service_test.go index a376e4bedbeb..1b0697ba6bef 100644 --- a/beacon-chain/attestation/service_test.go +++ b/beacon-chain/attestation/service_test.go @@ -131,52 +131,7 @@ func TestAttestationPool_UpdatesAttestationPool(t *testing.T) { } } -func TestLatestAttestation_ReturnsLatestAttestation(t *testing.T) { - beaconDB := internal.SetupDB(t) - defer internal.TeardownDB(t, beaconDB) - ctx := context.Background() - - pubKey := []byte{'A'} - if err := beaconDB.SaveState(ctx, &pb.BeaconState{ - ValidatorRegistry: []*pb.Validator{{Pubkey: pubKey}}, - }); err != nil { - t.Fatalf("could not save state: %v", err) - } - - service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB}) - pubKey48 := bytesutil.ToBytes48(pubKey) - attestation := &pb.Attestation{AggregationBitfield: []byte{'B'}} - service.store.m[pubKey48] = attestation - - latestAttestation, err := service.LatestAttestation(ctx, 0) - if err != nil { - t.Fatalf("Could not get latest attestation: %v", err) - } - if !reflect.DeepEqual(attestation, latestAttestation) { - t.Errorf("Wanted: %v, got: %v", attestation, latestAttestation) - } -} - -func TestLatestAttestation_InvalidIndex(t *testing.T) { - beaconDB := internal.SetupDB(t) - defer internal.TeardownDB(t, beaconDB) - ctx := context.Background() - - if err := beaconDB.SaveState(ctx, &pb.BeaconState{ - ValidatorRegistry: []*pb.Validator{}, - }); err != nil { - t.Fatalf("could not save state: %v", err) - } - service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB}) - - index := uint64(0) - want := fmt.Sprintf("invalid validator index %d", index) - if _, err := service.LatestAttestation(ctx, index); !strings.Contains(err.Error(), want) { - t.Errorf("Wanted error to contain %s, received %v", want, err) - } -} - -func TestLatestAttestation_NoAttestation(t *testing.T) { +func TestLatestAttestationTarget_CantGetAttestation(t *testing.T) { beaconDB := internal.SetupDB(t) defer internal.TeardownDB(t, beaconDB) ctx := context.Background() @@ -187,32 +142,14 @@ func TestLatestAttestation_NoAttestation(t *testing.T) { t.Fatalf("could not save state: %v", err) } service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB}) - - index := 0 - a, err := service.LatestAttestation(ctx, uint64(index)) + headState, err := beaconDB.HeadState(ctx) if err != nil { - t.Fatalf("could not run latest attestation: %v", err) - } - if a != nil { - t.Errorf("Wanted attesstation %v, received %v", nil, a) - } -} - -func TestLatestAttestationTarget_CantGetAttestation(t *testing.T) { - beaconDB := internal.SetupDB(t) - defer internal.TeardownDB(t, beaconDB) - ctx := context.Background() - - if err := beaconDB.SaveState(ctx, &pb.BeaconState{ - ValidatorRegistry: []*pb.Validator{{}}, - }); err != nil { - t.Fatalf("could not save state: %v", err) + t.Fatal(err) } - service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB}) index := uint64(100) - want := fmt.Sprintf("could not get attestation: invalid validator index %d", index) - if _, err := service.LatestAttestationTarget(ctx, index); !strings.Contains(err.Error(), want) { + want := fmt.Sprintf("invalid validator index %d", index) + if _, err := service.LatestAttestationTarget(headState, index); !strings.Contains(err.Error(), want) { t.Errorf("Wanted error to contain %s, received %v", want, err) } } @@ -254,7 +191,12 @@ func TestLatestAttestationTarget_ReturnsLatestAttestedBlock(t *testing.T) { pubKey48 := bytesutil.ToBytes48(pubKey) service.store.m[pubKey48] = attestation - latestAttestedTarget, err := service.LatestAttestationTarget(ctx, 0) + headState, err := beaconDB.HeadState(ctx) + if err != nil { + t.Fatal(err) + } + + latestAttestedTarget, err := service.LatestAttestationTarget(headState, 0) if err != nil { t.Fatalf("Could not get latest attestation: %v", err) } diff --git a/beacon-chain/blockchain/fork_choice.go b/beacon-chain/blockchain/fork_choice.go index 412a155c0789..394f10027ae9 100644 --- a/beacon-chain/blockchain/fork_choice.go +++ b/beacon-chain/blockchain/fork_choice.go @@ -132,7 +132,7 @@ func (c *ChainService) ApplyForkChoiceRule( if err != nil { return fmt.Errorf("could not retrieve justified state: %v", err) } - attestationTargets, err := c.attestationTargets(ctx, justifiedState) + attestationTargets, err := c.attestationTargets(justifiedState) if err != nil { return fmt.Errorf("could not retrieve attestation target: %v", err) } @@ -291,11 +291,11 @@ func (c *ChainService) blockChildren(ctx context.Context, block *pb.BeaconBlock, // attestationTargets retrieves the list of attestation targets since last finalized epoch, // each attestation target consists of validator index and its attestation target (i.e. the block // which the validator attested to) -func (c *ChainService) attestationTargets(ctx context.Context, state *pb.BeaconState) (map[uint64]*pb.AttestationTarget, error) { +func (c *ChainService) attestationTargets(state *pb.BeaconState) (map[uint64]*pb.AttestationTarget, error) { indices := helpers.ActiveValidatorIndices(state.ValidatorRegistry, helpers.CurrentEpoch(state)) attestationTargets := make(map[uint64]*pb.AttestationTarget) for i, index := range indices { - target, err := c.attsService.LatestAttestationTarget(ctx, index) + target, err := c.attsService.LatestAttestationTarget(state, index) if err != nil { return nil, fmt.Errorf("could not retrieve attestation target: %v", err) } diff --git a/beacon-chain/blockchain/fork_choice_test.go b/beacon-chain/blockchain/fork_choice_test.go index 3eca3c01ff24..5ab18d3405e6 100644 --- a/beacon-chain/blockchain/fork_choice_test.go +++ b/beacon-chain/blockchain/fork_choice_test.go @@ -260,7 +260,7 @@ func TestAttestationTargets_RetrieveWorks(t *testing.T) { attsService.InsertAttestationIntoStore(pubKey48, att) chainService := setupBeaconChain(t, beaconDB, attsService) - attestationTargets, err := chainService.attestationTargets(ctx, beaconState) + attestationTargets, err := chainService.attestationTargets(beaconState) if err != nil { t.Fatalf("Could not get attestation targets: %v", err) } From 5bf1032b7da6ee373ecc700fc84acb1be03f2262 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Thu, 25 Apr 2019 12:21:45 -0500 Subject: [PATCH 24/54] Prevent Reading Head State on Every Received Attestation (#2383) * no more calls * fix lint unused * Update beacon-chain/attestation/service.go Co-Authored-By: rauljordan * fix tests * fixed broken tests * using ctx * fixed single state req * tests pass * fix one of last major bottlenecks * fix test * lint --- beacon-chain/sync/regular_sync.go | 12 ++++-------- beacon-chain/sync/regular_sync_test.go | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/beacon-chain/sync/regular_sync.go b/beacon-chain/sync/regular_sync.go index d7e0c5ca5b47..b62a188d70d1 100644 --- a/beacon-chain/sync/regular_sync.go +++ b/beacon-chain/sync/regular_sync.go @@ -433,20 +433,16 @@ func (rs *RegularSync) receiveAttestation(msg p2p.Message) error { } // Skip if attestation slot is older than last finalized slot in state. - beaconState, err := rs.db.HeadState(ctx) - if err != nil { - log.Errorf("Failed to get beacon state: %v", err) - return err - } + highestSlot := rs.db.HighestBlockSlot() span.AddAttributes( trace.Int64Attribute("attestation.Data.Slot", int64(attestation.Data.Slot)), - trace.Int64Attribute("finalized state slot", int64(beaconState.Slot-params.BeaconConfig().SlotsPerEpoch)), + trace.Int64Attribute("finalized state slot", int64(highestSlot-params.BeaconConfig().SlotsPerEpoch)), ) - if attestation.Data.Slot < beaconState.Slot-params.BeaconConfig().SlotsPerEpoch { + if attestation.Data.Slot < highestSlot-params.BeaconConfig().SlotsPerEpoch { log.WithFields(logrus.Fields{ "receivedSlot": attestation.Data.Slot, - "epochSlot": beaconState.Slot - params.BeaconConfig().SlotsPerEpoch}, + "epochSlot": highestSlot - params.BeaconConfig().SlotsPerEpoch}, ).Debug("Skipping received attestation with slot smaller than one epoch ago") return nil } diff --git a/beacon-chain/sync/regular_sync_test.go b/beacon-chain/sync/regular_sync_test.go index 5d54d8dcfc49..200077a9fe51 100644 --- a/beacon-chain/sync/regular_sync_test.go +++ b/beacon-chain/sync/regular_sync_test.go @@ -400,11 +400,21 @@ func TestReceiveAttestation_OK(t *testing.T) { db := internal.SetupDB(t) defer internal.TeardownDB(t, db) - if err := db.SaveState(ctx, &pb.BeaconState{ + beaconState := &pb.BeaconState{ Slot: params.BeaconConfig().GenesisSlot + 2, - }); err != nil { + } + if err := db.SaveState(ctx, beaconState); err != nil { t.Fatalf("Could not save state: %v", err) } + beaconBlock := &pb.BeaconBlock{ + Slot: beaconState.Slot, + } + if err := db.SaveBlock(beaconBlock); err != nil { + t.Fatal(err) + } + if err := db.UpdateChainHead(ctx, beaconBlock, beaconState); err != nil { + t.Fatal(err) + } cfg := &RegularSyncConfig{ ChainService: ms, AttsService: &mockAttestationService{}, From 82d0bbba2e15c861e2efd2c9cf2a549d2f93e5d4 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 25 Apr 2019 15:57:54 -0700 Subject: [PATCH 25/54] RPC Validator Index Lookup Failover (#2386) * implemented valIdxLookup and tests * comments * comments * look up in db * uncomment --- beacon-chain/db/validator.go | 18 ++++++++- beacon-chain/db/validator_test.go | 9 ++++- beacon-chain/rpc/validator_server_test.go | 46 ++++++++++++++++++++++- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/beacon-chain/db/validator.go b/beacon-chain/db/validator.go index f3213d325047..3fc4fc225b0c 100644 --- a/beacon-chain/db/validator.go +++ b/beacon-chain/db/validator.go @@ -2,6 +2,7 @@ package db import ( "bytes" + "context" "encoding/binary" "fmt" @@ -37,13 +38,28 @@ func (db *BeaconDB) SaveValidatorIndexBatch(pubKey []byte, index int) error { } // ValidatorIndex accepts a public key and returns the corresponding validator index. +// If the validator index is not found in DB, as a fail over, it searches the state and +// saves it to the DB when found. func (db *BeaconDB) ValidatorIndex(pubKey []byte) (uint64, error) { if !db.HasValidator(pubKey) { + state, err := db.HeadState(context.Background()) + if err != nil { + return 0, err + } + for i := 0; i < len(state.ValidatorRegistry); i++ { + v := state.ValidatorRegistry[i] + if bytes.Equal(v.Pubkey, pubKey) { + if err := db.SaveValidatorIndex(pubKey, i); err != nil { + return 0, err + } + return uint64(i), nil + } + } return 0, fmt.Errorf("validator %#x does not exist", pubKey) } + var index uint64 h := hashutil.Hash(pubKey) - err := db.view(func(tx *bolt.Tx) error { bucket := tx.Bucket(validatorBucket) diff --git a/beacon-chain/db/validator_test.go b/beacon-chain/db/validator_test.go index f7383c8fb6cc..ab80876354d5 100644 --- a/beacon-chain/db/validator_test.go +++ b/beacon-chain/db/validator_test.go @@ -1,11 +1,13 @@ package db import ( + "context" "fmt" "strings" "testing" "github.com/boltdb/bolt" + pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/hashutil" ) @@ -47,11 +49,14 @@ func TestSaveAndDeleteValidatorIndex_OK(t *testing.T) { p1 := []byte{'1', '2', '3'} if err := db.SaveValidatorIndex(p1, 3); err != nil { - t.Fatalf("Failed to save vallidator index: %v", err) + t.Fatalf("Failed to save validator index: %v", err) + } + if err := db.SaveState(context.Background(), &pb.BeaconState{}); err != nil { + t.Fatalf("Failed to save state: %v", err) } index, err := db.ValidatorIndex(p1) if err != nil { - t.Fatalf("Failed to call Attestation: %v", err) + t.Fatalf("Failed to call validator Index: %v", err) } if index != 3 { t.Fatalf("Saved index and retrieved index are not equal: %#x and %#x", 3, index) diff --git a/beacon-chain/rpc/validator_server_test.go b/beacon-chain/rpc/validator_server_test.go index 025c2b6521aa..84da12a5debb 100644 --- a/beacon-chain/rpc/validator_server_test.go +++ b/beacon-chain/rpc/validator_server_test.go @@ -64,6 +64,48 @@ func TestValidatorIndex_OK(t *testing.T) { } } +func TestValidatorIndex_InStateNotInDB(t *testing.T) { + db := internal.SetupDB(t) + defer internal.TeardownDB(t, db) + + pubKey := []byte{'A'} + + // Wanted validator with public key 'A' is in index '1'. + s := &pbp2p.BeaconState{ + ValidatorRegistry: []*pbp2p.Validator{{Pubkey: []byte{0}}, {Pubkey: []byte{'A'}}, {Pubkey: []byte{'B'}}}, + } + + if err := db.SaveState(context.Background(), s); err != nil { + t.Fatal(err) + } + + validatorServer := &ValidatorServer{ + beaconDB: db, + } + + req := &pb.ValidatorIndexRequest{ + PublicKey: pubKey, + } + + // Verify index can be retrieved from state when it's not saved in DB. + res, err := validatorServer.ValidatorIndex(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if res.Index != 1 { + t.Errorf("Wanted index 1 got %d", res.Index) + } + + // Verify index is also saved in DB. + idx, err := validatorServer.beaconDB.ValidatorIndex(pubKey) + if err != nil { + t.Fatal(err) + } + if idx != 1 { + t.Errorf("Wanted index 1 in DB got %d", res.Index) + } +} + func TestNextEpochCommitteeAssignment_WrongPubkeyLength(t *testing.T) { db := internal.SetupDB(t) ctx := context.Background() @@ -691,7 +733,9 @@ func TestAddNonActivePublicKeysAssignmentStatus(t *testing.T) { }, }, } - + if err := db.SaveState(context.Background(), beaconState); err != nil { + t.Fatal(err) + } vs := &ValidatorServer{ beaconDB: db, } From 2318a53fa562b6fcb8817367295ca062260828e8 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Thu, 25 Apr 2019 23:06:16 -0500 Subject: [PATCH 26/54] Lock Down the ReceiveBlock Function (#2387) * lock down receive block * imports --- beacon-chain/blockchain/block_processing.go | 2 ++ beacon-chain/blockchain/service.go | 1 + 2 files changed, 3 insertions(+) diff --git a/beacon-chain/blockchain/block_processing.go b/beacon-chain/blockchain/block_processing.go index c1245d3705bd..c5bb82f53cea 100644 --- a/beacon-chain/blockchain/block_processing.go +++ b/beacon-chain/blockchain/block_processing.go @@ -54,6 +54,8 @@ func (b *BlockFailedProcessingErr) Error() string { // 4. Process and cleanup any block operations, such as attestations and deposits, which would need to be // either included or flushed from the beacon node's runtime. func (c *ChainService) ReceiveBlock(ctx context.Context, block *pb.BeaconBlock) (*pb.BeaconState, error) { + c.receiveBlockLock.Lock() + defer c.receiveBlockLock.Unlock() ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlock") defer span.End() parentRoot := bytesutil.ToBytes32(block.ParentRootHash32) diff --git a/beacon-chain/blockchain/service.go b/beacon-chain/blockchain/service.go index c37d9fe18eae..daede40b9126 100644 --- a/beacon-chain/blockchain/service.go +++ b/beacon-chain/blockchain/service.go @@ -48,6 +48,7 @@ type ChainService struct { p2p p2p.Broadcaster canonicalBlocks map[uint64][]byte canonicalBlocksLock sync.RWMutex + receiveBlockLock sync.Mutex } // Config options for the service. From 89600e897869fb22acc4318c466098a8e57e019a Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Fri, 26 Apr 2019 12:31:27 +0800 Subject: [PATCH 27/54] Remove Duplicated Functions in the Slice Package (#2389) * remove duplicated funcs * lint * remove duplicated tests --- .../core/balances/rewards_penalties.go | 14 +- beacon-chain/core/state/transition.go | 8 +- beacon-chain/core/validators/validator.go | 4 +- beacon-chain/sync/receive_block.go | 2 + shared/sliceutil/slice.go | 136 ------------- shared/sliceutil/slice_generic_test.go | 8 +- shared/sliceutil/slice_test.go | 179 +----------------- 7 files changed, 20 insertions(+), 331 deletions(-) diff --git a/beacon-chain/core/balances/rewards_penalties.go b/beacon-chain/core/balances/rewards_penalties.go index 40703e29f3f1..3f3b0295cab4 100644 --- a/beacon-chain/core/balances/rewards_penalties.go +++ b/beacon-chain/core/balances/rewards_penalties.go @@ -41,7 +41,7 @@ func ExpectedFFGSource( totalBalance } activeValidatorIndices := helpers.ActiveValidatorIndices(state.ValidatorRegistry, helpers.CurrentEpoch(state)) - didNotAttestIndices := sliceutil.Not(justifiedAttesterIndices, activeValidatorIndices) + didNotAttestIndices := sliceutil.NotUint64(justifiedAttesterIndices, activeValidatorIndices) for _, index := range didNotAttestIndices { state.ValidatorBalances[index] -= @@ -77,7 +77,7 @@ func ExpectedFFGTarget( totalBalance } activeValidatorIndices := helpers.ActiveValidatorIndices(state.ValidatorRegistry, helpers.CurrentEpoch(state)) - didNotAttestIndices := sliceutil.Not(boundaryAttesterIndices, activeValidatorIndices) + didNotAttestIndices := sliceutil.NotUint64(boundaryAttesterIndices, activeValidatorIndices) for _, index := range didNotAttestIndices { state.ValidatorBalances[index] -= @@ -113,7 +113,7 @@ func ExpectedBeaconChainHead( totalBalance } activeValidatorIndices := helpers.ActiveValidatorIndices(state.ValidatorRegistry, helpers.CurrentEpoch(state)) - didNotAttestIndices := sliceutil.Not(headAttesterIndices, activeValidatorIndices) + didNotAttestIndices := sliceutil.NotUint64(headAttesterIndices, activeValidatorIndices) for _, index := range didNotAttestIndices { state.ValidatorBalances[index] -= @@ -169,7 +169,7 @@ func InactivityFFGSource( baseRewardQuotient := helpers.BaseRewardQuotient(totalBalance) activeValidatorIndices := helpers.ActiveValidatorIndices(state.ValidatorRegistry, helpers.CurrentEpoch(state)) - didNotAttestIndices := sliceutil.Not(justifiedAttesterIndices, activeValidatorIndices) + didNotAttestIndices := sliceutil.NotUint64(justifiedAttesterIndices, activeValidatorIndices) for _, index := range didNotAttestIndices { state.ValidatorBalances[index] -= @@ -193,7 +193,7 @@ func InactivityFFGTarget( baseRewardQuotient := helpers.BaseRewardQuotient(totalBalance) activeValidatorIndices := helpers.ActiveValidatorIndices(state.ValidatorRegistry, helpers.CurrentEpoch(state)) - didNotAttestIndices := sliceutil.Not(boundaryAttesterIndices, activeValidatorIndices) + didNotAttestIndices := sliceutil.NotUint64(boundaryAttesterIndices, activeValidatorIndices) for _, index := range didNotAttestIndices { state.ValidatorBalances[index] -= @@ -216,7 +216,7 @@ func InactivityChainHead( baseRewardQuotient := helpers.BaseRewardQuotient(totalBalance) activeValidatorIndices := helpers.ActiveValidatorIndices(state.ValidatorRegistry, helpers.CurrentEpoch(state)) - didNotAttestIndices := sliceutil.Not(headAttesterIndices, activeValidatorIndices) + didNotAttestIndices := sliceutil.NotUint64(headAttesterIndices, activeValidatorIndices) for _, index := range didNotAttestIndices { state.ValidatorBalances[index] -= @@ -366,7 +366,7 @@ func Crosslinks( } for _, index := range committee { baseReward := helpers.BaseReward(state, index, baseRewardQuotient) - if sliceutil.IsIn(index, attestingIndices) { + if sliceutil.IsInUint64(index, attestingIndices) { state.ValidatorBalances[index] += baseReward * totalAttestingBalance / totalBalance } else { diff --git a/beacon-chain/core/state/transition.go b/beacon-chain/core/state/transition.go index 18145bc3af78..a5f03838e99f 100644 --- a/beacon-chain/core/state/transition.go +++ b/beacon-chain/core/state/transition.go @@ -256,14 +256,14 @@ func ProcessEpoch(ctx context.Context, state *pb.BeaconState, block *pb.BeaconBl sameRoot := bytes.Equal(attestationData.EpochBoundaryRootHash32, boundaryBlockRoot) if sameRoot { currentEpochBoundaryAttestations = append(currentEpochBoundaryAttestations, attestation) - currentBoundaryAttesterIndices = sliceutil.Union(currentBoundaryAttesterIndices, attesterIndices) + currentBoundaryAttesterIndices = sliceutil.UnionUint64(currentBoundaryAttesterIndices, attesterIndices) } } // We extract the attestations from the previous epoch. if prevEpoch == helpers.SlotToEpoch(attestation.Data.Slot) { prevEpochAttestations = append(prevEpochAttestations, attestation) - prevEpochAttesterIndices = sliceutil.Union(prevEpochAttesterIndices, attesterIndices) + prevEpochAttesterIndices = sliceutil.UnionUint64(prevEpochAttesterIndices, attesterIndices) // We extract the previous epoch boundary attestations. prevBoundaryBlockRoot, err := b.BlockRoot(state, @@ -273,7 +273,7 @@ func ProcessEpoch(ctx context.Context, state *pb.BeaconState, block *pb.BeaconBl } if bytes.Equal(attestation.Data.EpochBoundaryRootHash32, prevBoundaryBlockRoot) { prevEpochBoundaryAttestations = append(prevEpochBoundaryAttestations, attestation) - prevEpochBoundaryAttesterIndices = sliceutil.Union(prevEpochBoundaryAttesterIndices, attesterIndices) + prevEpochBoundaryAttesterIndices = sliceutil.UnionUint64(prevEpochBoundaryAttesterIndices, attesterIndices) } // We extract the previous epoch head attestations. @@ -285,7 +285,7 @@ func ProcessEpoch(ctx context.Context, state *pb.BeaconState, block *pb.BeaconBl attestationData := attestation.Data if bytes.Equal(attestationData.BeaconBlockRootHash32, canonicalBlockRoot) { prevEpochHeadAttestations = append(prevEpochHeadAttestations, attestation) - prevEpochHeadAttesterIndices = sliceutil.Union(prevEpochHeadAttesterIndices, attesterIndices) + prevEpochHeadAttesterIndices = sliceutil.UnionUint64(prevEpochHeadAttesterIndices, attesterIndices) } } } diff --git a/beacon-chain/core/validators/validator.go b/beacon-chain/core/validators/validator.go index f28a3e09025e..65f36a77cad9 100644 --- a/beacon-chain/core/validators/validator.go +++ b/beacon-chain/core/validators/validator.go @@ -54,7 +54,7 @@ func ValidatorIndices( return nil, err } - attesterIndicesIntersection = sliceutil.Union(attesterIndicesIntersection, attesterIndices) + attesterIndicesIntersection = sliceutil.UnionUint64(attesterIndicesIntersection, attesterIndices) } return attesterIndicesIntersection, nil @@ -87,7 +87,7 @@ func AttestingValidatorIndices( if err != nil { return nil, fmt.Errorf("could not get attester indices: %v", err) } - validatorIndicesCommittees = sliceutil.Union(validatorIndicesCommittees, validatorIndicesCommittee) + validatorIndicesCommittees = sliceutil.UnionUint64(validatorIndicesCommittees, validatorIndicesCommittee) } } return validatorIndicesCommittees, nil diff --git a/beacon-chain/sync/receive_block.go b/beacon-chain/sync/receive_block.go index cbcb0fadb77a..ff7be3e6ddb7 100644 --- a/beacon-chain/sync/receive_block.go +++ b/beacon-chain/sync/receive_block.go @@ -191,6 +191,8 @@ func (rs *RegularSync) validateAndProcessBlock( span.AddAttributes(trace.BoolAttribute("invalidBlock", true)) return nil, nil, false, err } + } else { + log.Errorf("Received Block from a Forked Chain %#x with slot %d", blockRoot, block.Slot) } sentBlocks.Inc() diff --git a/shared/sliceutil/slice.go b/shared/sliceutil/slice.go index e58914eba801..de1df0cd8843 100644 --- a/shared/sliceutil/slice.go +++ b/shared/sliceutil/slice.go @@ -1,73 +1,5 @@ package sliceutil -// Intersection of two uint64 slices with time -// complexity of approximately O(n) leveraging a map to -// check for element existence off by a constant factor -// of underlying map efficiency. -func Intersection(a []uint64, b []uint64) []uint64 { - set := make([]uint64, 0) - m := make(map[uint64]bool) - - for i := 0; i < len(a); i++ { - m[a[i]] = true - } - for i := 0; i < len(b); i++ { - if _, found := m[b[i]]; found { - set = append(set, b[i]) - } - } - return set -} - -// Union of two uint64 slices with time -// complexity of approximately O(n) leveraging a map to -// check for element existence off by a constant factor -// of underlying map efficiency. -func Union(a []uint64, b []uint64) []uint64 { - set := make([]uint64, 0) - m := make(map[uint64]bool) - - for i := 0; i < len(a); i++ { - m[a[i]] = true - set = append(set, a[i]) - } - for i := 0; i < len(b); i++ { - if _, found := m[b[i]]; !found { - set = append(set, b[i]) - } - } - return set -} - -// Not returns the uint64 in slice a that are -// not in slice b with time complexity of approximately -// O(n) leveraging a map to check for element existence -// off by a constant factor of underlying map efficiency. -func Not(a []uint64, b []uint64) []uint64 { - set := make([]uint64, 0) - m := make(map[uint64]bool) - - for i := 0; i < len(a); i++ { - m[a[i]] = true - } - for i := 0; i < len(b); i++ { - if _, found := m[b[i]]; !found { - set = append(set, b[i]) - } - } - return set -} - -// IsIn returns true if a is in b and False otherwise. -func IsIn(a uint64, b []uint64) bool { - for _, v := range b { - if a == v { - return true - } - } - return false -} - // IntersectionUint64 of two uint64 slices with time // complexity of approximately O(n) leveraging a map to // check for element existence off by a constant factor @@ -136,74 +68,6 @@ func IsInUint64(a uint64, b []uint64) bool { return false } -// Intersectionint64 of two int64 slices with time -// complexity of approximately O(n) leveraging a map to -// check for element existence off by a constant factor -// of underlying map efficiency. -func Intersectionint64(a []int64, b []int64) []int64 { - set := make([]int64, 0) - m := make(map[int64]bool) - - for i := 0; i < len(a); i++ { - m[a[i]] = true - } - for i := 0; i < len(b); i++ { - if _, found := m[b[i]]; found { - set = append(set, b[i]) - } - } - return set -} - -// Unionint64 of two int64 slices with time -// complexity of approximately O(n) leveraging a map to -// check for element existence off by a constant factor -// of underlying map efficiency. -func Unionint64(a []int64, b []int64) []int64 { - set := make([]int64, 0) - m := make(map[int64]bool) - - for i := 0; i < len(a); i++ { - m[a[i]] = true - set = append(set, a[i]) - } - for i := 0; i < len(b); i++ { - if _, found := m[b[i]]; !found { - set = append(set, b[i]) - } - } - return set -} - -// Notint64 returns the int64 in slice a that are -// not in slice b with time complexity of approximately -// O(n) leveraging a map to check for element existence -// off by a constant factor of underlying map efficiency. -func Notint64(a []int64, b []int64) []int64 { - set := make([]int64, 0) - m := make(map[int64]bool) - - for i := 0; i < len(a); i++ { - m[a[i]] = true - } - for i := 0; i < len(b); i++ { - if _, found := m[b[i]]; !found { - set = append(set, b[i]) - } - } - return set -} - -// IsInint64 returns true if a is in b and False otherwise. -func IsInint64(a int64, b []int64) bool { - for _, v := range b { - if a == v { - return true - } - } - return false -} - // IntersectionInt64 of two int64 slices with time // complexity of approximately O(n) leveraging a map to // check for element existence off by a constant factor diff --git a/shared/sliceutil/slice_generic_test.go b/shared/sliceutil/slice_generic_test.go index 642bd436616b..4f7f1596894d 100644 --- a/shared/sliceutil/slice_generic_test.go +++ b/shared/sliceutil/slice_generic_test.go @@ -370,7 +370,7 @@ func BenchmarkIntersection(b *testing.B) { {[]uint64{1}, []uint64{1}, []uint64{1}}, } for _, tt := range testCases { - Intersection(tt.setA, tt.setB) + IntersectionUint64(tt.setA, tt.setB) } } @@ -393,7 +393,7 @@ func BenchmarkUnion(b *testing.B) { {[]uint64{1}, []uint64{1}, []uint64{1}}, } for _, tt := range testCases { - Union(tt.setA, tt.setB) + UnionUint64(tt.setA, tt.setB) } @@ -443,7 +443,7 @@ func BenchmarkNot(b *testing.B) { {[]uint64{1}, []uint64{1}, []uint64{1}}, } for _, tt := range testCases { - Not(tt.setA, tt.setB) + NotUint64(tt.setA, tt.setB) } @@ -490,7 +490,7 @@ func BenchmarkIsIn(b *testing.B) { {100, []uint64{2, 3, 5, 4, 6}, false}, } for _, tt := range testCases { - IsIn(tt.a, tt.b) + IsInUint64(tt.a, tt.b) } diff --git a/shared/sliceutil/slice_test.go b/shared/sliceutil/slice_test.go index f2d54b9329db..19b57edcfc1b 100644 --- a/shared/sliceutil/slice_test.go +++ b/shared/sliceutil/slice_test.go @@ -5,93 +5,6 @@ import ( "testing" ) -func TestIntersection(t *testing.T) { - testCases := []struct { - setA []uint64 - setB []uint64 - out []uint64 - }{ - {[]uint64{2, 3, 5}, []uint64{3}, []uint64{3}}, - {[]uint64{2, 3, 5}, []uint64{3, 5}, []uint64{3, 5}}, - {[]uint64{2, 3, 5}, []uint64{5, 3, 2}, []uint64{5, 3, 2}}, - {[]uint64{2, 3, 5}, []uint64{2, 3, 5}, []uint64{2, 3, 5}}, - {[]uint64{2, 3, 5}, []uint64{}, []uint64{}}, - {[]uint64{}, []uint64{2, 3, 5}, []uint64{}}, - {[]uint64{}, []uint64{}, []uint64{}}, - {[]uint64{1}, []uint64{1}, []uint64{1}}, - } - for _, tt := range testCases { - result := Intersection(tt.setA, tt.setB) - if !reflect.DeepEqual(result, tt.out) { - t.Errorf("got %d, want %d", result, tt.out) - } - } -} - -func TestUnion(t *testing.T) { - testCases := []struct { - setA []uint64 - setB []uint64 - out []uint64 - }{ - {[]uint64{2, 3, 5}, []uint64{4, 6}, []uint64{2, 3, 5, 4, 6}}, - {[]uint64{2, 3, 5}, []uint64{3, 5}, []uint64{2, 3, 5}}, - {[]uint64{2, 3, 5}, []uint64{2, 3, 5}, []uint64{2, 3, 5}}, - {[]uint64{2, 3, 5}, []uint64{}, []uint64{2, 3, 5}}, - {[]uint64{}, []uint64{2, 3, 5}, []uint64{2, 3, 5}}, - {[]uint64{}, []uint64{}, []uint64{}}, - {[]uint64{1}, []uint64{1}, []uint64{1}}, - } - for _, tt := range testCases { - result := Union(tt.setA, tt.setB) - if !reflect.DeepEqual(result, tt.out) { - t.Errorf("got %d, want %d", result, tt.out) - } - } -} - -func TestNot(t *testing.T) { - testCases := []struct { - setA []uint64 - setB []uint64 - out []uint64 - }{ - {[]uint64{4, 6}, []uint64{2, 3, 5, 4, 6}, []uint64{2, 3, 5}}, - {[]uint64{3, 5}, []uint64{2, 3, 5}, []uint64{2}}, - {[]uint64{2, 3, 5}, []uint64{2, 3, 5}, []uint64{}}, - {[]uint64{2}, []uint64{2, 3, 5}, []uint64{3, 5}}, - {[]uint64{}, []uint64{2, 3, 5}, []uint64{2, 3, 5}}, - {[]uint64{}, []uint64{}, []uint64{}}, - {[]uint64{1}, []uint64{1}, []uint64{}}, - } - for _, tt := range testCases { - result := Not(tt.setA, tt.setB) - if !reflect.DeepEqual(result, tt.out) { - t.Errorf("got %d, want %d", result, tt.out) - } - } -} - -func TestIsIn(t *testing.T) { - testCases := []struct { - a uint64 - b []uint64 - result bool - }{ - {0, []uint64{}, false}, - {0, []uint64{0}, true}, - {4, []uint64{2, 3, 5, 4, 6}, true}, - {100, []uint64{2, 3, 5, 4, 6}, false}, - } - for _, tt := range testCases { - result := IsIn(tt.a, tt.b) - if result != tt.result { - t.Errorf("IsIn(%d, %v)=%v, wanted: %v", - tt.a, tt.b, result, tt.result) - } - } -} - func TestIntersectionUint64(t *testing.T) { testCases := []struct { setA []uint64 @@ -116,31 +29,6 @@ func TestIntersectionUint64(t *testing.T) { } } -func TestIntersectionint64(t *testing.T) { - testCases := []struct { - setA []int64 - setB []int64 - out []int64 - }{ - {[]int64{2, 3, 5}, []int64{3}, []int64{3}}, - {[]int64{2, 3, 5}, []int64{3, 5}, []int64{3, 5}}, - {[]int64{2, 3, 5}, []int64{5, 3, 2}, []int64{5, 3, 2}}, - {[]int64{2, 3, 5}, []int64{2, 3, 5}, []int64{2, 3, 5}}, - {[]int64{2, 3, 5}, []int64{}, []int64{}}, - {[]int64{}, []int64{2, 3, 5}, []int64{}}, - {[]int64{}, []int64{}, []int64{}}, - {[]int64{1}, []int64{1}, []int64{1}}, - } - for _, tt := range testCases { - result := Intersectionint64(tt.setA, tt.setB) - if !reflect.DeepEqual(result, tt.out) { - t.Errorf("got %d, want %d", result, tt.out) - } - - } - -} - func TestIntersectionInt64(t *testing.T) { testCases := []struct { setA []int64 @@ -189,30 +77,6 @@ func TestUnionUint64(t *testing.T) { } } -func TestUnionint64(t *testing.T) { - testCases := []struct { - setA []int64 - setB []int64 - out []int64 - }{ - {[]int64{2, 3, 5}, []int64{4, 6}, []int64{2, 3, 5, 4, 6}}, - {[]int64{2, 3, 5}, []int64{3, 5}, []int64{2, 3, 5}}, - {[]int64{2, 3, 5}, []int64{2, 3, 5}, []int64{2, 3, 5}}, - {[]int64{2, 3, 5}, []int64{}, []int64{2, 3, 5}}, - {[]int64{}, []int64{2, 3, 5}, []int64{2, 3, 5}}, - {[]int64{}, []int64{}, []int64{}}, - {[]int64{1}, []int64{1}, []int64{1}}, - } - for _, tt := range testCases { - result := Unionint64(tt.setA, tt.setB) - if !reflect.DeepEqual(result, tt.out) { - t.Errorf("got %d, want %d", result, tt.out) - } - - } - -} - func TestUnionInt64(t *testing.T) { testCases := []struct { setA []int64 @@ -234,6 +98,7 @@ func TestUnionInt64(t *testing.T) { } } + } func TestNotUint64(t *testing.T) { @@ -280,28 +145,6 @@ func TestNotInt64(t *testing.T) { } } -func TestNotint64(t *testing.T) { - testCases := []struct { - setA []int64 - setB []int64 - out []int64 - }{ - {[]int64{4, 6}, []int64{2, 3, 5, 4, 6}, []int64{2, 3, 5}}, - {[]int64{3, 5}, []int64{2, 3, 5}, []int64{2}}, - {[]int64{2, 3, 5}, []int64{2, 3, 5}, []int64{}}, - {[]int64{2}, []int64{2, 3, 5}, []int64{3, 5}}, - {[]int64{}, []int64{2, 3, 5}, []int64{2, 3, 5}}, - {[]int64{}, []int64{}, []int64{}}, - {[]int64{1}, []int64{1}, []int64{}}, - } - for _, tt := range testCases { - result := Notint64(tt.setA, tt.setB) - if !reflect.DeepEqual(result, tt.out) { - t.Errorf("got %d, want %d", result, tt.out) - } - } -} - func TestIsInUint64(t *testing.T) { testCases := []struct { a uint64 @@ -322,26 +165,6 @@ func TestIsInUint64(t *testing.T) { } } -func TestIsInint64(t *testing.T) { - testCases := []struct { - a int64 - b []int64 - result bool - }{ - {0, []int64{}, false}, - {0, []int64{0}, true}, - {4, []int64{2, 3, 5, 4, 6}, true}, - {100, []int64{2, 3, 5, 4, 6}, false}, - } - for _, tt := range testCases { - result := IsInint64(tt.a, tt.b) - if result != tt.result { - t.Errorf("IsIn(%d, %v)=%v, wanted: %v", - tt.a, tt.b, result, tt.result) - } - } -} - func TestIsInInt64(t *testing.T) { testCases := []struct { a int64 From 09083b072ca6c450d66ad4b5ba4a021f56bfb400 Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Fri, 26 Apr 2019 13:15:53 +0800 Subject: [PATCH 28/54] add log (#2390) --- beacon-chain/sync/metrics.go | 4 ++++ beacon-chain/sync/receive_block.go | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/beacon-chain/sync/metrics.go b/beacon-chain/sync/metrics.go index 299ef9409830..7a96cf94a68b 100644 --- a/beacon-chain/sync/metrics.go +++ b/beacon-chain/sync/metrics.go @@ -24,6 +24,10 @@ var ( Name: "regsync_received_blocks", Help: "The number of received blocks", }) + forkedBlock = promauto.NewCounter(prometheus.CounterOpts{ + Name: "regsync_received_forked_blocks", + Help: "The number of received forked blocks", + }) recBlockAnnounce = promauto.NewCounter(prometheus.CounterOpts{ Name: "regsync_received_block_announce", Help: "The number of received block announcements", diff --git a/beacon-chain/sync/receive_block.go b/beacon-chain/sync/receive_block.go index ff7be3e6ddb7..f5d93d0718a0 100644 --- a/beacon-chain/sync/receive_block.go +++ b/beacon-chain/sync/receive_block.go @@ -192,7 +192,8 @@ func (rs *RegularSync) validateAndProcessBlock( return nil, nil, false, err } } else { - log.Errorf("Received Block from a Forked Chain %#x with slot %d", blockRoot, block.Slot) + forkedBlock.Inc() + log.Warnf("Received Block from a Forked Chain with root %#x and slot %d", blockRoot, block.Slot) } sentBlocks.Inc() From a170c69653050fbbfe4cf513f2a8408f5311f7ed Mon Sep 17 00:00:00 2001 From: frederickalcantara Date: Fri, 26 Apr 2019 02:24:01 -0400 Subject: [PATCH 29/54] upgrading linter from gometalinter to golangci-lint (#2100) * upgrading linter from gometalinter to golangci-lint * fixed golangci-lint linting * removed linting before_script command * removed disable-all command * Fixed golang config file * fixed golang config file v2 * removed gosec issue rule * formatting * fixed travis build to run golangci-lint * Add install golangci-lint command * fixing golangci-lint script * removed https:// * Added golangci-lint cmd script * added go get for local lint install * created a before_script * add install before script * Added get script * added go mod download * removed go mod downloads * changed * removed before script * Added before script go get lint * added exit zero to see what went wrong * removed golang run script * removed before script * change lint command * verbose output * removed verbose * change linter enable and disable configuration * Update .golangci.yml Removed gotype as a linter * Update .golangci.yml Added typecheck linter * Update .golangci.yml Added fixed lint version * Update .golangci.yml Added gotype * Update .golangci.yml Added typecheck * removed env:lint * Added env lint * fixing lint upgrade * Changing travis configuration * FIxed spelling errors * disabled typecheck * Enabled typecheck * remove binary * Deleting lib binary * adding more linters * fixed constants * fix spelling * fixed all lint issues * Revert "Changing travis configuration" This reverts commit 334afe9d05e96261b01f275aa3ada20e7f36aac4. * Merge branch 'master' of https://github.com/prysmaticlabs/prysm into update-linter * Changed from Infof to Info * Fixing commits * fixing commits with linter config * added install * Fixing * fix log statement --- .golangci.yml | 69 +++++++++++++++++++ .gometalinter.json | 20 ------ .travis.yml | 3 +- README.md | 4 +- .../blockchain/block_processing_test.go | 12 ++-- beacon-chain/blockchain/fork_choice_test.go | 44 +++++------- beacon-chain/blockchain/service_test.go | 9 +-- beacon-chain/core/helpers/committee.go | 2 +- beacon-chain/db/state_test.go | 2 +- beacon-chain/operations/service_test.go | 4 +- beacon-chain/powchain/block_reader_test.go | 7 +- beacon-chain/powchain/log_processing.go | 2 +- beacon-chain/powchain/log_processing_test.go | 8 --- beacon-chain/powchain/service_test.go | 18 +---- beacon-chain/rpc/beacon_server_test.go | 5 +- beacon-chain/rpc/validator_server.go | 2 +- beacon-chain/sync/initial-sync/helpers.go | 4 +- beacon-chain/sync/regular_sync_test.go | 16 ++--- beacon-chain/sync/service_test.go | 9 +-- shared/messagehandler/messagehandler.go | 4 +- shared/p2p/service_norace_test.go | 2 +- shared/p2p/service_test.go | 23 ++++--- validator/client/runner.go | 2 +- validator/client/service_test.go | 4 +- validator/client/validator_metrics.go | 4 +- validator/client/validator_test.go | 6 +- 26 files changed, 153 insertions(+), 132 deletions(-) create mode 100644 .golangci.yml delete mode 100644 .gometalinter.json diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000000..05efd04a9f84 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,69 @@ +linters-settings: + govet: + check-shadowing: true + settings: + printf: + funcs: + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + golint: + min-confidence: 0 + gocyclo: + min-complexity: 10 + maligned: + suggest-new: true + dupl: + threshold: 100 + goconst: + min-len: 2 + min-occurrences: 2 + depguard: + list-type: blacklist + packages: + # logging is allowed only by logutils.Log, logrus + # is allowed to use only in logutils package + - github.com/sirupsen/logrus + misspell: + locale: US + lll: + line-length: 140 + goimports: + local-prefixes: github.com/golangci/golangci-lint + gocritic: + enabled-tags: + - performance + - style + - experimental + disabled-checks: + - wrapperFunc + +linters: + enable: + - deadcode + - goconst + - goimports + - golint + - gosec + - misspell + - structcheck + - typecheck + - unparam + - varcheck + - gofmt + - unused + disable-all: true + +run: + skip-dirs: + - proto/ + - ^contracts/ + deadline: 10m + +# golangci.com configuration +# https://github.com/golangci/golangci/wiki/Configuration +service: + golangci-lint-version: 1.15.0 # use the fixed version to not introduce new linters unexpectedly + prepare: + - echo "here I can run custom commands, but no preparation needed for this repo" diff --git a/.gometalinter.json b/.gometalinter.json deleted file mode 100644 index 927c09f481b7..000000000000 --- a/.gometalinter.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "Deadline": "10m", - "Exclude": [ - "^proto/", - "^contracts/" - ], - "DisableAll": true, - "Enable": [ - "deadcode", - "goconst", - "goimports", - "golint", - "gosec", - "gotype", - "misspell", - "structcheck", - "unparam", - "varcheck" - ] -} diff --git a/.travis.yml b/.travis.yml index a972f2245532..e4785b7a0d73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,9 @@ matrix: - lint install: - go get ${gobuild_args} -t ./... + - go get ${gobuild_args} github.com/golangci/golangci-lint/cmd/golangci-lint script: - - go get ${gobuild_args} github.com/alecthomas/gometalinter && gometalinter --install && gometalinter ./... + - golangci-lint run email: false after_success: - wget https://raw.githubusercontent.com/k3rn31p4nic/travis-ci-discord-webhook/master/send.sh diff --git a/README.md b/README.md index b1b92ea1d21b..0f146ae8221d 100644 --- a/README.md +++ b/README.md @@ -169,10 +169,10 @@ To run the unit tests of our system do: bazel test //... ``` -To run our linter, make sure you have [gometalinter](https://github.com/alecthomas/gometalinter) installed and then run: +To run our linter, make sure you have [golangci-lint](https://https://github.com/golangci/golangci-lint) installed and then run: ``` -gometalinter ./... +golangci-lint run ``` # Contributing diff --git a/beacon-chain/blockchain/block_processing_test.go b/beacon-chain/blockchain/block_processing_test.go index 0d34221e135a..3e9c54fd464a 100644 --- a/beacon-chain/blockchain/block_processing_test.go +++ b/beacon-chain/blockchain/block_processing_test.go @@ -191,7 +191,7 @@ func TestReceiveBlock_UsesParentBlockState(t *testing.T) { t.Fatalf("Could not tree hash state: %v", err) } - parentHash, genesisBlock := setupGenesisBlock(t, chainService, beaconState) + parentHash, genesisBlock := setupGenesisBlock(t, chainService) if err := chainService.beaconDB.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil { t.Fatal(err) } @@ -246,7 +246,7 @@ func TestReceiveBlock_DeletesBadBlock(t *testing.T) { t.Fatal(err) } - parentHash, genesisBlock := setupGenesisBlock(t, chainService, beaconState) + parentHash, genesisBlock := setupGenesisBlock(t, chainService) if err := chainService.beaconDB.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil { t.Fatal(err) } @@ -324,7 +324,7 @@ func TestReceiveBlock_CheckBlockStateRoot_GoodState(t *testing.T) { if err := chainService.beaconDB.SaveHistoricalState(ctx, beaconState); err != nil { t.Fatal(err) } - parentHash, genesisBlock := setupGenesisBlock(t, chainService, beaconState) + parentHash, genesisBlock := setupGenesisBlock(t, chainService) beaconState.Slot++ if err := chainService.beaconDB.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil { t.Fatal(err) @@ -368,7 +368,7 @@ func TestReceiveBlock_CheckBlockStateRoot_BadState(t *testing.T) { if err := chainService.beaconDB.SaveHistoricalState(ctx, beaconState); err != nil { t.Fatal(err) } - parentHash, genesisBlock := setupGenesisBlock(t, chainService, beaconState) + parentHash, genesisBlock := setupGenesisBlock(t, chainService) beaconState.Slot++ if err := chainService.beaconDB.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil { t.Fatal(err) @@ -423,7 +423,7 @@ func TestReceiveBlock_RemovesPendingDeposits(t *testing.T) { if err != nil { t.Fatalf("Could not tree hash state: %v", err) } - parentHash, genesisBlock := setupGenesisBlock(t, chainService, beaconState) + parentHash, genesisBlock := setupGenesisBlock(t, chainService) beaconState.Slot++ if err := chainService.beaconDB.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil { t.Fatal(err) @@ -578,7 +578,7 @@ func TestReceiveBlock_OnChainSplit(t *testing.T) { if err != nil { t.Fatalf("Could not tree hash state: %v", err) } - parentHash, genesisBlock := setupGenesisBlock(t, chainService, beaconState) + parentHash, genesisBlock := setupGenesisBlock(t, chainService) if err := db.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil { t.Fatal(err) } diff --git a/beacon-chain/blockchain/fork_choice_test.go b/beacon-chain/blockchain/fork_choice_test.go index 5ab18d3405e6..0902c19a32e3 100644 --- a/beacon-chain/blockchain/fork_choice_test.go +++ b/beacon-chain/blockchain/fork_choice_test.go @@ -29,6 +29,7 @@ import ( // Ensure ChainService implements interfaces. var _ = ForkChoice(&ChainService{}) +var endpoint = "ws://127.0.0.1" func TestApplyForkChoice_SetsCanonicalHead(t *testing.T) { deposits, _ := setupInitialDeposits(t, 5) @@ -804,7 +805,7 @@ func BenchmarkLMDGhost_8Slots_8Validators(b *testing.B) { balances[i] = params.BeaconConfig().MaxDepositAmount } - chainService := setupBeaconChainBenchmark(b, false, beaconDB, true, nil) + chainService := setupBeaconChainBenchmark(b, beaconDB) // Construct 8 blocks. (Epoch length = 8) epochLength := uint64(8) @@ -885,7 +886,7 @@ func BenchmarkLMDGhost_32Slots_8Validators(b *testing.B) { balances[i] = params.BeaconConfig().MaxDepositAmount } - chainService := setupBeaconChainBenchmark(b, false, beaconDB, true, nil) + chainService := setupBeaconChainBenchmark(b, beaconDB) // Construct 8 blocks. (Epoch length = 8) epochLength := uint64(8) @@ -964,7 +965,7 @@ func BenchmarkLMDGhost_32Slots_64Validators(b *testing.B) { balances[i] = params.BeaconConfig().MaxDepositAmount } - chainService := setupBeaconChainBenchmark(b, false, beaconDB, true, nil) + chainService := setupBeaconChainBenchmark(b, beaconDB) // Construct 64 blocks. (Epoch length = 64) epochLength := uint64(32) @@ -1043,7 +1044,7 @@ func BenchmarkLMDGhost_64Slots_16384Validators(b *testing.B) { balances[i] = params.BeaconConfig().MaxDepositAmount } - chainService := setupBeaconChainBenchmark(b, false, beaconDB, true, nil) + chainService := setupBeaconChainBenchmark(b, beaconDB) // Construct 64 blocks. (Epoch length = 64) epochLength := uint64(64) @@ -1107,32 +1108,19 @@ func BenchmarkLMDGhost_64Slots_16384Validators(b *testing.B) { } } -func setupBeaconChainBenchmark(b *testing.B, faultyPoWClient bool, beaconDB *db.BeaconDB, enablePOWChain bool, attsService *attestation.Service) *ChainService { - endpoint := "ws://127.0.0.1" +func setupBeaconChainBenchmark(b *testing.B, beaconDB *db.BeaconDB) *ChainService { ctx := context.Background() var web3Service *powchain.Web3Service var err error - if enablePOWChain { - if faultyPoWClient { - client := &faultyClient{} - web3Service, err = powchain.NewWeb3Service(ctx, &powchain.Web3ServiceConfig{ - Endpoint: endpoint, - DepositContract: common.Address{}, - Reader: client, - Client: client, - Logger: client, - }) - } else { - client := &mockClient{} - web3Service, err = powchain.NewWeb3Service(ctx, &powchain.Web3ServiceConfig{ - Endpoint: endpoint, - DepositContract: common.Address{}, - Reader: client, - Client: client, - Logger: client, - }) - } - } + client := &faultyClient{} + web3Service, err = powchain.NewWeb3Service(ctx, &powchain.Web3ServiceConfig{ + Endpoint: endpoint, + DepositContract: common.Address{}, + Reader: client, + Client: client, + Logger: client, + }) + if err != nil { b.Fatalf("unable to set up web3 service: %v", err) } @@ -1142,7 +1130,7 @@ func setupBeaconChainBenchmark(b *testing.B, faultyPoWClient bool, beaconDB *db. BeaconDB: beaconDB, Web3Service: web3Service, OpsPoolService: &mockOperationService{}, - AttsService: attsService, + AttsService: nil, } if err != nil { b.Fatalf("could not register blockchain service: %v", err) diff --git a/beacon-chain/blockchain/service_test.go b/beacon-chain/blockchain/service_test.go index be07bf043d2c..5a108a85347a 100644 --- a/beacon-chain/blockchain/service_test.go +++ b/beacon-chain/blockchain/service_test.go @@ -206,7 +206,7 @@ func createRandaoReveal(t *testing.T, beaconState *pb.BeaconState, privKeys []*b return epochSignature.Marshal() } -func setupGenesisBlock(t *testing.T, cs *ChainService, beaconState *pb.BeaconState) ([32]byte, *pb.BeaconBlock) { +func setupGenesisBlock(t *testing.T, cs *ChainService) ([32]byte, *pb.BeaconBlock) { genesis := b.NewGenesisBlock([]byte{}) if err := cs.beaconDB.SaveBlock(genesis); err != nil { t.Fatalf("could not save block to db: %v", err) @@ -307,7 +307,6 @@ func TestChainStartStop_Initialized(t *testing.T) { hook := logTest.NewGlobal() db := internal.SetupDB(t) defer internal.TeardownDB(t, db) - ctx := context.Background() chainService := setupBeaconChain(t, db, nil) @@ -316,11 +315,7 @@ func TestChainStartStop_Initialized(t *testing.T) { if err := db.InitializeState(context.Background(), unixTime, deposits, &pb.Eth1Data{}); err != nil { t.Fatalf("Could not initialize beacon state to disk: %v", err) } - beaconState, err := db.HeadState(ctx) - if err != nil { - t.Fatalf("Could not fetch beacon state: %v", err) - } - setupGenesisBlock(t, chainService, beaconState) + setupGenesisBlock(t, chainService) // Test the start function. chainService.Start() diff --git a/beacon-chain/core/helpers/committee.go b/beacon-chain/core/helpers/committee.go index 949f6ca5631d..6e48f1d590be 100644 --- a/beacon-chain/core/helpers/committee.go +++ b/beacon-chain/core/helpers/committee.go @@ -364,7 +364,7 @@ func VerifyBitfield(bitfield []byte, committeeSize int) (bool, error) { // * ``assignment[0]`` is the list of validators in the committee // * ``assignment[1]`` is the shard to which the committee is assigned // * ``assignment[2]`` is the slot at which the committee is assigned -// * ``assignment[3]`` is a bool signalling if the validator is expected to propose +// * ``assignment[3]`` is a bool signaling if the validator is expected to propose // a beacon block at the assigned slot. // """ // previous_epoch = get_previous_epoch(state) diff --git a/beacon-chain/db/state_test.go b/beacon-chain/db/state_test.go index 91099117d5d6..f28028aaac33 100644 --- a/beacon-chain/db/state_test.go +++ b/beacon-chain/db/state_test.go @@ -93,7 +93,7 @@ func TestFinalizeState_OK(t *testing.T) { defer teardownDB(t, db) genesisTime := uint64(time.Now().Unix()) - deposits, _ := setupInitialDeposits(t, 10) + deposits, _ := setupInitialDeposits(t, 20) if err := db.InitializeState(context.Background(), genesisTime, deposits, &pb.Eth1Data{}); err != nil { t.Fatalf("Failed to initialize state: %v", err) } diff --git a/beacon-chain/operations/service_test.go b/beacon-chain/operations/service_test.go index e856015727eb..13ecb6a3e74f 100644 --- a/beacon-chain/operations/service_test.go +++ b/beacon-chain/operations/service_test.go @@ -72,11 +72,11 @@ func TestRoutineContextClosing_Ok(t *testing.T) { s := NewOpsPoolService(context.Background(), &Config{BeaconDB: db}) exitRoutine := make(chan bool) - go func(tt *testing.T) { + go func() { s.removeOperations() s.saveOperations() <-exitRoutine - }(t) + }() s.cancel() exitRoutine <- true testutil.AssertLogsContain(t, hook, "operations service context closed, exiting remove goroutine") diff --git a/beacon-chain/powchain/block_reader_test.go b/beacon-chain/powchain/block_reader_test.go index 70d5a5c43904..656a455a9fb0 100644 --- a/beacon-chain/powchain/block_reader_test.go +++ b/beacon-chain/powchain/block_reader_test.go @@ -13,8 +13,9 @@ import ( gethTypes "github.com/ethereum/go-ethereum/core/types" ) +var endpoint = "ws://127.0.0.1" + func TestLatestMainchainInfo_OK(t *testing.T) { - endpoint := "ws://127.0.0.1" testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -84,7 +85,6 @@ func TestLatestMainchainInfo_OK(t *testing.T) { } func TestBlockHashByHeight_ReturnsHash(t *testing.T) { - endpoint := "ws://127.0.0.1" web3Service, err := NewWeb3Service(context.Background(), &Web3ServiceConfig{ Endpoint: endpoint, BlockFetcher: &goodFetcher{}, @@ -123,7 +123,6 @@ func TestBlockHashByHeight_ReturnsHash(t *testing.T) { } func TestBlockExists_ValidHash(t *testing.T) { - endpoint := "ws://127.0.0.1" web3Service, err := NewWeb3Service(context.Background(), &Web3ServiceConfig{ Endpoint: endpoint, BlockFetcher: &goodFetcher{}, @@ -163,7 +162,6 @@ func TestBlockExists_ValidHash(t *testing.T) { } func TestBlockExists_InvalidHash(t *testing.T) { - endpoint := "ws://127.0.0.1" web3Service, err := NewWeb3Service(context.Background(), &Web3ServiceConfig{ Endpoint: endpoint, BlockFetcher: &goodFetcher{}, @@ -179,7 +177,6 @@ func TestBlockExists_InvalidHash(t *testing.T) { } func TestBlockExists_UsesCachedBlockInfo(t *testing.T) { - endpoint := "ws://127.0.0.1" web3Service, err := NewWeb3Service(context.Background(), &Web3ServiceConfig{ Endpoint: endpoint, BlockFetcher: nil, // nil blockFetcher would panic if cached value not used diff --git a/beacon-chain/powchain/log_processing.go b/beacon-chain/powchain/log_processing.go index 246fca1441b4..c9a922505f62 100644 --- a/beacon-chain/powchain/log_processing.go +++ b/beacon-chain/powchain/log_processing.go @@ -189,7 +189,7 @@ func (w *Web3Service) processPastLogs() error { // last polled to now. func (w *Web3Service) requestBatchedLogs() error { // We request for the nth block behind the current head, in order to have - // stabilised logs when we retrieve it from the 1.0 chain. + // stabilized logs when we retrieve it from the 1.0 chain. requestedBlock := big.NewInt(0).Sub(w.blockHeight, big.NewInt(params.BeaconConfig().LogBlockDelay)) query := ethereum.FilterQuery{ Addresses: []common.Address{ diff --git a/beacon-chain/powchain/log_processing_test.go b/beacon-chain/powchain/log_processing_test.go index 2ebd064fb194..4ef751136ee6 100644 --- a/beacon-chain/powchain/log_processing_test.go +++ b/beacon-chain/powchain/log_processing_test.go @@ -28,7 +28,6 @@ func init() { func TestProcessDepositLog_OK(t *testing.T) { hook := logTest.NewGlobal() - endpoint := "ws://127.0.0.1" testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -91,7 +90,6 @@ func TestProcessDepositLog_OK(t *testing.T) { } func TestProcessDepositLog_InsertsPendingDeposit(t *testing.T) { - endpoint := "ws://127.0.0.1" testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -153,7 +151,6 @@ func TestProcessDepositLog_InsertsPendingDeposit(t *testing.T) { } func TestUnpackDepositLogData_OK(t *testing.T) { - endpoint := "ws://127.0.0.1" testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -237,7 +234,6 @@ func TestUnpackDepositLogData_OK(t *testing.T) { func TestProcessChainStartLog_8DuplicatePubkeys(t *testing.T) { hook := logTest.NewGlobal() - endpoint := "ws://127.0.0.1" testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -323,7 +319,6 @@ func TestProcessChainStartLog_8DuplicatePubkeys(t *testing.T) { func TestProcessChainStartLog_8UniquePubkeys(t *testing.T) { hook := logTest.NewGlobal() - endpoint := "ws://127.0.0.1" testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -408,7 +403,6 @@ func TestProcessChainStartLog_8UniquePubkeys(t *testing.T) { } func TestUnpackChainStartLogData_OK(t *testing.T) { - endpoint := "ws://127.0.0.1" testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -477,7 +471,6 @@ func TestUnpackChainStartLogData_OK(t *testing.T) { } func TestHasChainStartLogOccurred_OK(t *testing.T) { - endpoint := "ws://127.0.0.1" testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -538,7 +531,6 @@ func TestHasChainStartLogOccurred_OK(t *testing.T) { } func TestETH1DataGenesis_OK(t *testing.T) { - endpoint := "ws://127.0.0.1" testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) diff --git a/beacon-chain/powchain/service_test.go b/beacon-chain/powchain/service_test.go index 73b7557ee2fc..d10d8e692f44 100644 --- a/beacon-chain/powchain/service_test.go +++ b/beacon-chain/powchain/service_test.go @@ -38,16 +38,6 @@ func (g *goodReader) SubscribeNewHead(ctx context.Context, ch chan<- *gethTypes. return new(event.Feed).Subscribe(ch), nil } -type badLogger struct{} - -func (b *badLogger) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]gethTypes.Log, error) { - return nil, errors.New("unable to retrieve logs") -} - -func (b *badLogger) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- gethTypes.Log) (ethereum.Subscription, error) { - return nil, errors.New("subscription has failed") -} - type goodLogger struct{} func (g *goodLogger) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- gethTypes.Log) (ethereum.Subscription, error) { @@ -198,7 +188,6 @@ func TestNewWeb3Service_OK(t *testing.T) { func TestStart_OK(t *testing.T) { hook := logTest.NewGlobal() - endpoint := "ws://127.0.0.1" testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -237,7 +226,6 @@ func TestStart_OK(t *testing.T) { func TestStop_OK(t *testing.T) { hook := logTest.NewGlobal() - endpoint := "ws://127.0.0.1" testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -274,7 +262,7 @@ func TestStop_OK(t *testing.T) { } func TestInitDataFromContract_OK(t *testing.T) { - endpoint := "ws://127.0.0.1" + testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -304,7 +292,7 @@ func TestInitDataFromContract_OK(t *testing.T) { func TestWeb3Service_BadReader(t *testing.T) { hook := logTest.NewGlobal() - endpoint := "ws://127.0.0.1" + testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -367,7 +355,7 @@ func TestStatus(t *testing.T) { func TestHandlePanic_OK(t *testing.T) { hook := logTest.NewGlobal() - endpoint := "ws://127.0.0.1" + web3Service, err := NewWeb3Service(context.Background(), &Web3ServiceConfig{ Endpoint: endpoint, BlockFetcher: nil, // nil blockFetcher would panic if cached value not used diff --git a/beacon-chain/rpc/beacon_server_test.go b/beacon-chain/rpc/beacon_server_test.go index 52c428969ebc..67166e519e24 100644 --- a/beacon-chain/rpc/beacon_server_test.go +++ b/beacon-chain/rpc/beacon_server_test.go @@ -24,6 +24,8 @@ import ( logTest "github.com/sirupsen/logrus/hooks/test" ) +var closedContext = "context closed" + type faultyPOWChainService struct { chainStartFeed *event.Feed hashesByHeight map[int][]byte @@ -129,8 +131,7 @@ func TestWaitForChainStart_ContextClosed(t *testing.T) { defer ctrl.Finish() mockStream := internal.NewMockBeaconService_WaitForChainStartServer(ctrl) go func(tt *testing.T) { - want := "context closed" - if err := beaconServer.WaitForChainStart(&ptypes.Empty{}, mockStream); !strings.Contains(err.Error(), want) { + if err := beaconServer.WaitForChainStart(&ptypes.Empty{}, mockStream); !strings.Contains(err.Error(), closedContext) { tt.Errorf("Could not call RPC method: %v", err) } <-exitRoutine diff --git a/beacon-chain/rpc/validator_server.go b/beacon-chain/rpc/validator_server.go index 39fce0986d8b..7830d4b37548 100644 --- a/beacon-chain/rpc/validator_server.go +++ b/beacon-chain/rpc/validator_server.go @@ -109,7 +109,7 @@ func (vs *ValidatorServer) ValidatorPerformance( // 1.) The list of validators in the committee. // 2.) The shard to which the committee is assigned. // 3.) The slot at which the committee is assigned. -// 4.) The bool signalling if the validator is expected to propose a block at the assigned slot. +// 4.) The bool signaling if the validator is expected to propose a block at the assigned slot. func (vs *ValidatorServer) CommitteeAssignment( ctx context.Context, req *pb.CommitteeAssignmentsRequest) (*pb.CommitteeAssignmentResponse, error) { diff --git a/beacon-chain/sync/initial-sync/helpers.go b/beacon-chain/sync/initial-sync/helpers.go index 3e2744b10663..74681d884583 100644 --- a/beacon-chain/sync/initial-sync/helpers.go +++ b/beacon-chain/sync/initial-sync/helpers.go @@ -15,6 +15,8 @@ import ( "go.opencensus.io/trace" ) +const noMsgData = "message contains no data" + func (s *InitialSync) checkBlockValidity(ctx context.Context, block *pb.BeaconBlock) error { ctx, span := trace.StartSpan(ctx, "beacon-chain.sync.initial-sync.checkBlockValidity") defer span.End() @@ -41,7 +43,7 @@ func (s *InitialSync) doesParentExist(block *pb.BeaconBlock) bool { func safelyHandleMessage(fn func(p2p.Message), msg p2p.Message) { defer func() { if r := recover(); r != nil { - printedMsg := "message contains no data" + printedMsg := noMsgData if msg.Data != nil { printedMsg = proto.MarshalTextString(msg.Data) } diff --git a/beacon-chain/sync/regular_sync_test.go b/beacon-chain/sync/regular_sync_test.go index 200077a9fe51..658b26ecc7fd 100644 --- a/beacon-chain/sync/regular_sync_test.go +++ b/beacon-chain/sync/regular_sync_test.go @@ -120,7 +120,7 @@ func (ma *mockAttestationService) IncomingAttestationFeed() *event.Feed { return new(event.Feed) } -func setupService(t *testing.T, db *db.BeaconDB) *RegularSync { +func setupService(db *db.BeaconDB) *RegularSync { cfg := &RegularSyncConfig{ BlockAnnounceBufferSize: 0, BlockBufferSize: 0, @@ -178,7 +178,7 @@ func TestProcessBlock_OK(t *testing.T) { } } genesisTime := uint64(time.Now().Unix()) - deposits, _ := setupInitialDeposits(t, 10) + deposits, _ := setupInitialDeposits(t) if err := db.InitializeState(context.Background(), genesisTime, deposits, &pb.Eth1Data{}); err != nil { t.Fatalf("Failed to initialize state: %v", err) } @@ -253,7 +253,7 @@ func TestProcessBlock_MultipleBlocksProcessedOK(t *testing.T) { } } genesisTime := uint64(time.Now().Unix()) - deposits, _ := setupInitialDeposits(t, 10) + deposits, _ := setupInitialDeposits(t) if err := db.InitializeState(context.Background(), genesisTime, deposits, &pb.Eth1Data{}); err != nil { t.Fatal(err) } @@ -347,7 +347,7 @@ func TestBlockRequest_InvalidMsg(t *testing.T) { db := internal.SetupDB(t) defer internal.TeardownDB(t, db) - ss := setupService(t, db) + ss := setupService(db) malformedRequest := &pb.BeaconBlockAnnounce{ Hash: []byte{'t', 'e', 's', 't'}, @@ -370,7 +370,7 @@ func TestBlockRequest_OK(t *testing.T) { db := internal.SetupDB(t) defer internal.TeardownDB(t, db) - ss := setupService(t, db) + ss := setupService(db) request1 := &pb.BeaconBlockRequestBySlotNumber{ SlotNumber: 20, @@ -670,10 +670,10 @@ func TestHandleStateReq_NOState(t *testing.T) { db := internal.SetupDB(t) defer internal.TeardownDB(t, db) - ss := setupService(t, db) + ss := setupService(db) genesisTime := uint64(time.Now().Unix()) - deposits, _ := setupInitialDeposits(t, 10) + deposits, _ := setupInitialDeposits(t) if err := db.InitializeState(context.Background(), genesisTime, deposits, &pb.Eth1Data{}); err != nil { t.Fatalf("Failed to initialize state: %v", err) } @@ -722,7 +722,7 @@ func TestHandleStateReq_OK(t *testing.T) { t.Fatalf("could not hash beacon state: %v", err) } - ss := setupService(t, db) + ss := setupService(db) request1 := &pb.BeaconStateRequest{ FinalizedStateRootHash32S: stateRoot[:], diff --git a/beacon-chain/sync/service_test.go b/beacon-chain/sync/service_test.go index 631ed9b8725d..e020fbcd3e38 100644 --- a/beacon-chain/sync/service_test.go +++ b/beacon-chain/sync/service_test.go @@ -45,9 +45,10 @@ func initializeTestSyncService(ctx context.Context, cfg *Config, synced bool) *S return services } -func setupInitialDeposits(t *testing.T, numDeposits int) ([]*pb.Deposit, []*bls.SecretKey) { - privKeys := make([]*bls.SecretKey, numDeposits) - deposits := make([]*pb.Deposit, numDeposits) +func setupInitialDeposits(t *testing.T) ([]*pb.Deposit, []*bls.SecretKey) { + numOfDeposits := 10 + privKeys := make([]*bls.SecretKey, numOfDeposits) + deposits := make([]*pb.Deposit, numOfDeposits) for i := 0; i < len(deposits); i++ { priv, err := bls.RandKey(rand.Reader) if err != nil { @@ -71,7 +72,7 @@ func setupTestSyncService(t *testing.T, synced bool) (*Service, *db.BeaconDB) { db := internal.SetupDB(t) unixTime := uint64(time.Now().Unix()) - deposits, _ := setupInitialDeposits(t, 10) + deposits, _ := setupInitialDeposits(t) if err := db.InitializeState(context.Background(), unixTime, deposits, &pb.Eth1Data{}); err != nil { t.Fatalf("Failed to initialize state: %v", err) } diff --git a/shared/messagehandler/messagehandler.go b/shared/messagehandler/messagehandler.go index 90ceeb9d02f3..250cb8e468b2 100644 --- a/shared/messagehandler/messagehandler.go +++ b/shared/messagehandler/messagehandler.go @@ -10,6 +10,8 @@ import ( "go.opencensus.io/trace" ) +const noMsgData = "message contains no data" + var log = logrus.WithField("prefix", "message-handler") // SafelyHandleMessage will recover and log any panic that occurs from the @@ -17,7 +19,7 @@ var log = logrus.WithField("prefix", "message-handler") func SafelyHandleMessage(ctx context.Context, fn func(ctx context.Context, message proto.Message) error, msg proto.Message) { defer func() { if r := recover(); r != nil { - printedMsg := "message contains no data" + printedMsg := noMsgData if msg != nil { printedMsg = proto.MarshalTextString(msg) } diff --git a/shared/p2p/service_norace_test.go b/shared/p2p/service_norace_test.go index 47e591b6ac21..f430f45bc1c4 100644 --- a/shared/p2p/service_norace_test.go +++ b/shared/p2p/service_norace_test.go @@ -28,7 +28,7 @@ func TestLifecycle(t *testing.T) { t.Errorf("incorrect log. wanted: %s. got: %v", want, msg) } - // The context should have been cancelled. + // The context should have been canceled. if s.ctx.Err() == nil { t.Error("Context was not cancelled") } diff --git a/shared/p2p/service_test.go b/shared/p2p/service_test.go index 60091f8058ef..3595be614870 100644 --- a/shared/p2p/service_test.go +++ b/shared/p2p/service_test.go @@ -32,6 +32,9 @@ var _ = shared.Service(&Server{}) var _ = Broadcaster(&Server{}) var _ = Sender(&Server{}) +const bar = "bar" +const testTopic = "test_topic" + func init() { logrus.SetLevel(logrus.DebugLevel) } @@ -84,7 +87,7 @@ func TestBroadcast_OK(t *testing.T) { func TestEmit_OK(t *testing.T) { s, _ := NewServer(&ServerConfig{}) - p := &testpb.TestMessage{Foo: "bar"} + p := &testpb.TestMessage{Foo: bar} ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -302,8 +305,8 @@ func TestRegisterTopic_WithoutAdapters(t *testing.T) { if err != nil { t.Fatalf("Failed to create new server: %v", err) } - topic := "test_topic" - testMessage := &testpb.TestMessage{Foo: "bar"} + topic := testTopic + testMessage := &testpb.TestMessage{Foo: bar} s.RegisterTopic(topic, testMessage) @@ -316,8 +319,8 @@ func TestRegisterTopic_WithoutAdapters(t *testing.T) { defer close(wait) msg := <-ch tmsg := msg.Data.(*testpb.TestMessage) - if tmsg.Foo != "bar" { - t.Errorf("Expected test message Foo: \"bar\". Got: %v", tmsg) + if tmsg.Foo != bar { + t.Errorf("Expected test message foo:\"bar\". Got: %v", tmsg) } }() @@ -338,8 +341,8 @@ func TestRegisterTopic_WithAdapters(t *testing.T) { if err != nil { t.Fatalf("Failed to create new server: %v", err) } - topic := "test_topic" - testMessage := &testpb.TestMessage{Foo: "bar"} + topic := testTopic + testMessage := &testpb.TestMessage{Foo: bar} i := 0 var testAdapter Adapter = func(next Handler) Handler { @@ -368,7 +371,7 @@ func TestRegisterTopic_WithAdapters(t *testing.T) { defer close(wait) msg := <-ch tmsg := msg.Data.(*testpb.TestMessage) - if tmsg.Foo != "bar" { + if tmsg.Foo != bar { t.Errorf("Expected test message Foo: \"bar\". Got: %v", tmsg) } }() @@ -395,8 +398,8 @@ func TestRegisterTopic_HandlesPanic(t *testing.T) { if err != nil { t.Fatalf("Failed to create new server: %v", err) } - topic := "test_topic" - testMessage := &testpb.TestMessage{Foo: "bar"} + topic := testTopic + testMessage := &testpb.TestMessage{Foo: bar} var panicAdapter Adapter = func(next Handler) Handler { return func(msg Message) { diff --git a/validator/client/runner.go b/validator/client/runner.go index fda25f50c914..5ac9f05ea70c 100644 --- a/validator/client/runner.go +++ b/validator/client/runner.go @@ -28,7 +28,7 @@ type Validator interface { } // Run the main validator routine. This routine exits if the context is -// cancelled. +// canceled. // // Order of operations: // 1 - Initialize validator data diff --git a/validator/client/service_test.go b/validator/client/service_test.go index 201ef3d23bf2..f920bdfc336f 100644 --- a/validator/client/service_test.go +++ b/validator/client/service_test.go @@ -69,7 +69,7 @@ func TestStop_CancelsContext(t *testing.T) { func TestLifecycle(t *testing.T) { hook := logTest.NewGlobal() - // Use cancelled context so that the run function exits immediately.. + // Use canceled context so that the run function exits immediately.. ctx, cancel := context.WithCancel(context.Background()) cancel() validatorService := &ValidatorService{ @@ -88,7 +88,7 @@ func TestLifecycle(t *testing.T) { func TestLifecycle_Insecure(t *testing.T) { hook := logTest.NewGlobal() - // Use cancelled context so that the run function exits immediately. + // Use canceled context so that the run function exits immediately. ctx, cancel := context.WithCancel(context.Background()) cancel() validatorService := &ValidatorService{ diff --git a/validator/client/validator_metrics.go b/validator/client/validator_metrics.go index a88e1426816c..9eee658c2b57 100644 --- a/validator/client/validator_metrics.go +++ b/validator/client/validator_metrics.go @@ -47,7 +47,7 @@ func (v *validator) LogValidatorGainsAndLosses(ctx context.Context, slot uint64) log.WithFields(logrus.Fields{ "totalValidators": resp.TotalValidators, "numActiveValidators": resp.TotalActiveValidators, - }).Infof("Validator registry information") + }).Info("Validator registry information") log.Info("Generating validator performance report from the previous epoch...") avgBalance := resp.AverageValidatorBalance / float32(params.BeaconConfig().GweiPerEth) log.WithField( @@ -66,7 +66,7 @@ func (v *validator) LogValidatorGainsAndLosses(ctx context.Context, slot uint64) "delta": fmt.Sprintf("%f", newBalance-prevBalance), "percentChange": fmt.Sprintf("%.2f%%", percentNet*100), "pubKey": tpk, - }).Infof("Net gains/losses in eth") + }).Info("Net gains/losses in eth") } totalPrevBalance += resp.Balance } diff --git a/validator/client/validator_test.go b/validator/client/validator_test.go index d9e8440ada04..29a6a571a9a9 100644 --- a/validator/client/validator_test.go +++ b/validator/client/validator_test.go @@ -28,6 +28,8 @@ func init() { var _ = Validator(&validator{}) +const cancelledCtx = "context has been canceled" + func publicKeys(keys map[string]*keystore.Key) [][]byte { pks := make([][]byte, 0, len(keys)) for _, value := range keys { @@ -94,7 +96,7 @@ func TestWaitForChainStart_ContextCanceled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() err := v.WaitForChainStart(ctx) - want := "context has been canceled" + want := cancelledCtx if !strings.Contains(err.Error(), want) { t.Errorf("Expected %v, received %v", want, err) } @@ -174,7 +176,7 @@ func TestWaitActivation_ContextCanceled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() err := v.WaitForActivation(ctx) - want := "context has been canceled" + want := cancelledCtx if !strings.Contains(err.Error(), want) { t.Errorf("Expected %v, received %v", want, err) } From 81c8b130c6fbcafeb64f846f177cec2059f20de3 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Fri, 26 Apr 2019 10:18:43 -0500 Subject: [PATCH 30/54] Only Sync With the Peer With the Highest Observed Slot (#2280) * only accept the highest finalized slot from peers * use best peer for state requests * peer id * rem old tests * req peers tests pass * warn level support for peer not having the same protocol * pretty * best peer * comments * req peers done * tests passing * fully functional * enforce receiving from the best peer * comments * lint fixes --- beacon-chain/sync/BUILD.bazel | 3 +- beacon-chain/sync/initial-sync/BUILD.bazel | 1 + beacon-chain/sync/initial-sync/metrics.go | 4 - beacon-chain/sync/initial-sync/service.go | 30 +- .../sync/initial-sync/service_test.go | 1 - beacon-chain/sync/initial-sync/sync_blocks.go | 18 +- beacon-chain/sync/initial-sync/sync_state.go | 2 +- beacon-chain/sync/querier.go | 32 +- beacon-chain/sync/querier_test.go | 2 +- beacon-chain/sync/regular_sync_test.go | 1 - beacon-chain/sync/service.go | 1 + beacon-chain/sync/simulated_sync_test.go | 290 ------------------ shared/p2p/service.go | 2 +- 13 files changed, 42 insertions(+), 345 deletions(-) delete mode 100644 beacon-chain/sync/simulated_sync_test.go diff --git a/beacon-chain/sync/BUILD.bazel b/beacon-chain/sync/BUILD.bazel index 27ada6fce953..5a84bbdf74b8 100644 --- a/beacon-chain/sync/BUILD.bazel +++ b/beacon-chain/sync/BUILD.bazel @@ -25,6 +25,7 @@ go_library( "//shared/params:go_default_library", "@com_github_ethereum_go_ethereum//common:go_default_library", "@com_github_gogo_protobuf//proto:go_default_library", + "@com_github_libp2p_go_libp2p_peer//:go_default_library", "@com_github_prometheus_client_golang//prometheus:go_default_library", "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", @@ -39,11 +40,9 @@ go_test( "receive_block_test.go", "regular_sync_test.go", "service_test.go", - "simulated_sync_test.go", ], embed = [":go_default_library"], deps = [ - "//beacon-chain/chaintest/backend:go_default_library", "//beacon-chain/core/helpers:go_default_library", "//beacon-chain/db:go_default_library", "//beacon-chain/internal:go_default_library", diff --git a/beacon-chain/sync/initial-sync/BUILD.bazel b/beacon-chain/sync/initial-sync/BUILD.bazel index a40a4ace373a..f87121bf0f20 100644 --- a/beacon-chain/sync/initial-sync/BUILD.bazel +++ b/beacon-chain/sync/initial-sync/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "//shared/params:go_default_library", "@com_github_ethereum_go_ethereum//common:go_default_library", "@com_github_gogo_protobuf//proto:go_default_library", + "@com_github_libp2p_go_libp2p_peer//:go_default_library", "@com_github_prometheus_client_golang//prometheus:go_default_library", "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", diff --git a/beacon-chain/sync/initial-sync/metrics.go b/beacon-chain/sync/initial-sync/metrics.go index ebea942d8f0d..c3052c93c266 100644 --- a/beacon-chain/sync/initial-sync/metrics.go +++ b/beacon-chain/sync/initial-sync/metrics.go @@ -19,10 +19,6 @@ var ( Name: "initsync_received_blocks", Help: "The number of received blocks", }) - recBlockAnnounce = promauto.NewCounter(prometheus.CounterOpts{ - Name: "initsync_received_block_announce", - Help: "The number of received block announce", - }) stateReq = promauto.NewCounter(prometheus.CounterOpts{ Name: "initsync_state_req", Help: "The number of sent state requests", diff --git a/beacon-chain/sync/initial-sync/service.go b/beacon-chain/sync/initial-sync/service.go index a528cc6e1e53..9c7e1464877f 100644 --- a/beacon-chain/sync/initial-sync/service.go +++ b/beacon-chain/sync/initial-sync/service.go @@ -17,14 +17,14 @@ import ( "sync" "time" - "github.com/prysmaticlabs/prysm/shared/hashutil" - "github.com/ethereum/go-ethereum/common" "github.com/gogo/protobuf/proto" + peer "github.com/libp2p/go-libp2p-peer" "github.com/prysmaticlabs/prysm/beacon-chain/blockchain" "github.com/prysmaticlabs/prysm/beacon-chain/db" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/event" + "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/p2p" "github.com/prysmaticlabs/prysm/shared/params" "github.com/sirupsen/logrus" @@ -110,6 +110,7 @@ type InitialSync struct { finalizedStateRoot [32]byte mutex *sync.Mutex nodeIsSynced bool + bestPeer peer.ID } // NewInitialSyncService constructs a new InitialSyncService. @@ -155,7 +156,6 @@ func (s *InitialSync) Start() { } s.currentSlot = cHead.Slot go s.run() - go s.listenForNewBlocks() go s.checkInMemoryBlocks() } @@ -181,6 +181,11 @@ func (s *InitialSync) InitializeFinalizedStateRoot(root [32]byte) { s.finalizedStateRoot = root } +// InitializeBestPeer sets the peer ID of the highest observed peer. +func (s *InitialSync) InitializeBestPeer(p peer.ID) { + s.bestPeer = p +} + // HighestObservedSlot returns the highest observed slot. func (s *InitialSync) HighestObservedSlot() uint64 { return s.highestObservedSlot @@ -274,24 +279,6 @@ func (s *InitialSync) checkInMemoryBlocks() { } } -// listenForNewBlocks listens for block announcements beyond the canonical head slot that may -// be received during initial sync - we must process these blocks to catch up with peers. -func (s *InitialSync) listenForNewBlocks() { - blockAnnounceSub := s.p2p.Subscribe(&pb.BeaconBlockAnnounce{}, s.blockAnnounceBuf) - defer func() { - blockAnnounceSub.Unsubscribe() - close(s.blockAnnounceBuf) - }() - for { - select { - case <-s.ctx.Done(): - return - case msg := <-s.blockAnnounceBuf: - safelyHandleMessage(s.processBlockAnnounce, msg) - } - } -} - // run is the main goroutine for the initial sync service. // delayChan is explicitly passed into this function to facilitate tests that don't require a timeout. // It is assumed that the goroutine `run` is only called once per instance. @@ -308,6 +295,7 @@ func (s *InitialSync) run() { close(s.stateBuf) }() + // We send out a state request to all peers. if err := s.requestStateFromPeer(s.ctx, s.finalizedStateRoot); err != nil { log.Errorf("Could not request state from peer %v", err) } diff --git a/beacon-chain/sync/initial-sync/service_test.go b/beacon-chain/sync/initial-sync/service_test.go index d61a61abe1ce..38308e2d4341 100644 --- a/beacon-chain/sync/initial-sync/service_test.go +++ b/beacon-chain/sync/initial-sync/service_test.go @@ -136,7 +136,6 @@ func TestSavingBlock_InSync(t *testing.T) { go func() { ss.run() - ss.listenForNewBlocks() exitRoutine <- true }() diff --git a/beacon-chain/sync/initial-sync/sync_blocks.go b/beacon-chain/sync/initial-sync/sync_blocks.go index 84b5eeb97c6d..1bae9e94ca9e 100644 --- a/beacon-chain/sync/initial-sync/sync_blocks.go +++ b/beacon-chain/sync/initial-sync/sync_blocks.go @@ -14,18 +14,6 @@ import ( "go.opencensus.io/trace" ) -func (s *InitialSync) processBlockAnnounce(msg p2p.Message) { - _, span := trace.StartSpan(msg.Ctx, "beacon-chain.sync.initial-sync.processBlockAnnounce") - defer span.End() - data := msg.Data.(*pb.BeaconBlockAnnounce) - recBlockAnnounce.Inc() - - if s.stateReceived && data.SlotNumber > s.highestObservedSlot { - s.requestBatchedBlocks(s.lastRequestedSlot, data.SlotNumber) - s.lastRequestedSlot = data.SlotNumber - } -} - // processBlock is the main method that validates each block which is received // for initial sync. It checks if the blocks are valid and then will continue to // process and save it into the db. @@ -93,6 +81,12 @@ func (s *InitialSync) processBatchedBlocks(msg p2p.Message) { // Do not process empty responses. return } + if msg.Peer != s.bestPeer { + // Only process batch block responses that come from the best peer + // we originally synced with. + log.WithField("peerID", msg.Peer.Pretty()).Debug("Received batch blocks from a different peer") + return + } log.Debug("Processing batched block response") for _, block := range batchedBlocks { diff --git a/beacon-chain/sync/initial-sync/sync_state.go b/beacon-chain/sync/initial-sync/sync_state.go index 9e82102134fa..09f4d395a044 100644 --- a/beacon-chain/sync/initial-sync/sync_state.go +++ b/beacon-chain/sync/initial-sync/sync_state.go @@ -102,5 +102,5 @@ func (s *InitialSync) requestStateFromPeer(ctx context.Context, lastFinalizedRoo stateReq.Inc() return s.p2p.Send(ctx, &pb.BeaconStateRequest{ FinalizedStateRootHash32S: lastFinalizedRoot[:], - }, p2p.AnyPeer) + }, s.bestPeer) } diff --git a/beacon-chain/sync/querier.go b/beacon-chain/sync/querier.go index 6eba811be7cc..7cd857a1169a 100644 --- a/beacon-chain/sync/querier.go +++ b/beacon-chain/sync/querier.go @@ -6,6 +6,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + peer "github.com/libp2p/go-libp2p-peer" "github.com/prysmaticlabs/prysm/beacon-chain/db" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/bytesutil" @@ -57,6 +58,7 @@ type Querier struct { powchain powChainService chainStarted bool atGenesis bool + bestPeer peer.ID } // NewQuerierService constructs a new Sync Querier Service. @@ -151,8 +153,7 @@ func (q *Querier) run() { ticker.Stop() }() - q.RequestLatestHead() - + timeout := time.After(5 * time.Second) for { select { case <-q.ctx.Done(): @@ -160,25 +161,34 @@ func (q *Querier) run() { return case <-ticker.C: q.RequestLatestHead() - case msg := <-q.responseBuf: - response := msg.Data.(*pb.ChainHeadResponse) + case <-timeout: + queryLog.WithField("peerID", q.bestPeer.Pretty()).Info("Peer with highest canonical head") queryLog.Infof( "Latest chain head is at slot: %d and state root: %#x", - response.CanonicalSlot-params.BeaconConfig().GenesisSlot, response.CanonicalStateRootHash32, + q.currentHeadSlot-params.BeaconConfig().GenesisSlot, q.currentStateRoot, ) - q.currentHeadSlot = response.CanonicalSlot - q.currentStateRoot = response.CanonicalStateRootHash32 - q.currentFinalizedStateRoot = bytesutil.ToBytes32(response.FinalizedStateRootHash32S) - ticker.Stop() responseSub.Unsubscribe() q.cancel() + case msg := <-q.responseBuf: + response := msg.Data.(*pb.ChainHeadResponse) + queryLog.WithFields(logrus.Fields{ + "peerID": msg.Peer.Pretty(), + "highestSlot": response.CanonicalSlot - params.BeaconConfig().GenesisSlot, + }).Info("Received chain head from peer") + if response.CanonicalSlot > q.currentHeadSlot { + q.currentHeadSlot = response.CanonicalSlot + q.bestPeer = msg.Peer + q.currentHeadSlot = response.CanonicalSlot + q.currentStateRoot = response.CanonicalStateRootHash32 + q.currentFinalizedStateRoot = bytesutil.ToBytes32(response.FinalizedStateRootHash32S) + } } } } -// RequestLatestHead broadcasts out a request for all -// the latest chain heads from the node's peers. +// RequestLatestHead broadcasts a request for +// the latest chain head slot and state root to a peer. func (q *Querier) RequestLatestHead() { request := &pb.ChainHeadRequest{} q.p2p.Broadcast(context.Background(), request) diff --git a/beacon-chain/sync/querier_test.go b/beacon-chain/sync/querier_test.go index 3dc8d9ba6107..a406c0150afd 100644 --- a/beacon-chain/sync/querier_test.go +++ b/beacon-chain/sync/querier_test.go @@ -146,7 +146,7 @@ func TestQuerier_ChainReqResponse(t *testing.T) { }() response := &pb.ChainHeadResponse{ - CanonicalSlot: 0, + CanonicalSlot: 1, CanonicalStateRootHash32: []byte{'a', 'b'}, } diff --git a/beacon-chain/sync/regular_sync_test.go b/beacon-chain/sync/regular_sync_test.go index 658b26ecc7fd..83ec3ba16000 100644 --- a/beacon-chain/sync/regular_sync_test.go +++ b/beacon-chain/sync/regular_sync_test.go @@ -48,7 +48,6 @@ func (mp *mockP2P) Send(ctx context.Context, msg proto.Message, peerID peer.ID) } type mockChainService struct { - bFeed *event.Feed sFeed *event.Feed cFeed *event.Feed db *db.BeaconDB diff --git a/beacon-chain/sync/service.go b/beacon-chain/sync/service.go index 14c8d5529a18..2b3082402650 100644 --- a/beacon-chain/sync/service.go +++ b/beacon-chain/sync/service.go @@ -129,6 +129,7 @@ func (ss *Service) run() { // Sets the highest observed slot from querier. ss.InitialSync.InitializeObservedSlot(ss.Querier.currentHeadSlot) + ss.InitialSync.InitializeBestPeer(ss.Querier.bestPeer) ss.InitialSync.InitializeObservedStateRoot(bytesutil.ToBytes32(ss.Querier.currentStateRoot)) // Sets the state root of the highest observed slot. ss.InitialSync.InitializeFinalizedStateRoot(ss.Querier.currentFinalizedStateRoot) diff --git a/beacon-chain/sync/simulated_sync_test.go b/beacon-chain/sync/simulated_sync_test.go deleted file mode 100644 index 1ca82a10b77e..000000000000 --- a/beacon-chain/sync/simulated_sync_test.go +++ /dev/null @@ -1,290 +0,0 @@ -package sync - -import ( - "context" - "reflect" - "sync" - "testing" - "time" - - "github.com/gogo/protobuf/proto" - peer "github.com/libp2p/go-libp2p-peer" - "github.com/prysmaticlabs/prysm/beacon-chain/chaintest/backend" - "github.com/prysmaticlabs/prysm/beacon-chain/db" - pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" - "github.com/prysmaticlabs/prysm/shared/bls" - "github.com/prysmaticlabs/prysm/shared/event" - "github.com/prysmaticlabs/prysm/shared/hashutil" - "github.com/prysmaticlabs/prysm/shared/p2p" - "github.com/prysmaticlabs/prysm/shared/params" -) - -type simulatedP2P struct { - subsChannels map[reflect.Type]*event.Feed - mutex *sync.RWMutex - ctx context.Context -} - -func (sim *simulatedP2P) Subscribe(msg proto.Message, channel chan p2p.Message) event.Subscription { - sim.mutex.Lock() - defer sim.mutex.Unlock() - - protoType := reflect.TypeOf(msg) - - feed, ok := sim.subsChannels[protoType] - if !ok { - nFeed := new(event.Feed) - sim.subsChannels[protoType] = nFeed - return nFeed.Subscribe(channel) - } - return feed.Subscribe(channel) -} - -func (sim *simulatedP2P) Broadcast(_ context.Context, msg proto.Message) { - sim.mutex.Lock() - defer sim.mutex.Unlock() - - protoType := reflect.TypeOf(msg) - - feed, ok := sim.subsChannels[protoType] - if !ok { - return - } - - feed.Send(p2p.Message{Ctx: sim.ctx, Data: msg}) -} - -func (sim *simulatedP2P) Send(ctx context.Context, msg proto.Message, peerID peer.ID) error { - sim.mutex.Lock() - defer sim.mutex.Unlock() - - protoType := reflect.TypeOf(msg) - - feed, ok := sim.subsChannels[protoType] - if !ok { - return nil - } - - feed.Send(p2p.Message{Ctx: sim.ctx, Data: msg}) - return nil -} - -func setupSimBackendAndDB(t *testing.T) (*backend.SimulatedBackend, *db.BeaconDB, []*bls.SecretKey) { - ctx := context.Background() - - bd, err := backend.NewSimulatedBackend() - if err != nil { - t.Fatalf("Could not set up simulated backend %v", err) - } - - privKeys, err := bd.SetupBackend(100) - if err != nil { - t.Fatalf("Could not set up backend %v", err) - } - - beacondb, err := db.SetupDB() - if err != nil { - t.Fatalf("Could not setup beacon db %v", err) - } - - memBlocks := bd.InMemoryBlocks() - if err := beacondb.SaveBlock(memBlocks[0]); err != nil { - t.Fatalf("Could not save block %v", err) - } - if err := beacondb.SaveJustifiedBlock(memBlocks[0]); err != nil { - t.Fatalf("Could not save block %v", err) - } - if err := beacondb.SaveFinalizedBlock(memBlocks[0]); err != nil { - t.Fatalf("Could not save block %v", err) - } - - state := bd.State() - state.LatestBlock = memBlocks[0] - state.LatestEth1Data = &pb.Eth1Data{ - BlockHash32: []byte{}, - } - - if err := beacondb.SaveState(ctx, state); err != nil { - t.Fatalf("Could not save state %v", err) - } - if err := beacondb.SaveJustifiedState(state); err != nil { - t.Fatalf("Could not save state %v", err) - } - if err := beacondb.SaveFinalizedState(state); err != nil { - t.Fatalf("Could not save state %v", err) - } - - if err := beacondb.UpdateChainHead(ctx, memBlocks[0], state); err != nil { - t.Fatalf("Could not update chain head %v", err) - } - - return bd, beacondb, privKeys -} - -func setUpSyncedService(numOfBlocks int, simP2P *simulatedP2P, t *testing.T) (*Service, *db.BeaconDB, [32]byte) { - bd, beacondb, _ := setupSimBackendAndDB(t) - defer bd.Shutdown() - defer db.TeardownDB(bd.DB()) - ctx := context.Background() - - mockPow := &genesisPowChain{ - feed: new(event.Feed), - } - - mockChain := &mockChainService{ - bFeed: new(event.Feed), - sFeed: new(event.Feed), - cFeed: new(event.Feed), - db: bd.DB(), - } - - cfg := &Config{ - ChainService: mockChain, - BeaconDB: beacondb, - OperationService: &mockOperationService{}, - P2P: simP2P, - PowChainService: mockPow, - } - - ss := NewSyncService(context.Background(), cfg) - - go ss.run() - for !ss.Querier.chainStarted { - mockChain.sFeed.Send(time.Now()) - } - - state, err := beacondb.HeadState(ctx) - if err != nil { - t.Fatal(err) - } - inMemoryBlocks := bd.InMemoryBlocks() - genesisBlock := inMemoryBlocks[0] - stateRoot, err := hashutil.HashProto(state) - if err != nil { - t.Fatal(err) - } - parentRoot, err := hashutil.HashBeaconBlock(genesisBlock) - if err != nil { - t.Fatal(err) - } - - for i := 1; i <= numOfBlocks; i++ { - block := &pb.BeaconBlock{ - Slot: params.BeaconConfig().GenesisSlot + uint64(i), - ParentRootHash32: parentRoot[:], - StateRootHash32: stateRoot[:], - } - state, err = mockChain.ApplyBlockStateTransition(ctx, block, state) - if err != nil { - t.Fatal(err) - } - stateRoot, err = hashutil.HashProto(state) - if err != nil { - t.Fatal(err) - } - parentRoot, err = hashutil.HashBeaconBlock(block) - if err := mockChain.CleanupBlockOperations(ctx, block); err != nil { - t.Fatal(err) - } - if err := beacondb.SaveBlock(block); err != nil { - t.Fatal(err) - } - if err := beacondb.UpdateChainHead(ctx, block, state); err != nil { - t.Fatal(err) - } - } - return ss, beacondb, stateRoot -} - -func setUpUnSyncedService(simP2P *simulatedP2P, stateRoot [32]byte, t *testing.T) (*Service, *db.BeaconDB) { - bd, beacondb, _ := setupSimBackendAndDB(t) - defer bd.Shutdown() - defer db.TeardownDB(bd.DB()) - - mockPow := &afterGenesisPowChain{ - feed: new(event.Feed), - } - - mockChain := &mockChainService{ - bFeed: new(event.Feed), - sFeed: new(event.Feed), - cFeed: new(event.Feed), - db: bd.DB(), - } - - cfg := &Config{ - ChainService: mockChain, - BeaconDB: beacondb, - OperationService: &mockOperationService{}, - P2P: simP2P, - PowChainService: mockPow, - } - - ss := NewSyncService(context.Background(), cfg) - - go ss.run() - ss.Querier.chainStarted = true - ss.Querier.atGenesis = false - - for ss.Querier.currentHeadSlot == 0 { - simP2P.Send(simP2P.ctx, &pb.ChainHeadResponse{ - CanonicalSlot: params.BeaconConfig().GenesisSlot + 12, - CanonicalStateRootHash32: stateRoot[:], - }, "") - } - - return ss, beacondb -} - -func TestSyncing_AFullySyncedNode(t *testing.T) { - numOfBlocks := 12 - ctx := context.Background() - newP2P := &simulatedP2P{ - subsChannels: make(map[reflect.Type]*event.Feed), - mutex: new(sync.RWMutex), - ctx: ctx, - } - - // Sets up a synced service which has its head at the current - // numOfBlocks from genesis. The blocks are generated through - // simulated backend. - ss, syncedDB, stateRoot := setUpSyncedService(numOfBlocks, newP2P, t) - defer ss.Stop() - defer db.TeardownDB(syncedDB) - - // Sets up a sync service which has its current head at genesis. - us, unSyncedDB := setUpUnSyncedService(newP2P, stateRoot, t) - defer us.Stop() - defer db.TeardownDB(unSyncedDB) - - us2, unSyncedDB2 := setUpUnSyncedService(newP2P, stateRoot, t) - defer us2.Stop() - defer db.TeardownDB(unSyncedDB2) - - finalized, err := syncedDB.FinalizedState() - if err != nil { - t.Fatal(err) - } - - newP2P.Send(newP2P.ctx, &pb.BeaconStateResponse{ - FinalizedState: finalized, - }, "") - - timeout := time.After(10 * time.Second) - tick := time.Tick(200 * time.Millisecond) -loop: - for { - select { - case <-timeout: - t.Error("Could not sync in time") - break loop - case <-tick: - _, slot1 := us.InitialSync.NodeIsSynced() - _, slot2 := us2.InitialSync.NodeIsSynced() - if slot1 == uint64(numOfBlocks)+params.BeaconConfig().GenesisSlot || - slot2 == uint64(numOfBlocks)+params.BeaconConfig().GenesisSlot { - break loop - } - } - } -} diff --git a/shared/p2p/service.go b/shared/p2p/service.go index 16743e00a588..18e2d62fb4d4 100644 --- a/shared/p2p/service.go +++ b/shared/p2p/service.go @@ -14,7 +14,7 @@ import ( "github.com/gogo/protobuf/proto" ds "github.com/ipfs/go-datastore" dsync "github.com/ipfs/go-datastore/sync" - libp2p "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p" host "github.com/libp2p/go-libp2p-host" kaddht "github.com/libp2p/go-libp2p-kad-dht" libp2pnet "github.com/libp2p/go-libp2p-net" From 6ddb5fa81a0e26c9b2992432bbce2cf7b0440f1e Mon Sep 17 00:00:00 2001 From: frederickalcantara Date: Fri, 26 Apr 2019 23:56:11 -0400 Subject: [PATCH 31/54] Added flag to disable rewards/penatlty logging (#2319) * Added flag to disable rewards/penatlty logging * Added flag disable log info validator function * Added flag to disable rewards/penatlty logging * Changed value to not have it log when it is on and have it logged when it's off * Added flag to disable rewards/penatlty logging * Built for cli & types * fixing flag issue * Added ctxCli to the validator struct * Accepted change * Fixing conditionals and merge conflicts * Added bracket * fixed the return statement to its proper place * Added validator conditional for logging penalties & rewards * Added conditional for logging penality/reward info * Making conditional command line log refactorable * also part of the last commit * Changed value variable to lowercase * Fixed if conditional for penalty reward validation * Synced with master * Fixed bazel build * Syncing with master * Sync with master * Added true values to logValidator Balances * Changed values from true to false * FIX WIP * Added variables to the validators * Added negation for logValidatorBalances variable The name of the flag is DisablePenaltyRewardLogFlag. Since the name of the var is logValidatorBalances. We are assuming that the variable will have a positive. It makes more sense to negate the disable flag as a value rather than keep it positive. Co-Authored-By: frederickalcantara * fixed password * Remove prevBalance line --- validator/client/service.go | 52 ++++++++++++++------------- validator/client/validator.go | 21 +++++------ validator/client/validator_metrics.go | 16 +++++---- validator/main.go | 1 + validator/node/node.go | 8 +++-- validator/types/flags.go | 5 +++ validator/usage.go | 1 + 7 files changed, 60 insertions(+), 44 deletions(-) diff --git a/validator/client/service.go b/validator/client/service.go index 5d467763c69f..685b275312d7 100644 --- a/validator/client/service.go +++ b/validator/client/service.go @@ -19,22 +19,24 @@ var log = logrus.WithField("prefix", "validator") // ValidatorService represents a service to manage the validator client // routine. type ValidatorService struct { - ctx context.Context - cancel context.CancelFunc - validator Validator - conn *grpc.ClientConn - endpoint string - withCert string - key *keystore.Key - keys map[string]*keystore.Key + ctx context.Context + cancel context.CancelFunc + validator Validator + conn *grpc.ClientConn + endpoint string + withCert string + key *keystore.Key + keys map[string]*keystore.Key + logValidatorBalances bool } // Config for the validator service. type Config struct { - Endpoint string - CertFlag string - KeystorePath string - Password string + Endpoint string + CertFlag string + KeystorePath string + Password string + LogValidatorBalances bool } // NewValidatorService creates a new validator service for the service @@ -55,12 +57,13 @@ func NewValidatorService(ctx context.Context, cfg *Config) (*ValidatorService, e break } return &ValidatorService{ - ctx: ctx, - cancel: cancel, - endpoint: cfg.Endpoint, - withCert: cfg.CertFlag, - keys: keys, - key: key, + ctx: ctx, + cancel: cancel, + endpoint: cfg.Endpoint, + withCert: cfg.CertFlag, + keys: keys, + key: key, + logValidatorBalances: cfg.LogValidatorBalances, }, nil } @@ -93,12 +96,13 @@ func (v *ValidatorService) Start() { log.Info("Successfully started gRPC connection") v.conn = conn v.validator = &validator{ - beaconClient: pb.NewBeaconServiceClient(v.conn), - validatorClient: pb.NewValidatorServiceClient(v.conn), - attesterClient: pb.NewAttesterServiceClient(v.conn), - proposerClient: pb.NewProposerServiceClient(v.conn), - keys: v.keys, - pubkeys: pubkeys, + beaconClient: pb.NewBeaconServiceClient(v.conn), + validatorClient: pb.NewValidatorServiceClient(v.conn), + attesterClient: pb.NewAttesterServiceClient(v.conn), + proposerClient: pb.NewProposerServiceClient(v.conn), + keys: v.keys, + pubkeys: pubkeys, + logValidatorBalances: v.logValidatorBalances, } go run(v.ctx, v.validator) } diff --git a/validator/client/validator.go b/validator/client/validator.go index 54de53308a74..b936baa8aea1 100644 --- a/validator/client/validator.go +++ b/validator/client/validator.go @@ -18,16 +18,17 @@ import ( ) type validator struct { - genesisTime uint64 - ticker *slotutil.SlotTicker - assignments *pb.CommitteeAssignmentResponse - proposerClient pb.ProposerServiceClient - validatorClient pb.ValidatorServiceClient - beaconClient pb.BeaconServiceClient - attesterClient pb.AttesterServiceClient - keys map[string]*keystore.Key - pubkeys [][]byte - prevBalance uint64 + genesisTime uint64 + ticker *slotutil.SlotTicker + assignments *pb.CommitteeAssignmentResponse + proposerClient pb.ProposerServiceClient + validatorClient pb.ValidatorServiceClient + beaconClient pb.BeaconServiceClient + attesterClient pb.AttesterServiceClient + keys map[string]*keystore.Key + pubkeys [][]byte + prevBalance uint64 + logValidatorBalances bool } // Done cleans up the validator. diff --git a/validator/client/validator_metrics.go b/validator/client/validator_metrics.go index 9eee658c2b57..356854c8c17d 100644 --- a/validator/client/validator_metrics.go +++ b/validator/client/validator_metrics.go @@ -60,13 +60,15 @@ func (v *validator) LogValidatorGainsAndLosses(ctx context.Context, slot uint64) if v.prevBalance > 0 { prevBalance := float64(v.prevBalance) / float64(params.BeaconConfig().GweiPerEth) percentNet := (newBalance - prevBalance) / prevBalance - log.WithFields(logrus.Fields{ - "prevBalance": prevBalance, - "newBalance": newBalance, - "delta": fmt.Sprintf("%f", newBalance-prevBalance), - "percentChange": fmt.Sprintf("%.2f%%", percentNet*100), - "pubKey": tpk, - }).Info("Net gains/losses in eth") + if v.logValidatorBalances { + log.WithFields(logrus.Fields{ + "prevBalance": prevBalance, + "newBalance": newBalance, + "delta": fmt.Sprintf("%f", newBalance-prevBalance), + "percentChange": fmt.Sprintf("%.2f%%", percentNet*100), + "pubKey": tpk, + }).Info("Net gains/losses in eth") + } } totalPrevBalance += resp.Balance } diff --git a/validator/main.go b/validator/main.go index cacc0d4ca7f9..fb3a8389c74d 100644 --- a/validator/main.go +++ b/validator/main.go @@ -138,6 +138,7 @@ contract in order to activate the validator client`, types.BeaconRPCProviderFlag, types.KeystorePathFlag, types.PasswordFlag, + types.DisablePenaltyRewardLogFlag, cmd.VerbosityFlag, cmd.DataDirFlag, cmd.EnableTracingFlag, diff --git a/validator/node/node.go b/validator/node/node.go index 249df05b5392..233416dc062b 100644 --- a/validator/node/node.go +++ b/validator/node/node.go @@ -129,10 +129,12 @@ func (s *ValidatorClient) registerPrometheusService(ctx *cli.Context) error { func (s *ValidatorClient) registerClientService(ctx *cli.Context, password string) error { endpoint := ctx.GlobalString(types.BeaconRPCProviderFlag.Name) keystoreDirectory := ctx.GlobalString(types.KeystorePathFlag.Name) + logValidatorBalances := !ctx.GlobalBool(types.DisablePenaltyRewardLogFlag.Name) v, err := client.NewValidatorService(context.Background(), &client.Config{ - Endpoint: endpoint, - KeystorePath: keystoreDirectory, - Password: password, + Endpoint: endpoint, + KeystorePath: keystoreDirectory, + Password: password, + LogValidatorBalances: logValidatorBalances, }) if err != nil { return fmt.Errorf("could not initialize client service: %v", err) diff --git a/validator/types/flags.go b/validator/types/flags.go index 676304122749..84c868b7ba25 100644 --- a/validator/types/flags.go +++ b/validator/types/flags.go @@ -38,6 +38,11 @@ var ( Name: "password", Usage: "string value of the password for your validator private keys", } + // DisablePenaltyRewardLogFlag defines the ability to not log reward/penalty information during deployment + DisablePenaltyRewardLogFlag = cli.BoolFlag{ + Name: "disable-rewards-penalties-logging", + Usage: "Disable reward/penalty logging during cluster deployment", + } ) func homeDir() string { diff --git a/validator/usage.go b/validator/usage.go index b6adda79fe9c..7e86a1d8fae3 100644 --- a/validator/usage.go +++ b/validator/usage.go @@ -71,6 +71,7 @@ var appHelpFlagGroups = []flagGroup{ types.BeaconRPCProviderFlag, types.KeystorePathFlag, types.PasswordFlag, + types.DisablePenaltyRewardLogFlag, }, }, { From 7a04af762124f88ffe1ad33cb7825a5a56dd9f92 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Fri, 26 Apr 2019 21:08:15 -0700 Subject: [PATCH 32/54] Use Withfields for Forked Block (#2394) * use with fields for forked block * spacing --- beacon-chain/sync/receive_block.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/beacon-chain/sync/receive_block.go b/beacon-chain/sync/receive_block.go index f5d93d0718a0..9821e70ce443 100644 --- a/beacon-chain/sync/receive_block.go +++ b/beacon-chain/sync/receive_block.go @@ -10,6 +10,7 @@ import ( "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/p2p" "github.com/prysmaticlabs/prysm/shared/params" + "github.com/sirupsen/logrus" "go.opencensus.io/trace" ) @@ -193,7 +194,10 @@ func (rs *RegularSync) validateAndProcessBlock( } } else { forkedBlock.Inc() - log.Warnf("Received Block from a Forked Chain with root %#x and slot %d", blockRoot, block.Slot) + log.WithFields(logrus.Fields{ + "slot": block.Slot, + "root": fmt.Sprintf("%#x", blockRoot)}, + ).Warn("Received Block from a forked chain") } sentBlocks.Inc() From 210edfc940e8d7ddfdc5e62bf4b53cfc6d7dc6e9 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Sat, 27 Apr 2019 15:08:27 -0400 Subject: [PATCH 33/54] P2P handshake handling (#2306) * proto: illegal wireType 7 :( * set addr to todo for now so somethign gets sent * push latest progress * Add feedback from @raulk. Stream never connects * working handshake handler * add exclusions for relay/bootstrap node * fix tests, still need to add new ones * remove race, fails coverage * Add test for negotiation * gazelle * regen pb * Update shared/p2p/handshake_handler.go Co-Authored-By: prestonvanloon --- beacon-chain/node/BUILD.bazel | 1 + beacon-chain/node/fetch_contract_address.go | 41 +++ beacon-chain/node/node.go | 19 +- beacon-chain/node/p2p_config.go | 19 +- proto/beacon/p2p/v1/messages.pb.go | 293 ++++++++++++++++---- proto/beacon/p2p/v1/messages.proto | 4 + shared/p2p/BUILD.bazel | 5 + shared/p2p/discovery.go | 14 +- shared/p2p/handshake_handler.go | 23 ++ shared/p2p/negotiation.go | 93 +++++++ shared/p2p/negotiation_test.go | 58 ++++ shared/p2p/service.go | 25 +- shared/p2p/service_test.go | 15 +- 13 files changed, 515 insertions(+), 95 deletions(-) create mode 100644 beacon-chain/node/fetch_contract_address.go create mode 100644 shared/p2p/handshake_handler.go create mode 100644 shared/p2p/negotiation.go create mode 100644 shared/p2p/negotiation_test.go diff --git a/beacon-chain/node/BUILD.bazel b/beacon-chain/node/BUILD.bazel index 85a834050317..6ca6abbfb73a 100644 --- a/beacon-chain/node/BUILD.bazel +++ b/beacon-chain/node/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "fetch_contract_address.go", "node.go", "p2p_config.go", ], diff --git a/beacon-chain/node/fetch_contract_address.go b/beacon-chain/node/fetch_contract_address.go new file mode 100644 index 000000000000..c423dad50913 --- /dev/null +++ b/beacon-chain/node/fetch_contract_address.go @@ -0,0 +1,41 @@ +package node + +import ( + "io/ioutil" + "net/http" + "sync" + + "github.com/prysmaticlabs/prysm/shared/params" +) + +var cachedDepositAddress string +var fetchLock sync.Mutex + +// fetchDepositContract from the cluster endpoint. +func fetchDepositContract() (string, error) { + fetchLock.Lock() + defer fetchLock.Unlock() + + if cachedDepositAddress != "" { + return cachedDepositAddress, nil + } + + log.WithField( + "endpoint", + params.BeaconConfig().TestnetContractEndpoint, + ).Info("Fetching testnet cluster address") + resp, err := http.Get(params.BeaconConfig().TestnetContractEndpoint) + if err != nil { + return "", err + } + contractResponse, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + if err := resp.Body.Close(); err != nil { + return "", err + } + + cachedDepositAddress = string(contractResponse) + return cachedDepositAddress, nil +} diff --git a/beacon-chain/node/node.go b/beacon-chain/node/node.go index 056f86b3d499..6fc8c8b08922 100644 --- a/beacon-chain/node/node.go +++ b/beacon-chain/node/node.go @@ -4,8 +4,6 @@ package node import ( "context" "fmt" - "io/ioutil" - "net/http" "os" "os/signal" "path" @@ -248,22 +246,11 @@ func (b *BeaconNode) registerPOWChainService(cliCtx *cli.Context) error { depAddress := cliCtx.GlobalString(utils.DepositContractFlag.Name) if depAddress == "" { - log.Infof("Fetching testnet cluster address from %s...", params.BeaconConfig().TestnetContractEndpoint) - resp, err := http.Get(params.BeaconConfig().TestnetContractEndpoint) + var err error + depAddress, err = fetchDepositContract() if err != nil { - log.Fatalf("Could not get latest deposit contract address: %v", err) + log.WithError(err).Fatal("Cannot fetch deposit contract") } - defer func() { - if err := resp.Body.Close(); err != nil { - log.Fatal(err) - } - }() - - contractResponse, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Fatal(err) - } - depAddress = string(contractResponse) } if !common.IsHexAddress(depAddress) { diff --git a/beacon-chain/node/p2p_config.go b/beacon-chain/node/p2p_config.go index b021b63ace70..e1250684c446 100644 --- a/beacon-chain/node/p2p_config.go +++ b/beacon-chain/node/p2p_config.go @@ -2,6 +2,7 @@ package node import ( "github.com/gogo/protobuf/proto" + "github.com/prysmaticlabs/prysm/beacon-chain/utils" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/cmd" "github.com/prysmaticlabs/prysm/shared/p2p" @@ -27,11 +28,21 @@ var topicMappings = map[pb.Topic]proto.Message{ } func configureP2P(ctx *cli.Context) (*p2p.Server, error) { + contractAddress := ctx.GlobalString(utils.DepositContractFlag.Name) + if contractAddress == "" { + var err error + contractAddress, err = fetchDepositContract() + if err != nil { + return nil, err + } + } + s, err := p2p.NewServer(&p2p.ServerConfig{ - NoDiscovery: ctx.GlobalBool(cmd.NoDiscovery.Name), - BootstrapNodeAddr: ctx.GlobalString(cmd.BootstrapNode.Name), - RelayNodeAddr: ctx.GlobalString(cmd.RelayNode.Name), - Port: ctx.GlobalInt(cmd.P2PPort.Name), + NoDiscovery: ctx.GlobalBool(cmd.NoDiscovery.Name), + BootstrapNodeAddr: ctx.GlobalString(cmd.BootstrapNode.Name), + RelayNodeAddr: ctx.GlobalString(cmd.RelayNode.Name), + Port: ctx.GlobalInt(cmd.P2PPort.Name), + DepositContractAddress: contractAddress, }) if err != nil { return nil, err diff --git a/proto/beacon/p2p/v1/messages.pb.go b/proto/beacon/p2p/v1/messages.pb.go index 84e72de86d1a..2b2bdee3f008 100755 --- a/proto/beacon/p2p/v1/messages.pb.go +++ b/proto/beacon/p2p/v1/messages.pb.go @@ -1435,6 +1435,53 @@ func (m *ExitResponse) GetVoluntaryExit() *VoluntaryExit { return nil } +type Handshake struct { + DepositContractAddress string `protobuf:"bytes,1,opt,name=deposit_contract_address,json=depositContractAddress,proto3" json:"deposit_contract_address,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Handshake) Reset() { *m = Handshake{} } +func (m *Handshake) String() string { return proto.CompactTextString(m) } +func (*Handshake) ProtoMessage() {} +func (*Handshake) Descriptor() ([]byte, []int) { + return fileDescriptor_a1d590cda035b632, []int{27} +} +func (m *Handshake) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Handshake) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Handshake.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Handshake) XXX_Merge(src proto.Message) { + xxx_messageInfo_Handshake.Merge(m, src) +} +func (m *Handshake) XXX_Size() int { + return m.Size() +} +func (m *Handshake) XXX_DiscardUnknown() { + xxx_messageInfo_Handshake.DiscardUnknown(m) +} + +var xxx_messageInfo_Handshake proto.InternalMessageInfo + +func (m *Handshake) GetDepositContractAddress() string { + if m != nil { + return m.DepositContractAddress + } + return "" +} + func init() { proto.RegisterEnum("ethereum.beacon.p2p.v1.Topic", Topic_name, Topic_value) proto.RegisterType((*Envelope)(nil), "ethereum.beacon.p2p.v1.Envelope") @@ -1464,68 +1511,71 @@ func init() { proto.RegisterType((*ExitAnnounce)(nil), "ethereum.beacon.p2p.v1.ExitAnnounce") proto.RegisterType((*ExitRequest)(nil), "ethereum.beacon.p2p.v1.ExitRequest") proto.RegisterType((*ExitResponse)(nil), "ethereum.beacon.p2p.v1.ExitResponse") + proto.RegisterType((*Handshake)(nil), "ethereum.beacon.p2p.v1.Handshake") } func init() { proto.RegisterFile("proto/beacon/p2p/v1/messages.proto", fileDescriptor_a1d590cda035b632) } var fileDescriptor_a1d590cda035b632 = []byte{ - // 884 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x96, 0xd1, 0x72, 0xda, 0x46, - 0x14, 0x86, 0xab, 0xd8, 0x0e, 0xce, 0x01, 0x13, 0xb2, 0x6e, 0x63, 0xec, 0x26, 0xd8, 0x56, 0xea, - 0xa9, 0xdb, 0x99, 0xe0, 0x89, 0x73, 0x95, 0x8b, 0x4e, 0x47, 0xc2, 0x9a, 0x92, 0xc4, 0x15, 0xa9, - 0x80, 0x76, 0x7a, 0xb5, 0x5d, 0xc4, 0xd6, 0x30, 0xc5, 0xbb, 0x5b, 0x76, 0x61, 0xec, 0xde, 0xf7, - 0x19, 0xfa, 0x04, 0x7d, 0x89, 0x3e, 0x41, 0x2f, 0xfb, 0x08, 0x1d, 0x3f, 0x49, 0x47, 0xab, 0x15, - 0x16, 0x20, 0xcb, 0x5c, 0xf4, 0x0e, 0x9d, 0xf3, 0xff, 0xff, 0x9e, 0x6f, 0x39, 0x62, 0x00, 0x5b, - 0x8c, 0xb9, 0xe2, 0x27, 0x3d, 0x4a, 0x42, 0xce, 0x4e, 0xc4, 0xa9, 0x38, 0x99, 0xbe, 0x3a, 0xb9, - 0xa4, 0x52, 0x92, 0x0b, 0x2a, 0xeb, 0xba, 0x89, 0x9e, 0x52, 0x35, 0xa0, 0x63, 0x3a, 0xb9, 0xac, - 0xc7, 0xb2, 0xba, 0x38, 0x15, 0xf5, 0xe9, 0xab, 0xbd, 0xfd, 0x2c, 0xaf, 0xba, 0x16, 0x89, 0xd1, - 0xfe, 0x06, 0x36, 0x3d, 0x36, 0xa5, 0x23, 0x2e, 0x28, 0x3a, 0x84, 0x92, 0x14, 0x84, 0xe1, 0x90, - 0x33, 0x45, 0xaf, 0x54, 0xd5, 0x3a, 0xb0, 0x8e, 0x4b, 0x41, 0x31, 0xaa, 0x35, 0xe2, 0x12, 0xaa, - 0x42, 0x41, 0x90, 0xeb, 0x11, 0x27, 0xfd, 0xea, 0x03, 0xdd, 0x4d, 0x1e, 0xed, 0x77, 0xb0, 0xed, - 0xea, 0x53, 0xdc, 0x11, 0x0f, 0x7f, 0x71, 0x18, 0xe3, 0x13, 0x16, 0x52, 0x84, 0x60, 0x7d, 0x40, - 0xe4, 0xc0, 0x64, 0xe9, 0xcf, 0x68, 0x1f, 0x8a, 0x72, 0xc4, 0x15, 0x66, 0x93, 0xcb, 0x1e, 0x1d, - 0xeb, 0xa0, 0xf5, 0x00, 0xa2, 0x92, 0xaf, 0x2b, 0xf6, 0x31, 0xa0, 0x54, 0x56, 0x40, 0x7f, 0x9d, - 0x50, 0xa9, 0xb2, 0xa2, 0x6c, 0x07, 0x6a, 0xcb, 0x4a, 0xf7, 0xba, 0x3d, 0xcb, 0x5a, 0x3c, 0xcc, - 0x5a, 0x3a, 0xec, 0x0f, 0x6b, 0x6e, 0xf2, 0x80, 0x4a, 0xc1, 0x99, 0xa4, 0xe8, 0x0d, 0x6c, 0xf4, - 0xa2, 0x82, 0xb6, 0x14, 0x4f, 0x5f, 0xd4, 0xb3, 0xaf, 0xb8, 0x9e, 0xf6, 0xc6, 0x0e, 0xe4, 0x41, - 0x91, 0x28, 0x45, 0xa5, 0x22, 0x6a, 0xc8, 0x99, 0x06, 0xcc, 0x09, 0x70, 0x6e, 0xa5, 0x41, 0xda, - 0x67, 0x77, 0x61, 0xd7, 0x25, 0x2a, 0x1c, 0xd0, 0x7e, 0xc6, 0x6d, 0x3c, 0x07, 0x90, 0x8a, 0x8c, - 0x15, 0x8e, 0x50, 0x0c, 0xd6, 0x23, 0x5d, 0x89, 0xe0, 0xd1, 0x2e, 0x6c, 0x52, 0xd6, 0x8f, 0x9b, - 0xf1, 0x05, 0x17, 0x28, 0xeb, 0x47, 0x2d, 0x7b, 0x00, 0x7b, 0x59, 0xb1, 0x06, 0xfb, 0x1d, 0x94, - 0x7b, 0x71, 0x17, 0x6b, 0x18, 0x59, 0xb5, 0x0e, 0xd6, 0x56, 0xe5, 0xdf, 0x32, 0x56, 0xfd, 0x24, - 0x6d, 0x04, 0x95, 0xc6, 0x80, 0x0c, 0x59, 0x93, 0x92, 0xbe, 0x99, 0xdb, 0xfe, 0xcb, 0x82, 0x27, - 0xa9, 0xa2, 0x39, 0xf5, 0x08, 0xca, 0x21, 0x61, 0x9c, 0x0d, 0x43, 0x32, 0x4a, 0x13, 0x6d, 0xcd, - 0xaa, 0x9a, 0xea, 0x2b, 0xf8, 0x34, 0x25, 0x53, 0x44, 0x51, 0x3c, 0xe6, 0x5c, 0xe1, 0x68, 0x17, - 0x5e, 0x9f, 0x9a, 0x95, 0xac, 0xde, 0x7a, 0x22, 0x45, 0xc0, 0xb9, 0x6a, 0xea, 0x3e, 0xfa, 0x1a, - 0x9e, 0xfd, 0x3c, 0x64, 0x64, 0x34, 0xfc, 0x8d, 0xf6, 0x97, 0xed, 0xb2, 0xba, 0xa6, 0xfd, 0xbb, - 0x33, 0xcd, 0x82, 0x5f, 0xda, 0x2f, 0x61, 0x27, 0xc6, 0xd5, 0x9d, 0xa8, 0x9a, 0xb7, 0xe8, 0x76, - 0x37, 0xd9, 0xe3, 0x38, 0xc8, 0x7c, 0x73, 0xf7, 0x4d, 0x61, 0xdd, 0x37, 0x45, 0x98, 0x2c, 0xac, - 0x89, 0x35, 0x77, 0x78, 0x0e, 0x8f, 0x17, 0x72, 0x57, 0x5b, 0xdd, 0x38, 0xa5, 0x3c, 0x7f, 0x9e, - 0xfd, 0x05, 0x6c, 0xa7, 0x16, 0x33, 0x17, 0xf3, 0x18, 0x50, 0x7a, 0x87, 0x73, 0x5e, 0x57, 0x31, - 0x17, 0x3a, 0x9b, 0x3c, 0xeb, 0x47, 0xe2, 0x7f, 0x7a, 0x87, 0xea, 0x50, 0xfd, 0x30, 0xe6, 0x82, - 0x4b, 0x3a, 0x6e, 0x8f, 0x88, 0x1c, 0x0c, 0xd9, 0x45, 0x2e, 0xcb, 0x4b, 0xd8, 0x59, 0xd4, 0xe7, - 0x01, 0xfd, 0x6e, 0x2d, 0xe7, 0xe7, 0x62, 0x75, 0xe1, 0x89, 0x30, 0x7a, 0x2c, 0x8d, 0xc1, 0xc0, - 0x1d, 0xdf, 0x05, 0xb7, 0x74, 0x40, 0x45, 0x2c, 0x54, 0x22, 0xcc, 0xf8, 0x0a, 0x56, 0xc7, 0x5c, - 0xd4, 0xdf, 0x87, 0xb9, 0xac, 0xcf, 0xc7, 0x4c, 0xf4, 0x2b, 0x63, 0x2e, 0x1d, 0x50, 0x59, 0xac, - 0xd8, 0x47, 0xf0, 0xf8, 0x8c, 0x0a, 0x2e, 0x87, 0x2a, 0x97, 0xee, 0x33, 0x28, 0x1b, 0x59, 0x1e, - 0xd4, 0x4f, 0xb3, 0xb0, 0x5c, 0x94, 0x37, 0x50, 0xe8, 0xc7, 0x32, 0x03, 0xb0, 0x7f, 0x17, 0x40, - 0x92, 0x96, 0xe8, 0x6d, 0x1b, 0x4a, 0xde, 0xd5, 0x3d, 0xb3, 0x1e, 0x42, 0x31, 0xd2, 0xe4, 0xbf, - 0x35, 0xa5, 0x58, 0x92, 0x33, 0xe5, 0x39, 0x94, 0xa7, 0x7c, 0x34, 0x61, 0x8a, 0x8c, 0xaf, 0x31, - 0xbd, 0x9a, 0x0d, 0x7b, 0x74, 0xd7, 0xb0, 0xdf, 0x27, 0x6a, 0x1d, 0xbd, 0x35, 0x4d, 0x3f, 0x7e, - 0xf9, 0xe7, 0x1a, 0x6c, 0x74, 0xb8, 0x18, 0x86, 0xa8, 0x08, 0x85, 0xae, 0xff, 0xde, 0x6f, 0xfd, - 0xe0, 0x57, 0x3e, 0x42, 0xbb, 0xf0, 0x89, 0xeb, 0x39, 0x8d, 0x96, 0x8f, 0xdd, 0xf3, 0x56, 0xe3, - 0x3d, 0x76, 0x7c, 0xbf, 0xd5, 0xf5, 0x1b, 0x5e, 0xc5, 0x42, 0x55, 0xf8, 0x78, 0xae, 0x15, 0x78, - 0xdf, 0x75, 0xbd, 0x76, 0xa7, 0xf2, 0x00, 0x7d, 0x0e, 0x2f, 0xb2, 0x3a, 0xd8, 0xfd, 0x11, 0xb7, - 0xcf, 0x5b, 0x1d, 0xec, 0x77, 0xbf, 0x75, 0xbd, 0xa0, 0xb2, 0xb6, 0x94, 0x1e, 0x78, 0xed, 0x0f, - 0x2d, 0xbf, 0xed, 0x55, 0xd6, 0xd1, 0x01, 0x3c, 0x73, 0x9d, 0x4e, 0xa3, 0xe9, 0x9d, 0xe1, 0xcc, - 0x53, 0x36, 0xd0, 0x21, 0x3c, 0xbf, 0x43, 0x61, 0x42, 0x1e, 0xa2, 0xa7, 0x80, 0x1a, 0x4d, 0xe7, - 0xad, 0x8f, 0x9b, 0x9e, 0x73, 0x36, 0xb3, 0x16, 0xd0, 0x0e, 0x6c, 0xcf, 0xd5, 0x8d, 0x61, 0x13, - 0xd5, 0x60, 0xcf, 0x64, 0xb5, 0x3b, 0x4e, 0xc7, 0xc3, 0x4d, 0xa7, 0xdd, 0xbc, 0x65, 0x7e, 0x94, - 0x62, 0x8e, 0xfb, 0x49, 0x24, 0xa4, 0x50, 0x92, 0x8e, 0x09, 0x2d, 0x46, 0x26, 0xa7, 0xd3, 0xf1, - 0xa2, 0xfa, 0xdb, 0x96, 0x7f, 0x1b, 0x57, 0x8a, 0xe6, 0x48, 0x77, 0x92, 0xb4, 0xad, 0x45, 0xcb, - 0x2c, 0xac, 0xec, 0x96, 0xfe, 0xbe, 0xa9, 0x59, 0xff, 0xdc, 0xd4, 0xac, 0x7f, 0x6f, 0x6a, 0x56, - 0xef, 0xa1, 0xfe, 0x4b, 0xf7, 0xfa, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb1, 0xd9, 0xa8, 0x18, - 0x31, 0x0a, 0x00, 0x00, + // 921 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x96, 0xdf, 0x72, 0xdb, 0x44, + 0x14, 0xc6, 0x51, 0x93, 0xd4, 0xc9, 0xb1, 0xe3, 0xba, 0x1b, 0x48, 0x94, 0xd0, 0x3a, 0x89, 0x4a, + 0x86, 0xc0, 0x4c, 0x9d, 0x69, 0x7a, 0x43, 0x2f, 0x18, 0x46, 0x72, 0x34, 0xb8, 0x6d, 0x90, 0x8b, + 0x6c, 0xc3, 0x70, 0xb5, 0xac, 0xa5, 0xa5, 0xf6, 0xd4, 0xd9, 0x15, 0xda, 0xb5, 0x27, 0xe1, 0x9e, + 0x67, 0xe0, 0x09, 0x78, 0x09, 0x9e, 0x80, 0x4b, 0x1e, 0x81, 0xc9, 0x93, 0x30, 0x5a, 0xad, 0x1c, + 0xf9, 0x4f, 0x94, 0x5c, 0x70, 0x17, 0x9d, 0xf3, 0x7d, 0xdf, 0x9e, 0xdf, 0xe6, 0xc8, 0x23, 0xb0, + 0xa2, 0x98, 0x4b, 0x7e, 0xd2, 0xa7, 0x24, 0xe0, 0xec, 0x24, 0x3a, 0x8d, 0x4e, 0x26, 0x2f, 0x4e, + 0x2e, 0xa8, 0x10, 0xe4, 0x3d, 0x15, 0x0d, 0xd5, 0x44, 0xdb, 0x54, 0x0e, 0x68, 0x4c, 0xc7, 0x17, + 0x8d, 0x54, 0xd6, 0x88, 0x4e, 0xa3, 0xc6, 0xe4, 0xc5, 0xde, 0xfe, 0x32, 0xaf, 0xbc, 0x8a, 0x32, + 0xa3, 0xf5, 0x2d, 0xac, 0xbb, 0x6c, 0x42, 0x47, 0x3c, 0xa2, 0xe8, 0x10, 0x2a, 0x22, 0x22, 0x0c, + 0x07, 0x9c, 0x49, 0x7a, 0x29, 0x4d, 0xe3, 0xc0, 0x38, 0xae, 0xf8, 0xe5, 0xa4, 0xd6, 0x4c, 0x4b, + 0xc8, 0x84, 0x52, 0x44, 0xae, 0x46, 0x9c, 0x84, 0xe6, 0x03, 0xd5, 0xcd, 0x1e, 0xad, 0x37, 0xb0, + 0xe5, 0xa8, 0x53, 0x9c, 0x11, 0x0f, 0x3e, 0xd8, 0x8c, 0xf1, 0x31, 0x0b, 0x28, 0x42, 0xb0, 0x3a, + 0x20, 0x62, 0xa0, 0xb3, 0xd4, 0xdf, 0x68, 0x1f, 0xca, 0x62, 0xc4, 0x25, 0x66, 0xe3, 0x8b, 0x3e, + 0x8d, 0x55, 0xd0, 0xaa, 0x0f, 0x49, 0xc9, 0x53, 0x15, 0xeb, 0x18, 0x50, 0x2e, 0xcb, 0xa7, 0xbf, + 0x8e, 0xa9, 0x90, 0xcb, 0xa2, 0x2c, 0x1b, 0xea, 0x8b, 0x4a, 0xe7, 0xaa, 0x33, 0xcd, 0x9a, 0x3f, + 0xcc, 0x58, 0x38, 0xec, 0x0f, 0x63, 0x66, 0x72, 0x9f, 0x8a, 0x88, 0x33, 0x41, 0xd1, 0x2b, 0x58, + 0xeb, 0x27, 0x05, 0x65, 0x29, 0x9f, 0x3e, 0x6b, 0x2c, 0xbf, 0xe2, 0x46, 0xde, 0x9b, 0x3a, 0x90, + 0x0b, 0x65, 0x22, 0x25, 0x15, 0x92, 0xc8, 0x21, 0x67, 0x0a, 0xb0, 0x20, 0xc0, 0xbe, 0x91, 0xfa, + 0x79, 0x9f, 0xd5, 0x83, 0x5d, 0x87, 0xc8, 0x60, 0x40, 0xc3, 0x25, 0xb7, 0xf1, 0x14, 0x40, 0x48, + 0x12, 0x4b, 0x9c, 0xa0, 0x68, 0xac, 0x0d, 0x55, 0x49, 0xe0, 0xd1, 0x2e, 0xac, 0x53, 0x16, 0xa6, + 0xcd, 0xf4, 0x82, 0x4b, 0x94, 0x85, 0x49, 0xcb, 0x1a, 0xc0, 0xde, 0xb2, 0x58, 0x8d, 0xfd, 0x06, + 0xaa, 0xfd, 0xb4, 0x8b, 0x15, 0x8c, 0x30, 0x8d, 0x83, 0x95, 0xfb, 0xf2, 0x6f, 0x6a, 0xab, 0x7a, + 0x12, 0x16, 0x82, 0x5a, 0x73, 0x40, 0x86, 0xac, 0x45, 0x49, 0xa8, 0xe7, 0xb6, 0xfe, 0x32, 0xe0, + 0x71, 0xae, 0xa8, 0x4f, 0x3d, 0x82, 0x6a, 0x40, 0x18, 0x67, 0xc3, 0x80, 0x8c, 0xf2, 0x44, 0x9b, + 0xd3, 0xaa, 0xa2, 0xfa, 0x1a, 0x3e, 0xcd, 0xc9, 0x24, 0x91, 0x14, 0xc7, 0x9c, 0x4b, 0x9c, 0xec, + 0xc2, 0xcb, 0x53, 0xbd, 0x92, 0xe6, 0x8d, 0x27, 0x51, 0xf8, 0x9c, 0xcb, 0x96, 0xea, 0xa3, 0x6f, + 0xe0, 0xc9, 0x2f, 0x43, 0x46, 0x46, 0xc3, 0xdf, 0x68, 0xb8, 0x68, 0x17, 0xe6, 0x8a, 0xf2, 0xef, + 0x4e, 0x35, 0x73, 0x7e, 0x61, 0x3d, 0x87, 0x9d, 0x14, 0x57, 0x75, 0x92, 0x6a, 0xd1, 0xa2, 0x5b, + 0xbd, 0x6c, 0x8f, 0xd3, 0x20, 0xfd, 0x9f, 0xbb, 0x6b, 0x0a, 0xe3, 0xae, 0x29, 0x82, 0x6c, 0x61, + 0x75, 0xac, 0xbe, 0xc3, 0x73, 0x78, 0x34, 0x97, 0x7b, 0xbf, 0xd5, 0x4d, 0x53, 0xaa, 0xb3, 0xe7, + 0x59, 0x5f, 0xc0, 0x56, 0x6e, 0x31, 0x0b, 0x31, 0x8f, 0x01, 0xe5, 0x77, 0xb8, 0xe0, 0x75, 0x8d, + 0x66, 0x42, 0xa7, 0x93, 0x2f, 0xfb, 0x91, 0xf8, 0x9f, 0xde, 0xa1, 0x06, 0x98, 0xef, 0x62, 0x1e, + 0x71, 0x41, 0xe3, 0xce, 0x88, 0x88, 0xc1, 0x90, 0xbd, 0x2f, 0x64, 0x79, 0x0e, 0x3b, 0xf3, 0xfa, + 0x22, 0xa0, 0xdf, 0x8d, 0xc5, 0xfc, 0x42, 0xac, 0x1e, 0x3c, 0x8e, 0xb4, 0x1e, 0x0b, 0x6d, 0xd0, + 0x70, 0xc7, 0xb7, 0xc1, 0x2d, 0x1c, 0x50, 0x8b, 0xe6, 0x2a, 0x09, 0x66, 0x7a, 0x05, 0xf7, 0xc7, + 0x9c, 0xd7, 0xdf, 0x85, 0xb9, 0xa8, 0x2f, 0xc6, 0xcc, 0xf4, 0xf7, 0xc6, 0x5c, 0x38, 0xa0, 0x36, + 0x5f, 0xb1, 0x8e, 0xe0, 0xd1, 0x19, 0x8d, 0xb8, 0x18, 0xca, 0x42, 0xba, 0xcf, 0xa0, 0xaa, 0x65, + 0x45, 0x50, 0x3f, 0x4f, 0xc3, 0x0a, 0x51, 0x5e, 0x41, 0x29, 0x4c, 0x65, 0x1a, 0x60, 0xff, 0x36, + 0x80, 0x2c, 0x2d, 0xd3, 0x5b, 0x16, 0x54, 0xdc, 0xcb, 0x3b, 0x66, 0x3d, 0x84, 0x72, 0xa2, 0x29, + 0x7e, 0x6b, 0x2a, 0xa9, 0xa4, 0x60, 0xca, 0x73, 0xa8, 0x4e, 0xf8, 0x68, 0xcc, 0x24, 0x89, 0xaf, + 0x30, 0xbd, 0x9c, 0x0e, 0x7b, 0x74, 0xdb, 0xb0, 0x3f, 0x64, 0x6a, 0x15, 0xbd, 0x39, 0xc9, 0x3f, + 0x5a, 0x2e, 0x6c, 0xb4, 0x08, 0x0b, 0xc5, 0x80, 0x7c, 0xa0, 0xe8, 0x2b, 0x30, 0x35, 0x90, 0xfa, + 0x32, 0x88, 0x49, 0x20, 0x31, 0x09, 0xc3, 0x98, 0x8a, 0xf4, 0xb7, 0x6a, 0xc3, 0xdf, 0xd6, 0xfd, + 0xa6, 0x6e, 0xdb, 0x69, 0xf7, 0xcb, 0x3f, 0x57, 0x60, 0xad, 0xcb, 0xa3, 0x61, 0x80, 0xca, 0x50, + 0xea, 0x79, 0x6f, 0xbd, 0xf6, 0x8f, 0x5e, 0xed, 0x23, 0xb4, 0x0b, 0x9f, 0x38, 0xae, 0xdd, 0x6c, + 0x7b, 0xd8, 0x39, 0x6f, 0x37, 0xdf, 0x62, 0xdb, 0xf3, 0xda, 0x3d, 0xaf, 0xe9, 0xd6, 0x0c, 0x64, + 0xc2, 0xc7, 0x33, 0x2d, 0xdf, 0xfd, 0xbe, 0xe7, 0x76, 0xba, 0xb5, 0x07, 0xe8, 0x73, 0x78, 0xb6, + 0xac, 0x83, 0x9d, 0x9f, 0x70, 0xe7, 0xbc, 0xdd, 0xc5, 0x5e, 0xef, 0x3b, 0xc7, 0xf5, 0x6b, 0x2b, + 0x0b, 0xe9, 0xbe, 0xdb, 0x79, 0xd7, 0xf6, 0x3a, 0x6e, 0x6d, 0x15, 0x1d, 0xc0, 0x13, 0xc7, 0xee, + 0x36, 0x5b, 0xee, 0x19, 0x5e, 0x7a, 0xca, 0x1a, 0x3a, 0x84, 0xa7, 0xb7, 0x28, 0x74, 0xc8, 0x43, + 0xb4, 0x0d, 0xa8, 0xd9, 0xb2, 0x5f, 0x7b, 0xb8, 0xe5, 0xda, 0x67, 0x53, 0x6b, 0x09, 0xed, 0xc0, + 0xd6, 0x4c, 0x5d, 0x1b, 0xd6, 0x51, 0x1d, 0xf6, 0x74, 0x56, 0xa7, 0x6b, 0x77, 0x5d, 0xdc, 0xb2, + 0x3b, 0xad, 0x1b, 0xe6, 0x8d, 0x1c, 0x73, 0xda, 0xcf, 0x22, 0x21, 0x87, 0x92, 0x75, 0x74, 0x68, + 0x39, 0x31, 0xd9, 0xdd, 0xae, 0x9b, 0xd4, 0x5f, 0xb7, 0xbd, 0x9b, 0xb8, 0x4a, 0x32, 0x47, 0xbe, + 0x93, 0xa5, 0x6d, 0xce, 0x5b, 0xa6, 0x61, 0x55, 0xa7, 0xf2, 0xf7, 0x75, 0xdd, 0xf8, 0xe7, 0xba, + 0x6e, 0xfc, 0x7b, 0x5d, 0x37, 0xfa, 0x0f, 0xd5, 0x97, 0xe1, 0xcb, 0xff, 0x02, 0x00, 0x00, 0xff, + 0xff, 0xec, 0x05, 0xc3, 0xc5, 0x78, 0x0a, 0x00, 0x00, } func (m *Envelope) Marshal() (dAtA []byte, err error) { @@ -2350,6 +2400,33 @@ func (m *ExitResponse) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *Handshake) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Handshake) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DepositContractAddress) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintMessages(dAtA, i, uint64(len(m.DepositContractAddress))) + i += copy(dAtA[i:], m.DepositContractAddress) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func encodeVarintMessages(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2828,6 +2905,22 @@ func (m *ExitResponse) Size() (n int) { return n } +func (m *Handshake) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.DepositContractAddress) + if l > 0 { + n += 1 + l + sovMessages(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovMessages(x uint64) (n int) { for { n++ @@ -5498,6 +5591,92 @@ func (m *ExitResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *Handshake) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessages + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Handshake: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Handshake: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DepositContractAddress", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessages + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMessages + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMessages + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DepositContractAddress = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMessages(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMessages + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthMessages + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipMessages(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/proto/beacon/p2p/v1/messages.proto b/proto/beacon/p2p/v1/messages.proto index 9ce168733b76..1c823c1fef1e 100644 --- a/proto/beacon/p2p/v1/messages.proto +++ b/proto/beacon/p2p/v1/messages.proto @@ -138,3 +138,7 @@ message ExitResponse { bytes hash = 1; VoluntaryExit voluntary_exit = 2; } + +message Handshake { + string deposit_contract_address = 1; +} diff --git a/shared/p2p/BUILD.bazel b/shared/p2p/BUILD.bazel index 76d164cf40f7..690a3e795134 100644 --- a/shared/p2p/BUILD.bazel +++ b/shared/p2p/BUILD.bazel @@ -7,9 +7,11 @@ go_library( "dial_relay_node.go", "discovery.go", "feed.go", + "handshake_handler.go", "interfaces.go", "message.go", "monitoring.go", + "negotiation.go", "options.go", "p2p.go", "service.go", @@ -54,6 +56,7 @@ go_test( "feed_test.go", "message_test.go", "monitoring_test.go", + "negotiation_test.go", "options_test.go", "register_topic_example_test.go", "service_test.go", @@ -71,6 +74,8 @@ go_test( "@com_github_gogo_protobuf//proto:go_default_library", "@com_github_golang_mock//gomock:go_default_library", "@com_github_libp2p_go_libp2p_blankhost//:go_default_library", + "@com_github_libp2p_go_libp2p_net//:go_default_library", + "@com_github_libp2p_go_libp2p_peer//:go_default_library", "@com_github_libp2p_go_libp2p_peerstore//:go_default_library", "@com_github_libp2p_go_libp2p_protocol//:go_default_library", "@com_github_libp2p_go_libp2p_pubsub//:go_default_library", diff --git a/shared/p2p/discovery.go b/shared/p2p/discovery.go index 4662c9ffbd1c..c9f7e378c7c6 100644 --- a/shared/p2p/discovery.go +++ b/shared/p2p/discovery.go @@ -37,11 +37,7 @@ func startDHTDiscovery(ctx context.Context, host host.Host, bootstrapAddr string ctx, span := trace.StartSpan(ctx, "p2p_startDHTDiscovery") defer span.End() - addr, err := iaddr.ParseString(bootstrapAddr) - if err != nil { - return err - } - peerinfo, err := ps.InfoFromP2pAddr(addr.Multiaddr()) + peerinfo, err := peerInfoFromAddr(bootstrapAddr) if err != nil { return err } @@ -50,6 +46,14 @@ func startDHTDiscovery(ctx context.Context, host host.Host, bootstrapAddr string return err } +func peerInfoFromAddr(address string) (*ps.PeerInfo, error) { + addr, err := iaddr.ParseString(address) + if err != nil { + return nil, err + } + return ps.InfoFromP2pAddr(addr.Multiaddr()) +} + // Discovery implements mDNS notifee interface. type discovery struct { ctx context.Context diff --git a/shared/p2p/handshake_handler.go b/shared/p2p/handshake_handler.go new file mode 100644 index 000000000000..127c98ef914d --- /dev/null +++ b/shared/p2p/handshake_handler.go @@ -0,0 +1,23 @@ +package p2p + +import ( + ggio "github.com/gogo/protobuf/io" + host "github.com/libp2p/go-libp2p-host" + inet "github.com/libp2p/go-libp2p-net" + pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" +) + +// setHandshakeHandler to respond to requests for p2p handshake messages. +func setHandshakeHandler(host host.Host, contractAddress string) { + host.SetStreamHandler(handshakeProtocol, func(stream inet.Stream) { + defer stream.Close() + log.Debug("Handling handshake stream") + w := ggio.NewDelimitedWriter(stream) + defer w.Close() + + hs := &pb.Handshake{DepositContractAddress: contractAddress} + if err := w.WriteMsg(hs); err != nil { + log.WithError(err).Error("Failed to write handshake response") + } + }) +} diff --git a/shared/p2p/negotiation.go b/shared/p2p/negotiation.go new file mode 100644 index 000000000000..c787bab2b4bd --- /dev/null +++ b/shared/p2p/negotiation.go @@ -0,0 +1,93 @@ +package p2p + +import ( + "context" + + ggio "github.com/gogo/protobuf/io" + host "github.com/libp2p/go-libp2p-host" + inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/sirupsen/logrus" +) + +const handshakeProtocol = prysmProtocolPrefix + "/handshake" + +// setupPeerNegotiation adds a "Connected" event handler which checks a peer's +// handshake to ensure the peer is on the same blockchain. This currently +// checks only the deposit contract address. Some peer IDs may be excluded. +// For example, a relay or bootnode will not support the handshake protocol, +// but we would not want to disconnect from those well known peer IDs. +func setupPeerNegotiation(h host.Host, contractAddress string, exclusions []peer.ID) { + h.Network().Notify(&inet.NotifyBundle{ + ConnectedF: func(net inet.Network, conn inet.Conn) { + // Must be handled in a goroutine as this callback cannot be blocking. + go func() { + // Exclude bootstrap node, relay node, etc. + for _, exclusion := range exclusions { + if conn.RemotePeer() == exclusion { + return + } + } + + log.WithField("peer", conn.RemotePeer()).Debug( + "Checking connection to peer", + ) + + s, err := h.NewStream( + context.Background(), + conn.RemotePeer(), + handshakeProtocol, + ) + if err != nil { + log.WithError(err).WithField("peer", conn.RemotePeer()).Error( + "Failed to open stream with newly connected peer", + ) + + if err := h.Network().ClosePeer(conn.RemotePeer()); err != nil { + log.WithError(err).Error("failed to disconnect peer") + } + return + } + defer s.Close() + + w := ggio.NewDelimitedWriter(s) + defer w.Close() + + hs := &pb.Handshake{DepositContractAddress: contractAddress} + if err := w.WriteMsg(hs); err != nil { + log.WithError(err).Error("Failed to write handshake to peer") + + if err := h.Network().ClosePeer(conn.RemotePeer()); err != nil { + log.WithError(err).Error("failed to disconnect peer") + } + return + } + + r := ggio.NewDelimitedReader(s, maxMessageSize) + resp := &pb.Handshake{} + if err := r.ReadMsg(resp); err != nil { + log.WithError(err).Error("Failed to read message") + + if err := h.Network().ClosePeer(conn.RemotePeer()); err != nil { + log.WithError(err).Error("failed to disconnect peer") + } + return + } + + log.WithField("msg", resp).Debug("Handshake received") + + if resp.DepositContractAddress != contractAddress { + log.WithFields(logrus.Fields{ + "peerContract": resp.DepositContractAddress, + "expectedContract": contractAddress, + }).Warn("Disconnecting from peer on different contract") + + if err := h.Network().ClosePeer(conn.RemotePeer()); err != nil { + log.WithError(err).Error("failed to disconnect peer") + } + } + }() + }, + }) +} diff --git a/shared/p2p/negotiation_test.go b/shared/p2p/negotiation_test.go new file mode 100644 index 000000000000..906328e2064e --- /dev/null +++ b/shared/p2p/negotiation_test.go @@ -0,0 +1,58 @@ +package p2p + +import ( + "context" + "testing" + "time" + + bhost "github.com/libp2p/go-libp2p-blankhost" + libp2pnet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" + swarmt "github.com/libp2p/go-libp2p-swarm/testing" +) + +func TestNegotiation_AcceptsValidPeer(t *testing.T) { + ctx := context.Background() + hostA := bhost.NewBlankHost(swarmt.GenSwarm(t, ctx)) + hostB := bhost.NewBlankHost(swarmt.GenSwarm(t, ctx)) + + address := "same" + setHandshakeHandler(hostA, address) + setHandshakeHandler(hostB, address) + + setupPeerNegotiation(hostA, address, []peer.ID{}) + setupPeerNegotiation(hostB, address, []peer.ID{}) + + if err := hostA.Connect(ctx, pstore.PeerInfo{ID: hostB.ID(), Addrs: hostB.Addrs()}); err != nil { + t.Fatal(err) + } + + // Allow short delay for async negotiation. + time.Sleep(200 * time.Millisecond) + if hostA.Network().Connectedness(hostB.ID()) != libp2pnet.Connected { + t.Error("hosts are not connected") + } +} + +func TestNegotiation_DisconnectsDifferentDepositContract(t *testing.T) { + ctx := context.Background() + hostA := bhost.NewBlankHost(swarmt.GenSwarm(t, ctx)) + hostB := bhost.NewBlankHost(swarmt.GenSwarm(t, ctx)) + + setHandshakeHandler(hostA, "hostA") + setHandshakeHandler(hostB, "hostB") + + setupPeerNegotiation(hostA, "hostA", []peer.ID{}) + setupPeerNegotiation(hostB, "hostB", []peer.ID{}) + + if err := hostA.Connect(ctx, pstore.PeerInfo{ID: hostB.ID(), Addrs: hostB.Addrs()}); err != nil { + t.Fatal(err) + } + + // Allow short delay for async negotiation. + time.Sleep(200 * time.Millisecond) + if hostA.Network().Connectedness(hostB.ID()) == libp2pnet.Connected { + t.Error("hosts are connected, but should not be connected") + } +} diff --git a/shared/p2p/service.go b/shared/p2p/service.go index 18e2d62fb4d4..205169e3c756 100644 --- a/shared/p2p/service.go +++ b/shared/p2p/service.go @@ -60,10 +60,11 @@ type Server struct { // ServerConfig for peer to peer networking. type ServerConfig struct { - NoDiscovery bool - BootstrapNodeAddr string - RelayNodeAddr string - Port int + NoDiscovery bool + BootstrapNodeAddr string + RelayNodeAddr string + Port int + DepositContractAddress string } // NewServer creates a new p2p server instance. @@ -103,6 +104,22 @@ func NewServer(cfg *ServerConfig) (*Server, error) { return nil, err } + // Blockchain peering negitiation; excludes negotiating with bootstrap or + // relay nodes. + exclusions := []peer.ID{} + for _, addr := range []string{cfg.BootstrapNodeAddr, cfg.RelayNodeAddr} { + if addr == "" { + continue + } + info, err := peerInfoFromAddr(addr) + if err != nil { + return nil, err + } + exclusions = append(exclusions, info.ID) + } + setupPeerNegotiation(h, cfg.DepositContractAddress, exclusions) + setHandshakeHandler(h, cfg.DepositContractAddress) + return &Server{ ctx: ctx, cancel: cancel, diff --git a/shared/p2p/service_test.go b/shared/p2p/service_test.go index 3595be614870..e496b3e70089 100644 --- a/shared/p2p/service_test.go +++ b/shared/p2p/service_test.go @@ -39,19 +39,14 @@ func init() { logrus.SetLevel(logrus.DebugLevel) } -func TestStartDialRelayNode_InvalidMultiaddress(t *testing.T) { - hook := logTest.NewGlobal() - - s, err := NewServer(&ServerConfig{ +func TestNewServer_InvalidMultiaddress(t *testing.T) { + _, err := NewServer(&ServerConfig{ RelayNodeAddr: "bad", }) - if err != nil { - t.Fatalf("Unable to create server: %v", err) + if err.Error() != "invalid multiaddr, must begin with /" { + t.Fatal("expected invalid multiaddr err") } - - s.Start() - logContains(t, hook, "Could not dial relay node: invalid multiaddr, must begin with /", logrus.ErrorLevel) } func TestP2P_PortTaken(t *testing.T) { @@ -448,6 +443,8 @@ func simulateIncomingMessage(t *testing.T, s *Server, topic string, msg proto.Me ctx := context.Background() h := bhost.NewBlankHost(swarmt.GenSwarm(t, ctx)) + setHandshakeHandler(h, "") + gsub, err := pubsub.NewFloodSub(ctx, h) if err != nil { return err From cd302e88c27ced2551670eea9f7155152748e35d Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Sun, 28 Apr 2019 03:36:53 +0800 Subject: [PATCH 34/54] Fix Block Processing And UpdateChainHead (#2399) * fix block processing * add in defer * do not update chainhead * save historical states from all blocks, so that fork choice can retrieve it * revert changes * make new changes * remove forkchoice at the end --- beacon-chain/sync/receive_block.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/beacon-chain/sync/receive_block.go b/beacon-chain/sync/receive_block.go index 9821e70ce443..adaa51d05376 100644 --- a/beacon-chain/sync/receive_block.go +++ b/beacon-chain/sync/receive_block.go @@ -66,6 +66,8 @@ func (rs *RegularSync) receiveBlock(msg p2p.Message) error { ctx, span := trace.StartSpan(msg.Ctx, "beacon-chain.sync.receiveBlock") defer span.End() recBlock.Inc() + rs.blockProcessingLock.Lock() + defer rs.blockProcessingLock.Unlock() return rs.processBlockAndFetchAncestors(ctx, msg) } @@ -74,7 +76,7 @@ func (rs *RegularSync) receiveBlock(msg p2p.Message) error { // At the end of the recursive call, we'll have a block which has no children in the map, and at that point // we can apply the fork choice rule for ETH 2.0. func (rs *RegularSync) processBlockAndFetchAncestors(ctx context.Context, msg p2p.Message) error { - block, beaconState, isValid, err := rs.validateAndProcessBlock(ctx, msg) + block, _, isValid, err := rs.validateAndProcessBlock(ctx, msg) if err != nil { return err } @@ -101,7 +103,7 @@ func (rs *RegularSync) processBlockAndFetchAncestors(ctx context.Context, msg p2 rs.clearPendingBlock(blockRoot) return rs.processBlockAndFetchAncestors(ctx, child) } - return rs.chainService.ApplyForkChoiceRule(ctx, block, beaconState) + return nil } func (rs *RegularSync) validateAndProcessBlock( @@ -109,8 +111,6 @@ func (rs *RegularSync) validateAndProcessBlock( ) (*pb.BeaconBlock, *pb.BeaconState, bool, error) { ctx, span := trace.StartSpan(ctx, "beacon-chain.sync.validateAndProcessBlock") defer span.End() - rs.blockProcessingLock.Lock() - defer rs.blockProcessingLock.Unlock() response := blockMsg.Data.(*pb.BeaconBlockResponse) block := response.Block @@ -185,19 +185,24 @@ func (rs *RegularSync) validateAndProcessBlock( return nil, nil, false, err } - // only update head of chain if block is a child of the chainhead. + // only run fork choice if the block has the chain head as the parent if headRoot == bytesutil.ToBytes32(block.ParentRootHash32) { - if err := rs.db.UpdateChainHead(ctx, block, beaconState); err != nil { - log.Errorf("Could not update chain head: %v", err) - span.AddAttributes(trace.BoolAttribute("invalidBlock", true)) + if err := rs.chainService.ApplyForkChoiceRule(ctx, block, beaconState); err != nil { + log.Errorf("Could not run fork choice on block %v", err) return nil, nil, false, err } + } else { + // Save historical state from forked block. forkedBlock.Inc() log.WithFields(logrus.Fields{ "slot": block.Slot, "root": fmt.Sprintf("%#x", blockRoot)}, ).Warn("Received Block from a forked chain") + if err := rs.db.SaveHistoricalState(ctx, beaconState); err != nil { + log.Errorf("Could not save historical state %v", err) + return nil, nil, false, err + } } sentBlocks.Inc() From dcc2250e82702deed9684c1d0933ac3343919c85 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Sat, 27 Apr 2019 14:45:13 -0500 Subject: [PATCH 35/54] Use HTTP for On Demand Block Requests (#2405) * use http instead of websocket for on demand block fetching * use http --- beacon-chain/node/node.go | 8 ++++---- beacon-chain/powchain/log_processing.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/beacon-chain/node/node.go b/beacon-chain/node/node.go index 6fc8c8b08922..4ffd1fdd5dd2 100644 --- a/beacon-chain/node/node.go +++ b/beacon-chain/node/node.go @@ -263,11 +263,11 @@ func (b *BeaconNode) registerPOWChainService(cliCtx *cli.Context) error { } powClient := ethclient.NewClient(rpcClient) - logRPCClient, err := gethRPC.Dial(cliCtx.GlobalString(utils.HTTPWeb3ProviderFlag.Name)) + httpRPCClient, err := gethRPC.Dial(cliCtx.GlobalString(utils.HTTPWeb3ProviderFlag.Name)) if err != nil { log.Fatalf("Access to PoW chain is required for validator. Unable to connect to Geth node: %v", err) } - pastLogHTTPClient := ethclient.NewClient(logRPCClient) + httpClient := ethclient.NewClient(httpRPCClient) ctx := context.Background() cfg := &powchain.Web3ServiceConfig{ @@ -276,8 +276,8 @@ func (b *BeaconNode) registerPOWChainService(cliCtx *cli.Context) error { Client: powClient, Reader: powClient, Logger: powClient, - HTTPLogger: pastLogHTTPClient, - BlockFetcher: powClient, + HTTPLogger: httpClient, + BlockFetcher: httpClient, ContractBackend: powClient, BeaconDB: b.db, } diff --git a/beacon-chain/powchain/log_processing.go b/beacon-chain/powchain/log_processing.go index c9a922505f62..c0e8a11f7551 100644 --- a/beacon-chain/powchain/log_processing.go +++ b/beacon-chain/powchain/log_processing.go @@ -198,7 +198,7 @@ func (w *Web3Service) requestBatchedLogs() error { FromBlock: w.lastRequestedBlock.Add(w.lastRequestedBlock, big.NewInt(1)), ToBlock: requestedBlock, } - logs, err := w.logger.FilterLogs(w.ctx, query) + logs, err := w.httpLogger.FilterLogs(w.ctx, query) if err != nil { return err } From 769fcb50abaac7954e0faa61f9c0940190b5b3b8 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Sat, 27 Apr 2019 16:05:19 -0400 Subject: [PATCH 36/54] use Network, not Peerstore (#2404) --- shared/p2p/discovery.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/p2p/discovery.go b/shared/p2p/discovery.go index c9f7e378c7c6..656bf159a400 100644 --- a/shared/p2p/discovery.go +++ b/shared/p2p/discovery.go @@ -73,6 +73,6 @@ func (d *discovery) HandlePeerFound(pi ps.PeerInfo) { } log.WithFields(logrus.Fields{ - "peers": d.host.Peerstore().Peers(), + "peers": d.host.Network().Peers(), }).Debug("Peers are now") } From 5167fadbd7eb1fc2811efcc87d460179398b1c7c Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Sat, 27 Apr 2019 16:41:46 -0400 Subject: [PATCH 37/54] Serve pprof http (#2406) * serve pprof http * add comment for linter --- shared/debug/debug.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shared/debug/debug.go b/shared/debug/debug.go index e5cbc4026aa0..68462a8f91cc 100644 --- a/shared/debug/debug.go +++ b/shared/debug/debug.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "net/http" + _ "net/http/pprof" // required to serve pprof http endpoints. "os" "os/user" "path/filepath" @@ -354,7 +355,7 @@ func Setup(ctx *cli.Context) error { func startPProf(address string) { http.Handle("/memsize/", http.StripPrefix("/memsize", &Memsize)) - log.Info("Starting pprof server", "addr", fmt.Sprintf("http://%s/debug/pprof", address)) + log.WithField("addr", fmt.Sprintf("http://%s/debug/pprof", address)).Info("Starting pprof server") go func() { if err := http.ListenAndServe(address, nil); err != nil { log.Error("Failure in running pprof server", "err", err) From 80d47468e4900ba32f02938126e40be2ddfe559f Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Sat, 27 Apr 2019 22:56:28 -0400 Subject: [PATCH 38/54] Add a temporary commenting for issue #2408 (#2409) --- shared/p2p/negotiation.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/shared/p2p/negotiation.go b/shared/p2p/negotiation.go index c787bab2b4bd..0e936d0c8508 100644 --- a/shared/p2p/negotiation.go +++ b/shared/p2p/negotiation.go @@ -40,13 +40,15 @@ func setupPeerNegotiation(h host.Host, contractAddress string, exclusions []peer handshakeProtocol, ) if err != nil { - log.WithError(err).WithField("peer", conn.RemotePeer()).Error( - "Failed to open stream with newly connected peer", - ) + log.WithError(err).WithFields(logrus.Fields{ + "peer": conn.RemotePeer(), + "address": conn.RemoteMultiaddr(), + }).Error("Failed to open stream with newly connected peer") - if err := h.Network().ClosePeer(conn.RemotePeer()); err != nil { - log.WithError(err).Error("failed to disconnect peer") - } + log.Warn("Temporarily disabled -- not disconnecting peer. See https://github.com/prysmaticlabs/prysm/issues/2408") + // if err := h.Network().ClosePeer(conn.RemotePeer()); err != nil { + // log.WithError(err).Error("failed to disconnect peer") + // } return } defer s.Close() From 63378793b4fc0db7925788cb2a885ccae1e8ee04 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Sun, 28 Apr 2019 09:44:45 -0500 Subject: [PATCH 39/54] do not run assign every slot (#2412) --- validator/client/validator.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/validator/client/validator.go b/validator/client/validator.go index b936baa8aea1..2275929707db 100644 --- a/validator/client/validator.go +++ b/validator/client/validator.go @@ -140,9 +140,7 @@ func (v *validator) SlotDeadline(slot uint64) time.Time { // list of upcoming assignments needs to be updated. For example, at the // beginning of a new epoch. func (v *validator) UpdateAssignments(ctx context.Context, slot uint64) error { - // Testing run time for fetching every slot. This is not meant for production! - // https://github.com/prysmaticlabs/prysm/issues/2167 - if slot%params.BeaconConfig().SlotsPerEpoch != 0 && v.assignments != nil && false { + if slot%params.BeaconConfig().SlotsPerEpoch != 0 && v.assignments != nil { // Do nothing if not epoch start AND assignments already exist. return nil } From 85daf6e8078c4ba70b5d68e8b1f5a3d5643530fb Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sun, 28 Apr 2019 14:39:05 -0700 Subject: [PATCH 40/54] update to 3.2ETH (#2415) --- shared/params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/params/config.go b/shared/params/config.go index c1e3bfdfe2bc..6fb268963154 100644 --- a/shared/params/config.go +++ b/shared/params/config.go @@ -213,7 +213,7 @@ func DemoBeaconConfig() *BeaconChainConfig { demoConfig.SlotsPerEpoch = 8 demoConfig.GenesisEpoch = demoConfig.GenesisSlot / 8 demoConfig.MinDepositAmount = 100 - demoConfig.MaxDepositAmount = 3200000 + demoConfig.MaxDepositAmount = 3.2 * 1e9 demoConfig.EjectionBalance = 1600000 demoConfig.SyncPollingInterval = 1 * 10 // Query nodes over the network every slot. demoConfig.Eth1FollowDistance = 5 From 52250c9e7a82031015937352ef55a9b89d44dac4 Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Mon, 29 Apr 2019 05:53:51 +0800 Subject: [PATCH 41/54] Fix Fork Choice Reorg Conditions (#2413) * make fixes * change condition * comment * fixed up a few var names for clarity * added test for isDescendant * chain split reorg test * interface for atts service * comprehensive reorg regression test added * space * all tests pass * target handler * ci lint * Update beacon-chain/sync/receive_block.go Co-Authored-By: rauljordan --- beacon-chain/attestation/service.go | 7 + beacon-chain/attestation/service_test.go | 2 + beacon-chain/blockchain/BUILD.bazel | 1 + beacon-chain/blockchain/fork_choice.go | 66 +++-- .../blockchain/fork_choice_reorg_test.go | 225 ++++++++++++++++++ beacon-chain/blockchain/fork_choice_test.go | 90 +++++++ beacon-chain/blockchain/service.go | 4 +- beacon-chain/sync/receive_block.go | 13 +- 8 files changed, 383 insertions(+), 25 deletions(-) create mode 100644 beacon-chain/blockchain/fork_choice_reorg_test.go diff --git a/beacon-chain/attestation/service.go b/beacon-chain/attestation/service.go index f7d003a7bc60..570112cc6a38 100644 --- a/beacon-chain/attestation/service.go +++ b/beacon-chain/attestation/service.go @@ -25,6 +25,13 @@ import ( var log = logrus.WithField("prefix", "attestation") var committeeCache = cache.NewCommitteesCache() +// TargetHandler provides an interface for fetching latest attestation targets +// and updating attestations in batches. +type TargetHandler interface { + LatestAttestationTarget(state *pb.BeaconState, validatorIndex uint64) (*pb.AttestationTarget, error) + BatchUpdateLatestAttestation(ctx context.Context, atts []*pb.Attestation) error +} + type attestationStore struct { sync.RWMutex m map[[48]byte]*pb.Attestation diff --git a/beacon-chain/attestation/service_test.go b/beacon-chain/attestation/service_test.go index 1b0697ba6bef..906ce744214c 100644 --- a/beacon-chain/attestation/service_test.go +++ b/beacon-chain/attestation/service_test.go @@ -23,6 +23,8 @@ func init() { logrus.SetLevel(logrus.DebugLevel) } +var _ = TargetHandler(&Service{}) + func TestUpdateLatestAttestation_UpdatesLatest(t *testing.T) { beaconDB := internal.SetupDB(t) defer internal.TeardownDB(t, beaconDB) diff --git a/beacon-chain/blockchain/BUILD.bazel b/beacon-chain/blockchain/BUILD.bazel index fc3b537c3716..32eaf491a9b9 100644 --- a/beacon-chain/blockchain/BUILD.bazel +++ b/beacon-chain/blockchain/BUILD.bazel @@ -37,6 +37,7 @@ go_test( name = "go_default_test", srcs = [ "block_processing_test.go", + "fork_choice_reorg_test.go", "fork_choice_test.go", "service_test.go", ], diff --git a/beacon-chain/blockchain/fork_choice.go b/beacon-chain/blockchain/fork_choice.go index 394f10027ae9..3486d43df866 100644 --- a/beacon-chain/blockchain/fork_choice.go +++ b/beacon-chain/blockchain/fork_choice.go @@ -140,50 +140,65 @@ func (c *ChainService) ApplyForkChoiceRule( if err != nil { return fmt.Errorf("could not retrieve justified head: %v", err) } - head, err := c.lmdGhost(ctx, justifiedHead, justifiedState, attestationTargets) + + newHead, err := c.lmdGhost(ctx, justifiedHead, justifiedState, attestationTargets) if err != nil { return fmt.Errorf("could not run fork choice: %v", err) } - headRoot, err := hashutil.HashBeaconBlock(head) + newHeadRoot, err := hashutil.HashBeaconBlock(newHead) if err != nil { return fmt.Errorf("could not hash head block: %v", err) } c.canonicalBlocksLock.Lock() defer c.canonicalBlocksLock.Unlock() - c.canonicalBlocks[head.Slot] = headRoot[:] + c.canonicalBlocks[newHead.Slot] = newHeadRoot[:] + + currentHead, err := c.beaconDB.ChainHead() + if err != nil { + return fmt.Errorf("could not retrieve chain head: %v", err) + } + + isDescendant, err := c.isDescendant(currentHead, newHead) + if err != nil { + return fmt.Errorf("could not check if block is descendant: %v", err) + } newState := postState - if head.Slot != block.Slot { - log.Warnf("Reorg happened, last processed block at slot %d, new head block at slot %d", - block.Slot-params.BeaconConfig().GenesisSlot, head.Slot-params.BeaconConfig().GenesisSlot) + if !isDescendant { + log.Warnf("Reorg happened, last head at slot %d, new head block at slot %d", + currentHead.Slot-params.BeaconConfig().GenesisSlot, newHead.Slot-params.BeaconConfig().GenesisSlot) // Only regenerate head state if there was a reorg. - newState, err = c.beaconDB.HistoricalStateFromSlot(ctx, head.Slot) + newState, err = c.beaconDB.HistoricalStateFromSlot(ctx, newHead.Slot) if err != nil { return fmt.Errorf("could not gen state: %v", err) } - if newState.Slot != postState.Slot { - log.Warnf("Reorg happened, post state slot at %d, new head state at slot %d", - postState.Slot-params.BeaconConfig().GenesisSlot, newState.Slot-params.BeaconConfig().GenesisSlot) - } - - for revertedSlot := block.Slot; revertedSlot > head.Slot; revertedSlot-- { + for revertedSlot := currentHead.Slot; revertedSlot > newHead.Slot; revertedSlot-- { delete(c.canonicalBlocks, revertedSlot) } reorgCount.Inc() } - if err := c.beaconDB.UpdateChainHead(ctx, head, newState); err != nil { + // If we receive forked blocks. + if newHead.Slot != newState.Slot { + newState, err = c.beaconDB.HistoricalStateFromSlot(ctx, newHead.Slot) + if err != nil { + return fmt.Errorf("could not gen state: %v", err) + } + } + + if err := c.beaconDB.UpdateChainHead(ctx, newHead, newState); err != nil { return fmt.Errorf("failed to update chain: %v", err) } - h, err := hashutil.HashBeaconBlock(head) + h, err := hashutil.HashBeaconBlock(newHead) if err != nil { return fmt.Errorf("could not hash head: %v", err) } log.WithFields(logrus.Fields{ "headRoot": fmt.Sprintf("0x%x", h), }).Info("Chain head block and state updated") + return nil } @@ -288,6 +303,27 @@ func (c *ChainService) blockChildren(ctx context.Context, block *pb.BeaconBlock, return children, nil } +// isDescendant checks if the new head block is a descendant block of the current head. +func (c *ChainService) isDescendant(currentHead *pb.BeaconBlock, newHead *pb.BeaconBlock) (bool, error) { + currentHeadRoot, err := hashutil.HashBeaconBlock(currentHead) + if err != nil { + return false, nil + } + for newHead.Slot > currentHead.Slot { + if bytesutil.ToBytes32(newHead.ParentRootHash32) == currentHeadRoot { + return true, nil + } + newHead, err = c.beaconDB.Block(bytesutil.ToBytes32(newHead.ParentRootHash32)) + if err != nil { + return false, err + } + if newHead == nil { + return false, nil + } + } + return false, nil +} + // attestationTargets retrieves the list of attestation targets since last finalized epoch, // each attestation target consists of validator index and its attestation target (i.e. the block // which the validator attested to) diff --git a/beacon-chain/blockchain/fork_choice_reorg_test.go b/beacon-chain/blockchain/fork_choice_reorg_test.go new file mode 100644 index 000000000000..2612b9d8e84a --- /dev/null +++ b/beacon-chain/blockchain/fork_choice_reorg_test.go @@ -0,0 +1,225 @@ +package blockchain + +import ( + "context" + "fmt" + "testing" + + "github.com/gogo/protobuf/proto" + "github.com/prysmaticlabs/prysm/beacon-chain/core/state" + "github.com/prysmaticlabs/prysm/beacon-chain/internal" + pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/hashutil" + "github.com/prysmaticlabs/prysm/shared/params" + "github.com/prysmaticlabs/prysm/shared/testutil" + logTest "github.com/sirupsen/logrus/hooks/test" +) + +type mockAttestationHandler struct { + targets map[uint64]*pb.AttestationTarget +} + +func (m *mockAttestationHandler) LatestAttestationTarget(beaconState *pb.BeaconState, idx uint64) (*pb.AttestationTarget, error) { + return m.targets[idx], nil +} + +func (m *mockAttestationHandler) BatchUpdateLatestAttestation(ctx context.Context, atts []*pb.Attestation) error { + return nil +} + +func TestApplyForkChoice_ChainSplitReorg(t *testing.T) { + hook := logTest.NewGlobal() + beaconDB := internal.SetupDB(t) + defer internal.TeardownDB(t, beaconDB) + + ctx := context.Background() + deposits, _ := setupInitialDeposits(t, 100) + eth1Data := &pb.Eth1Data{ + DepositRootHash32: []byte{}, + BlockHash32: []byte{}, + } + justifiedState, err := state.GenesisBeaconState(deposits, 0, eth1Data) + if err != nil { + t.Fatalf("Can't generate genesis state: %v", err) + } + + chainService := setupBeaconChain(t, beaconDB, nil) + + // Construct a forked chain that looks as follows: + // /------B1 ----B3 ----- B5 (current head) + // B0 --B2 -------------B4 + blocks, roots := constructForkedChain(t, justifiedState) + + // We then setup a canonical chain of the following blocks: + // B0->B1->B3->B5. + if err := chainService.beaconDB.SaveBlock(blocks[0]); err != nil { + t.Fatal(err) + } + justifiedState.LatestBlock = blocks[0] + if err := chainService.beaconDB.SaveJustifiedState(justifiedState); err != nil { + t.Fatal(err) + } + if err := chainService.beaconDB.SaveJustifiedBlock(blocks[0]); err != nil { + t.Fatal(err) + } + if err := chainService.beaconDB.UpdateChainHead(ctx, blocks[0], justifiedState); err != nil { + t.Fatal(err) + } + canonicalBlockIndices := []int{1, 3, 5} + postState := proto.Clone(justifiedState).(*pb.BeaconState) + for _, canonicalIndex := range canonicalBlockIndices { + postState, err = chainService.ApplyBlockStateTransition(ctx, blocks[canonicalIndex], postState) + if err != nil { + t.Fatal(err) + } + if err := chainService.beaconDB.SaveBlock(blocks[canonicalIndex]); err != nil { + t.Fatal(err) + } + if err := chainService.beaconDB.UpdateChainHead(ctx, blocks[canonicalIndex], postState); err != nil { + t.Fatal(err) + } + } + + chainHead, err := chainService.beaconDB.ChainHead() + if err != nil { + t.Fatal(err) + } + if chainHead.Slot != justifiedState.Slot+5 { + t.Errorf( + "Expected chain head with slot %d, received %d", + justifiedState.Slot+5-params.BeaconConfig().GenesisSlot, + chainHead.Slot-params.BeaconConfig().GenesisSlot, + ) + } + + // We then save forked blocks and their historical states (but do not update chain head). + // The fork is from B0->B2->B4. + forkedBlockIndices := []int{2, 4} + forkState := proto.Clone(justifiedState).(*pb.BeaconState) + for _, forkIndex := range forkedBlockIndices { + forkState, err = chainService.ApplyBlockStateTransition(ctx, blocks[forkIndex], forkState) + if err != nil { + t.Fatal(err) + } + if err := chainService.beaconDB.SaveBlock(blocks[forkIndex]); err != nil { + t.Fatal(err) + } + if err := chainService.beaconDB.SaveHistoricalState(ctx, forkState); err != nil { + t.Fatal(err) + } + } + + // Give the block from the forked chain, B4, the most votes. + voteTargets := make(map[uint64]*pb.AttestationTarget) + voteTargets[0] = &pb.AttestationTarget{ + Slot: blocks[5].Slot, + BlockRoot: roots[5][:], + ParentRoot: blocks[5].ParentRootHash32, + } + for i := 1; i < len(deposits); i++ { + voteTargets[uint64(i)] = &pb.AttestationTarget{ + Slot: blocks[4].Slot, + BlockRoot: roots[4][:], + ParentRoot: blocks[4].ParentRootHash32, + } + } + attHandler := &mockAttestationHandler{ + targets: voteTargets, + } + chainService.attsService = attHandler + + block4State, err := chainService.beaconDB.HistoricalStateFromSlot(ctx, blocks[4].Slot) + if err != nil { + t.Fatal(err) + } + // Applying the fork choice rule should reorg to B4 successfully. + if err := chainService.ApplyForkChoiceRule(ctx, blocks[4], block4State); err != nil { + t.Fatal(err) + } + + newHead, err := chainService.beaconDB.ChainHead() + if err != nil { + t.Fatal(err) + } + if !proto.Equal(newHead, blocks[4]) { + t.Errorf( + "Expected chain head %v, received %v", + blocks[4], + newHead, + ) + } + want := fmt.Sprintf( + "Reorg happened, last head at slot %d, new head block at slot %d", + blocks[5].Slot-params.BeaconConfig().GenesisSlot, blocks[4].Slot-params.BeaconConfig().GenesisSlot, + ) + testutil.AssertLogsContain(t, hook, want) +} + +func constructForkedChain(t *testing.T, beaconState *pb.BeaconState) ([]*pb.BeaconBlock, [][32]byte) { + // Construct the following chain: + // /------B1 ----B3 ----- B5 (current head) + // B0 --B2 -------------B4 + blocks := make([]*pb.BeaconBlock, 6) + roots := make([][32]byte, 6) + var err error + blocks[0] = &pb.BeaconBlock{ + Slot: beaconState.Slot, + ParentRootHash32: []byte{'A'}, + Body: &pb.BeaconBlockBody{}, + } + roots[0], err = hashutil.HashBeaconBlock(blocks[0]) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } + + blocks[1] = &pb.BeaconBlock{ + Slot: beaconState.Slot + 2, + ParentRootHash32: roots[0][:], + Body: &pb.BeaconBlockBody{}, + } + roots[1], err = hashutil.HashBeaconBlock(blocks[1]) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } + + blocks[2] = &pb.BeaconBlock{ + Slot: beaconState.Slot + 1, + ParentRootHash32: roots[0][:], + Body: &pb.BeaconBlockBody{}, + } + roots[2], err = hashutil.HashBeaconBlock(blocks[2]) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } + + blocks[3] = &pb.BeaconBlock{ + Slot: beaconState.Slot + 3, + ParentRootHash32: roots[1][:], + Body: &pb.BeaconBlockBody{}, + } + roots[3], err = hashutil.HashBeaconBlock(blocks[3]) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } + + blocks[4] = &pb.BeaconBlock{ + Slot: beaconState.Slot + 4, + ParentRootHash32: roots[2][:], + Body: &pb.BeaconBlockBody{}, + } + roots[4], err = hashutil.HashBeaconBlock(blocks[4]) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } + + blocks[5] = &pb.BeaconBlock{ + Slot: beaconState.Slot + 5, + ParentRootHash32: roots[3][:], + Body: &pb.BeaconBlockBody{}, + } + roots[5], err = hashutil.HashBeaconBlock(blocks[5]) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } + return blocks, roots +} diff --git a/beacon-chain/blockchain/fork_choice_test.go b/beacon-chain/blockchain/fork_choice_test.go index 0902c19a32e3..80bf26c74081 100644 --- a/beacon-chain/blockchain/fork_choice_test.go +++ b/beacon-chain/blockchain/fork_choice_test.go @@ -653,6 +653,96 @@ func TestLMDGhost_3WayChainSplitsSameHeight(t *testing.T) { } } +func TestIsDescendant_Ok(t *testing.T) { + beaconDB := internal.SetupDB(t) + defer internal.TeardownDB(t, beaconDB) + chainService := setupBeaconChain(t, beaconDB, nil) + + // Construct the following chain: + // B1 - B2 - B3 + // \- B4 - B5 + // Prove the following: + // B5 is not a descendant of B2 + // B3 is not a descendant of B4 + // B5 and B3 are descendants of B1 + + block1 := &pb.BeaconBlock{ + Slot: 1, + ParentRootHash32: []byte{'A'}, + } + root1, err := hashutil.HashBeaconBlock(block1) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } + if err = chainService.beaconDB.SaveBlock(block1); err != nil { + t.Fatalf("Could not save block: %v", err) + } + block2 := &pb.BeaconBlock{ + Slot: 2, + ParentRootHash32: root1[:], + } + root2, err := hashutil.HashBeaconBlock(block2) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } + if err = chainService.beaconDB.SaveBlock(block2); err != nil { + t.Fatalf("Could not save block: %v", err) + } + block3 := &pb.BeaconBlock{ + Slot: 3, + ParentRootHash32: root2[:], + } + _, err = hashutil.HashBeaconBlock(block3) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } + if err = chainService.beaconDB.SaveBlock(block3); err != nil { + t.Fatalf("Could not save block: %v", err) + } + block4 := &pb.BeaconBlock{ + Slot: 4, + ParentRootHash32: root1[:], + } + root4, err := hashutil.HashBeaconBlock(block4) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } + if err = chainService.beaconDB.SaveBlock(block4); err != nil { + t.Fatalf("Could not save block: %v", err) + } + block5 := &pb.BeaconBlock{ + Slot: 5, + ParentRootHash32: root4[:], + } + _, err = hashutil.HashBeaconBlock(block5) + if err != nil { + t.Fatalf("Could not hash block: %v", err) + } + if err = chainService.beaconDB.SaveBlock(block5); err != nil { + t.Fatalf("Could not save block: %v", err) + } + + isDescendant, err := chainService.isDescendant(block2, block5) + if err != nil { + t.Fatal(err) + } + if isDescendant { + t.Errorf("block%d can't be descendant of block%d", block5.Slot, block2.Slot) + } + isDescendant, _ = chainService.isDescendant(block4, block3) + if isDescendant { + t.Errorf("block%d can't be descendant of block%d", block3.Slot, block4.Slot) + } + isDescendant, _ = chainService.isDescendant(block1, block5) + if !isDescendant { + t.Errorf("block%d is the descendant of block%d", block3.Slot, block1.Slot) + } + isDescendant, _ = chainService.isDescendant(block1, block3) + if !isDescendant { + t.Errorf("block%d is the descendant of block%d", block3.Slot, block1.Slot) + } +} + func TestLMDGhost_2WayChainSplitsDiffHeight(t *testing.T) { beaconDB := internal.SetupDB(t) defer internal.TeardownDB(t, beaconDB) diff --git a/beacon-chain/blockchain/service.go b/beacon-chain/blockchain/service.go index daede40b9126..75da89cf49da 100644 --- a/beacon-chain/blockchain/service.go +++ b/beacon-chain/blockchain/service.go @@ -38,7 +38,7 @@ type ChainService struct { cancel context.CancelFunc beaconDB *db.BeaconDB web3Service *powchain.Web3Service - attsService *attestation.Service + attsService attestation.TargetHandler opsPoolService operations.OperationFeeds chainStartChan chan time.Time canonicalBlockFeed *event.Feed @@ -55,7 +55,7 @@ type ChainService struct { type Config struct { BeaconBlockBuf int Web3Service *powchain.Web3Service - AttsService *attestation.Service + AttsService attestation.TargetHandler BeaconDB *db.BeaconDB OpsPoolService operations.OperationFeeds DevMode bool diff --git a/beacon-chain/sync/receive_block.go b/beacon-chain/sync/receive_block.go index adaa51d05376..bf1fd3067255 100644 --- a/beacon-chain/sync/receive_block.go +++ b/beacon-chain/sync/receive_block.go @@ -185,14 +185,7 @@ func (rs *RegularSync) validateAndProcessBlock( return nil, nil, false, err } - // only run fork choice if the block has the chain head as the parent - if headRoot == bytesutil.ToBytes32(block.ParentRootHash32) { - if err := rs.chainService.ApplyForkChoiceRule(ctx, block, beaconState); err != nil { - log.Errorf("Could not run fork choice on block %v", err) - return nil, nil, false, err - } - - } else { + if headRoot != bytesutil.ToBytes32(block.ParentRootHash32) { // Save historical state from forked block. forkedBlock.Inc() log.WithFields(logrus.Fields{ @@ -205,6 +198,10 @@ func (rs *RegularSync) validateAndProcessBlock( } } + if err := rs.chainService.ApplyForkChoiceRule(ctx, block, beaconState); err != nil { + log.WithError(err).Error("Could not run fork choice on block") + return nil, nil, false, err + } sentBlocks.Inc() // We update the last observed slot to the received canonical block's slot. if block.Slot > rs.highestObservedSlot { From 524913470458f97b4eded3135a96255463590528 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Sun, 28 Apr 2019 18:23:03 -0400 Subject: [PATCH 42/54] set cluster manager to deposit 3.2 eth (#2416) --- k8s/beacon-chain/cluster-manager.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/beacon-chain/cluster-manager.yaml b/k8s/beacon-chain/cluster-manager.yaml index 80151a461b94..e5cdbd538aed 100644 --- a/k8s/beacon-chain/cluster-manager.yaml +++ b/k8s/beacon-chain/cluster-manager.yaml @@ -58,7 +58,7 @@ spec: - --rpc=ws://public-rpc-nodes.pow.svc.cluster.local:8546 - --port=8000 - --metrics-port=9090 - - --deposit-amount=3200000000000000 + - --deposit-amount=3200000000000000000 - --db-path=/data - --verbose ports: From 63633a3bee166855809996b6daa29fb309af5261 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Sun, 28 Apr 2019 21:53:03 -0500 Subject: [PATCH 43/54] Enable Historical State Pruning By Default (#2433) * disable historical state pruning * imports * lint --- beacon-chain/db/state.go | 2 +- beacon-chain/db/state_test.go | 2 +- shared/featureconfig/config.go | 20 ++++++++++---------- shared/featureconfig/flags.go | 10 +++++----- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/beacon-chain/db/state.go b/beacon-chain/db/state.go index ccce45ae57fe..fdc19601457c 100644 --- a/beacon-chain/db/state.go +++ b/beacon-chain/db/state.go @@ -445,7 +445,7 @@ func createState(enc []byte) (*pb.BeaconState, error) { } func (db *BeaconDB) deleteHistoricalStates(slot uint64) error { - if !featureconfig.FeatureConfig().EnableHistoricalStatePruning { + if featureconfig.FeatureConfig().DisableHistoricalStatePruning { return nil } return db.update(func(tx *bolt.Tx) error { diff --git a/beacon-chain/db/state_test.go b/beacon-chain/db/state_test.go index f28028aaac33..4a78b68e5e23 100644 --- a/beacon-chain/db/state_test.go +++ b/beacon-chain/db/state_test.go @@ -18,7 +18,7 @@ import ( func init() { featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{ - EnableHistoricalStatePruning: true, + DisableHistoricalStatePruning: false, }) } diff --git a/shared/featureconfig/config.go b/shared/featureconfig/config.go index 4e13ceec125e..4ebced1bb43e 100644 --- a/shared/featureconfig/config.go +++ b/shared/featureconfig/config.go @@ -25,14 +25,14 @@ var log = logrus.WithField("prefix", "flags") // FeatureFlagConfig is a struct to represent what features the client will perform on runtime. type FeatureFlagConfig struct { - VerifyAttestationSigs bool // VerifyAttestationSigs declares if the client will verify attestations. - EnableComputeStateRoot bool // EnableComputeStateRoot implementation on server side. - EnableCrosslinks bool // EnableCrosslinks in epoch processing. - EnableCheckBlockStateRoot bool // EnableCheckBlockStateRoot in block processing. - EnableHistoricalStatePruning bool // EnableHistoricalStatePruning when updating finalized states. - DisableGossipSub bool // DisableGossipSub in p2p messaging. - EnableCommitteesCache bool // EnableCommitteesCache for state transition. - CacheTreeHash bool // CacheTreeHash determent whether tree hashes will be cached. + VerifyAttestationSigs bool // VerifyAttestationSigs declares if the client will verify attestations. + EnableComputeStateRoot bool // EnableComputeStateRoot implementation on server side. + EnableCrosslinks bool // EnableCrosslinks in epoch processing. + EnableCheckBlockStateRoot bool // EnableCheckBlockStateRoot in block processing. + DisableHistoricalStatePruning bool // DisableHistoricalStatePruning when updating finalized states. + DisableGossipSub bool // DisableGossipSub in p2p messaging. + EnableCommitteesCache bool // EnableCommitteesCache for state transition. + CacheTreeHash bool // CacheTreeHash determent whether tree hashes will be cached. } var featureConfig *FeatureFlagConfig @@ -74,9 +74,9 @@ func ConfigureBeaconFeatures(ctx *cli.Context) { log.Info("Cache tree hashes for ssz") cfg.CacheTreeHash = true } - if ctx.GlobalBool(EnableHistoricalStatePruningFlag.Name) { + if ctx.GlobalBool(DisableHistoricalStatePruningFlag.Name) { log.Info("Enabled historical state pruning") - cfg.EnableHistoricalStatePruning = true + cfg.DisableHistoricalStatePruning = true } if ctx.GlobalBool(DisableGossipSubFlag.Name) { log.Info("Disabled gossipsub, using floodsub") diff --git a/shared/featureconfig/flags.go b/shared/featureconfig/flags.go index a1345abe97fd..13e39b154d6c 100644 --- a/shared/featureconfig/flags.go +++ b/shared/featureconfig/flags.go @@ -35,10 +35,10 @@ var ( Name: "enable-check-block-state-root", Usage: "Enable check block state root in block processing, default is disabled.", } - // EnableHistoricalStatePruningFlag allows the database to prune old historical states. - EnableHistoricalStatePruningFlag = cli.BoolFlag{ - Name: "enable-historical-state-pruning", - Usage: "Enable database pruning of historical states after finalized epochs", + // DisableHistoricalStatePruningFlag allows the database to keep old historical states. + DisableHistoricalStatePruningFlag = cli.BoolFlag{ + Name: "disable-historical-state-pruning", + Usage: "Disable database pruning of historical states after finalized epochs.", } // DisableGossipSubFlag uses floodsub in place of gossipsub. DisableGossipSubFlag = cli.BoolFlag{ @@ -57,7 +57,7 @@ var BeaconChainFlags = []cli.Flag{ EnableComputeStateRootFlag, EnableCrosslinksFlag, EnableCheckBlockStateRootFlag, - EnableHistoricalStatePruningFlag, + DisableHistoricalStatePruningFlag, DisableGossipSubFlag, CacheTreeHashFlag, } From a9d2003623957ac19d8419b64b9f99f90f95b4e4 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Mon, 29 Apr 2019 00:09:31 -0400 Subject: [PATCH 44/54] Add SSL support for prylabs.net, support opengraph on alpha.prylabs.net (#2419) --- k8s/beacon-chain/testnet-site.yaml | 20 ++++++++++++++++++++ k8s/letsencrypt-issuer.yaml | 3 ++- k8s/prylabs-wildcard-gateway.yaml | 2 ++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/k8s/beacon-chain/testnet-site.yaml b/k8s/beacon-chain/testnet-site.yaml index 9cb28412a9ef..ae0cc7ec8d1b 100644 --- a/k8s/beacon-chain/testnet-site.yaml +++ b/k8s/beacon-chain/testnet-site.yaml @@ -91,4 +91,24 @@ spec: port: number: 80 host: testnet-site-alpha.beacon-chain.svc.cluster.local +--- +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: prylabs-net + namespace: istio-system +spec: + hosts: + - prylabs.net + gateways: + - prylabs-wildcard-gateway + http: + - match: + - uri: + prefix: /assets + route: + - destination: + port: + number: 80 + host: testnet-site-alpha.beacon-chain.svc.cluster.local diff --git a/k8s/letsencrypt-issuer.yaml b/k8s/letsencrypt-issuer.yaml index bcae7479ba91..ae5eeef0aea1 100644 --- a/k8s/letsencrypt-issuer.yaml +++ b/k8s/letsencrypt-issuer.yaml @@ -29,10 +29,11 @@ spec: issuerRef: name: letsencrypt-prod commonName: "*.prylabs.net" + dnsNames: + - "prylabs.net" acme: config: - dns01: provider: cloud-dns domains: - - "*.prylabs.net" - "prylabs.net" diff --git a/k8s/prylabs-wildcard-gateway.yaml b/k8s/prylabs-wildcard-gateway.yaml index ba98262fe9c5..f7d69dde20bc 100644 --- a/k8s/prylabs-wildcard-gateway.yaml +++ b/k8s/prylabs-wildcard-gateway.yaml @@ -16,6 +16,7 @@ spec: protocol: HTTP hosts: - "*.prylabs.net" + - "prylabs.net" tls: httpsRedirect: false - port: @@ -24,6 +25,7 @@ spec: protocol: HTTPS hosts: - "*.prylabs.net" + - "prylabs.net" tls: mode: SIMPLE privateKey: /etc/istio/ingressgateway-certs/tls.key From 4a3b2f6d1585360c64063115b06b02ac8cb6c11d Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sun, 28 Apr 2019 23:56:36 -0700 Subject: [PATCH 45/54] Renovates Major Updates (#2434) * Update io_bazel_rules_k8s commit hash to c1daa04 * Update dependency build_bazel_rules_nodejs to v0.28.0 * Update dependency com_github_rjeczalik_notify to v0.9.2 * Update dependency io_opencensus_go to v0.21.0 * Update dependency org_golang_google_api to v0.4.0 * Update libp2p * Update dependency com_github_deckarep_golang_set to v1 * Update dependency com_github_edsrzf_mmap_go to v1 * Update dependency com_github_go_stack_stack to v1 * Update dependency com_github_karlseguin_ccache to v2 * Update dependency com_github_rs_cors to v1 * Update dependency in_gopkg_urfave_cli_v1 to v1 * Update prysm_testnet_site commit hash to 3848e91 * extra bracket * fix build --- WORKSPACE | 62 +++++++++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 8bce741d1f43..8551553ce2f6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,15 +2,15 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "bazel_skylib", - url = "https://github.com/bazelbuild/bazel-skylib/archive/0.8.0.tar.gz", sha256 = "2ea8a5ed2b448baf4a6855d3ce049c4c452a6470b1efd1504fdb7c1c134d220a", strip_prefix = "bazel-skylib-0.8.0", + url = "https://github.com/bazelbuild/bazel-skylib/archive/0.8.0.tar.gz", ) http_archive( name = "io_bazel_rules_go", - url = "https://github.com/bazelbuild/rules_go/releases/download/0.18.3/rules_go-0.18.3.tar.gz", sha256 = "86ae934bd4c43b99893fc64be9d9fc684b81461581df7ea8fc291c816f5ee8c5", + url = "https://github.com/bazelbuild/rules_go/releases/download/0.18.3/rules_go-0.18.3.tar.gz", ) http_archive( @@ -28,22 +28,22 @@ http_archive( http_archive( name = "io_bazel_rules_docker", - url = "https://github.com/bazelbuild/rules_docker/archive/v0.7.0.tar.gz", - strip_prefix = "rules_docker-0.7.0", sha256 = "aed1c249d4ec8f703edddf35cbe9dfaca0b5f5ea6e4cd9e83e99f3b0d1136c3d", + strip_prefix = "rules_docker-0.7.0", + url = "https://github.com/bazelbuild/rules_docker/archive/v0.7.0.tar.gz", ) http_archive( name = "build_bazel_rules_nodejs", - sha256 = "3a3efbf223f6de733475602844ad3a8faa02abda25ab8cfe1d1ed0db134887cf", - urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/0.27.12/rules_nodejs-0.27.12.tar.gz"], + sha256 = "4c702ffeeab2d24dd4101601b6d27cf582d2e0d4cdc3abefddd4834664669b6b", + urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/0.28.0/rules_nodejs-0.28.0.tar.gz"], ) http_archive( name = "io_bazel_rules_k8s", - sha256 = "b4aabc3f931a67b4970076ae9490afc18caa1690360c952fba9681225739f6d5", - strip_prefix = "rules_k8s-e7b408f07785e3b660ee7e99173b0e4328c7b65e", - url = "https://github.com/bazelbuild/rules_k8s/archive/e7b408f07785e3b660ee7e99173b0e4328c7b65e.tar.gz", + sha256 = "00606b1de959028881220e2cdc1186068242de491f8782e30922e8285b5f3ad1", + strip_prefix = "rules_k8s-c1daa04ce0a83fbaa4396fb4987d7675cd24f25e", + url = "https://github.com/bazelbuild/rules_k8s/archive/c1daa04ce0a83fbaa4396fb4987d7675cd24f25e.tar.gz", ) load( @@ -64,7 +64,7 @@ yarn_install( ) # This requires rules_docker to be fully instantiated before it is pulled in. -load("@io_bazel_rules_k8s//k8s:k8s.bzl", "k8s_repositories", "k8s_defaults") +load("@io_bazel_rules_k8s//k8s:k8s.bzl", "k8s_defaults", "k8s_repositories") k8s_repositories() @@ -88,7 +88,7 @@ k8s_repositories() "service_account", ]] -load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains") +load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") go_rules_dependencies() @@ -111,15 +111,15 @@ _go_image_repos() http_archive( name = "prysm_testnet_site", - url = "https://github.com/prestonvanloon/prysm-testnet-site/archive/3d0a15deb03aeefe0c69d4400a1f9486383fde27.tar.gz", - strip_prefix = "prysm-testnet-site-3d0a15deb03aeefe0c69d4400a1f9486383fde27", - sha256 = "f56cb21ab184ab2bfa7e8503ef1d3ec189bd2004aac9681012e6a58a0ba3f3e3", build_file_content = """ proto_library( name = "faucet_proto", srcs = ["src/proto/faucet.proto"], visibility = ["//visibility:public"], )""", + sha256 = "d57cbb11cbd8126d31385064315df731132401f5c5ea110e05ed86da350c9523", + strip_prefix = "prysm-testnet-site-3848e91aab9332a0b2c15291f65f09c94db1a84d", + url = "https://github.com/prestonvanloon/prysm-testnet-site/archive/3848e91aab9332a0b2c15291f65f09c94db1a84d.tar.gz", ) http_archive( @@ -182,7 +182,7 @@ go_repository( go_repository( name = "com_github_libp2p_go_libp2p", - commit = "5bce62bbecc0c07087af4cae2490b764afe1c594", # v0.0.15 + commit = "4d4f98f9ae7a78727e76c0cf5609bfa09e70715a", # v0.0.20 importpath = "github.com/libp2p/go-libp2p", ) @@ -213,13 +213,13 @@ go_repository( go_repository( name = "com_github_multiformats_go_multihash", - commit = "922cbd7915f407488e3f4413ada76400a1b29b49", # v0.0.3 + commit = "043d98569fdc31461b049f0c9fe267291fcbc087", # v0.0.5 importpath = "github.com/multiformats/go-multihash", ) go_repository( name = "com_github_libp2p_go_libp2p_swarm", - commit = "51da4b6c9c2e869dce87a455a7c788d7181711c7", # v0.0.2 + commit = "688f64125c469829608b4c5b75d500d93646604e", # v0.0.3 importpath = "github.com/libp2p/go-libp2p-swarm", ) @@ -231,7 +231,7 @@ go_repository( go_repository( name = "com_github_libp2p_go_libp2p_peerstore", - commit = "47258995923bfab02b60b8f45257642568b0a358", # v0.0.2 + commit = "2dd94506d962d6b86d9d730ff49b3d3965dce5d8", # v0.0.5 importpath = "github.com/libp2p/go-libp2p-peerstore", ) @@ -255,7 +255,7 @@ go_repository( go_repository( name = "com_github_libp2p_go_conn_security_multistream", - commit = "d4c3bf7e5688390e4106d3ec73aafd7d0856143b", # v0.0.1 + commit = "6b0762ae2e6aa1f4bdd58d96b26a67ad75961c2d", # v0.0.2 importpath = "github.com/libp2p/go-conn-security-multistream", ) @@ -340,7 +340,7 @@ go_repository( go_repository( name = "com_github_libp2p_go_libp2p_secio", build_file_proto_mode = "disable_global", - commit = "c3eee8a92db23bd65c434512810c60bec85e7ac9", # v0.0.1 + commit = "a1c285be4bd81a8ced3179064005e9e35ea4f495", # v0.0.3 importpath = "github.com/libp2p/go-libp2p-secio", ) @@ -412,7 +412,7 @@ go_repository( go_repository( name = "com_github_libp2p_go_libp2p_transport_upgrader", - commit = "b4ab689796cdae99ad6641abcedcd7c23258de0c", # v0.0.1 + commit = "ff9702e0f87f607f8f51c98b84962eb34c10fb3e", # v0.0.2 importpath = "github.com/libp2p/go-libp2p-transport-upgrader", ) @@ -532,7 +532,7 @@ go_repository( go_repository( name = "com_github_libp2p_go_msgio", - commit = "2efd92d62d528c56c6711346e82df0c74a5742a8", # v0.0.1 + commit = "d166a44c787c5308c27c8dae8b75dc7b182752b7", # v0.0.2 importpath = "github.com/libp2p/go-msgio", ) @@ -628,7 +628,7 @@ go_repository( go_repository( name = "io_opencensus_go", - commit = "bd64e5eff7498c1c67eba74dc87ad96aa525bf28", # v0.20.2 + commit = "fd90d04f12c9f72a32364672932b5a21005b8d7e", # v0.21.0 importpath = "go.opencensus.io", ) @@ -642,7 +642,7 @@ go_repository( go_repository( name = "org_golang_google_api", - commit = "0cbcb99a9ea0c8023c794b2693cbe1def82ed4d7", # v0.3.2 + commit = "067bed655e9cbc26f4dbac8f8897b30756d90990", # v0.4.0 importpath = "google.golang.org/api", ) @@ -758,7 +758,7 @@ go_repository( go_repository( name = "com_github_libp2p_go_libp2p_kad_dht", build_file_proto_mode = "disable_global", - commit = "b220ac23e82c0df788178e806b1fefa92e936766", # v0.0.9 + commit = "f279358375f2ce20df85ebdf2afa211055513b60", # v0.0.10 importpath = "github.com/libp2p/go-libp2p-kad-dht", ) @@ -961,19 +961,19 @@ go_repository( go_repository( name = "com_github_deckarep_golang_set", - commit = "699df6a3acf6867538e50931511e9dc403da108a", + commit = "cbaa98ba5575e67703b32b4b19f73c91f3c4159e", # v1.7.1 importpath = "github.com/deckarep/golang-set", ) go_repository( name = "com_github_go_stack_stack", - commit = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a", + commit = "f66e05c21cd224e01c8a3ee7bc867aa79439e207", # v1.8.0 importpath = "github.com/go-stack/stack", ) go_repository( name = "com_github_rs_cors", - commit = "76f58f330d76a55c5badc74f6212e8a15e742c77", + commit = "9a47f48565a795472d43519dd49aac781f3034fb", # v1.6.0 importpath = "github.com/rs/cors", ) @@ -985,19 +985,19 @@ go_repository( go_repository( name = "in_gopkg_urfave_cli_v1", - commit = "cfb38830724cc34fedffe9a2a29fb54fa9169cd1", + commit = "cfb38830724cc34fedffe9a2a29fb54fa9169cd1", # v1.20.0 importpath = "gopkg.in/urfave/cli.v1", ) go_repository( name = "com_github_rjeczalik_notify", - commit = "629144ba06a1c6af28c1e42c228e3d42594ce081", + commit = "69d839f37b13a8cb7a78366f7633a4071cb43be7", # v0.9.2 importpath = "github.com/rjeczalik/notify", ) go_repository( name = "com_github_edsrzf_mmap_go", - commit = "904c4ced31cdffe19e971afa0b3d319ff06d9c72", + commit = "188cc3b666ba704534fa4f96e9e61f21f1e1ba7c", # v1.0.0 importpath = "github.com/edsrzf/mmap-go", ) From e16f1e1533ac8d75a18fd546a083eaedc7809eb5 Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Mon, 29 Apr 2019 21:23:24 +0800 Subject: [PATCH 46/54] Reverts RPC Disabling (#2435) * make changes * more changes --- beacon-chain/node/node.go | 6 ------ beacon-chain/rpc/service.go | 10 ---------- beacon-chain/rpc/service_test.go | 20 +++++--------------- beacon-chain/sync/service.go | 18 ++++++------------ 4 files changed, 11 insertions(+), 43 deletions(-) diff --git a/beacon-chain/node/node.go b/beacon-chain/node/node.go index 4ffd1fdd5dd2..1d7479a7f310 100644 --- a/beacon-chain/node/node.go +++ b/beacon-chain/node/node.go @@ -348,11 +348,6 @@ func (b *BeaconNode) registerRPCService(ctx *cli.Context) error { return err } - var syncService *rbcsync.Service - if err := b.services.FetchService(&syncService); err != nil { - return err - } - port := ctx.GlobalString(utils.RPCPort.Name) cert := ctx.GlobalString(utils.CertFlag.Name) key := ctx.GlobalString(utils.KeyFlag.Name) @@ -364,7 +359,6 @@ func (b *BeaconNode) registerRPCService(ctx *cli.Context) error { ChainService: chainService, OperationService: operationService, POWChainService: web3Service, - SyncService: syncService, }) return b.services.RegisterService(rpcService) diff --git a/beacon-chain/rpc/service.go b/beacon-chain/rpc/service.go index 135260b1c75f..a67176a0336d 100644 --- a/beacon-chain/rpc/service.go +++ b/beacon-chain/rpc/service.go @@ -55,10 +55,6 @@ type powChainService interface { ChainStartDeposits() [][]byte } -type syncService interface { - Status() error -} - // Service defining an RPC server for a beacon node. type Service struct { ctx context.Context @@ -67,7 +63,6 @@ type Service struct { chainService chainService powChainService powChainService operationService operationService - syncService syncService port string listener net.Listener withCert string @@ -87,7 +82,6 @@ type Config struct { ChainService chainService POWChainService powChainService OperationService operationService - SyncService syncService } // NewRPCService creates a new instance of a struct implementing the BeaconServiceServer @@ -101,7 +95,6 @@ func NewRPCService(ctx context.Context, cfg *Config) *Service { chainService: cfg.ChainService, powChainService: cfg.POWChainService, operationService: cfg.OperationService, - syncService: cfg.SyncService, port: cfg.Port, withCert: cfg.CertFlag, withKey: cfg.KeyFlag, @@ -181,9 +174,6 @@ func (s *Service) Start() { reflection.Register(s.grpcServer) go func() { - for s.syncService.Status() != nil { - time.Sleep(time.Second * params.BeaconConfig().RPCSyncCheck) - } if s.listener != nil { if err := s.grpcServer.Serve(s.listener); err != nil { log.Errorf("Could not serve gRPC: %v", err) diff --git a/beacon-chain/rpc/service_test.go b/beacon-chain/rpc/service_test.go index 722b56f1ad6c..7cd09f2ff0d1 100644 --- a/beacon-chain/rpc/service_test.go +++ b/beacon-chain/rpc/service_test.go @@ -121,20 +121,12 @@ func newMockChainService() *mockChainService { } } -type mockSyncService struct { -} - -func (ms *mockSyncService) Status() error { - return nil -} - func TestLifecycle_OK(t *testing.T) { hook := logTest.NewGlobal() rpcService := NewRPCService(context.Background(), &Config{ - Port: "7348", - CertFlag: "alice.crt", - KeyFlag: "alice.key", - SyncService: &mockSyncService{}, + Port: "7348", + CertFlag: "alice.crt", + KeyFlag: "alice.key", }) rpcService.Start() @@ -158,8 +150,7 @@ func TestRPC_BadEndpoint(t *testing.T) { hook := logTest.NewLocal(fl.Logger) rpcService := NewRPCService(context.Background(), &Config{ - Port: "ralph merkle!!!", - SyncService: &mockSyncService{}, + Port: "ralph merkle!!!", }) if val, ok := log.(*TestLogger).testMap["error"]; ok { @@ -188,8 +179,7 @@ func TestStatus_CredentialError(t *testing.T) { func TestRPC_InsecureEndpoint(t *testing.T) { hook := logTest.NewGlobal() rpcService := NewRPCService(context.Background(), &Config{ - Port: "7777", - SyncService: &mockSyncService{}, + Port: "7777", }) rpcService.Start() diff --git a/beacon-chain/sync/service.go b/beacon-chain/sync/service.go index 2b3082402650..d98bbc252987 100644 --- a/beacon-chain/sync/service.go +++ b/beacon-chain/sync/service.go @@ -17,10 +17,9 @@ var slog = logrus.WithField("prefix", "sync") // Service defines the main routines used in the sync service. type Service struct { - RegularSync *RegularSync - InitialSync *initialsync.InitialSync - Querier *Querier - querierFinished bool + RegularSync *RegularSync + InitialSync *initialsync.InitialSync + Querier *Querier } // Config defines the configured services required for sync to work. @@ -62,10 +61,9 @@ func NewSyncService(ctx context.Context, cfg *Config) *Service { is := initialsync.NewInitialSyncService(ctx, isCfg) return &Service{ - RegularSync: rs, - InitialSync: is, - Querier: sq, - querierFinished: false, + RegularSync: rs, + InitialSync: is, + Querier: sq, } } @@ -94,9 +92,6 @@ func (ss *Service) Stop() error { // Status checks the status of the node. It returns nil if it's synced // with the rest of the network and no errors occurred. Otherwise, it returns an error. func (ss *Service) Status() error { - if !ss.querierFinished { - return errors.New("querier is still running") - } if !ss.Querier.chainStarted { return nil @@ -125,7 +120,6 @@ func (ss *Service) run() { if err != nil { slog.Fatalf("Unable to retrieve result from sync querier %v", err) } - ss.querierFinished = true // Sets the highest observed slot from querier. ss.InitialSync.InitializeObservedSlot(ss.Querier.currentHeadSlot) From 4a90f82bcc8ee20b6f3e777371d0dc04a221f4a0 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 29 Apr 2019 07:39:31 -0700 Subject: [PATCH 47/54] Fix Validator Ejection for Testnet (#2438) * set ejection threshold and add gauge * spaces * gaz --- beacon-chain/core/epoch/BUILD.bazel | 2 ++ beacon-chain/core/epoch/epoch_processing.go | 17 ++++++++++++++++- shared/params/config.go | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/beacon-chain/core/epoch/BUILD.bazel b/beacon-chain/core/epoch/BUILD.bazel index d81115fe1228..09f0a34c0a37 100644 --- a/beacon-chain/core/epoch/BUILD.bazel +++ b/beacon-chain/core/epoch/BUILD.bazel @@ -18,6 +18,8 @@ go_library( "//shared/hashutil:go_default_library", "//shared/mathutil:go_default_library", "//shared/params:go_default_library", + "@com_github_prometheus_client_golang//prometheus:go_default_library", + "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", ], ) diff --git a/beacon-chain/core/epoch/epoch_processing.go b/beacon-chain/core/epoch/epoch_processing.go index fbc766bd0583..a37d209bde41 100644 --- a/beacon-chain/core/epoch/epoch_processing.go +++ b/beacon-chain/core/epoch/epoch_processing.go @@ -8,8 +8,9 @@ import ( "encoding/binary" "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks" - "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" "github.com/prysmaticlabs/prysm/beacon-chain/core/validators" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" @@ -22,6 +23,14 @@ import ( var log = logrus.WithField("prefix", "core/state") +var ( + ejectedCount float64 + validatorEjectedGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "validator_ejected_count", + Help: "Total number of ejected validators", + }) +) + // CanProcessEpoch checks the eligibility to process epoch. // The epoch can be processed at the end of the last slot of every epoch // @@ -260,6 +269,12 @@ func ProcessEjections(state *pb.BeaconState, enableLogging bool) (*pb.BeaconStat "index": index}).Info("Validator ejected") } state = validators.ExitValidator(state, index) + // Verify the validator has properly exited due to ejection before setting the + // ejection count for gauge. + if state.ValidatorRegistry[index].ExitEpoch != params.BeaconConfig().FarFutureEpoch { + ejectedCount++ + validatorEjectedGauge.Set(ejectedCount) + } } } return state, nil diff --git a/shared/params/config.go b/shared/params/config.go index 6fb268963154..264820970f20 100644 --- a/shared/params/config.go +++ b/shared/params/config.go @@ -214,7 +214,7 @@ func DemoBeaconConfig() *BeaconChainConfig { demoConfig.GenesisEpoch = demoConfig.GenesisSlot / 8 demoConfig.MinDepositAmount = 100 demoConfig.MaxDepositAmount = 3.2 * 1e9 - demoConfig.EjectionBalance = 1600000 + demoConfig.EjectionBalance = 3 * 1e9 demoConfig.SyncPollingInterval = 1 * 10 // Query nodes over the network every slot. demoConfig.Eth1FollowDistance = 5 demoConfig.EpochsPerEth1VotingPeriod = 1 From c288af5e9b8c3f061e140d86c4c02881d1614b9a Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Mon, 29 Apr 2019 22:54:55 +0800 Subject: [PATCH 48/54] Continue Processing Invalid Deposits (#2437) * fix for pending deposits * add regression tests --- beacon-chain/core/blocks/BUILD.bazel | 2 + beacon-chain/core/blocks/block_operations.go | 15 +- .../core/blocks/block_operations_test.go | 191 ++++++++++++++++-- beacon-chain/core/validators/validator.go | 6 +- 4 files changed, 196 insertions(+), 18 deletions(-) diff --git a/beacon-chain/core/blocks/BUILD.bazel b/beacon-chain/core/blocks/BUILD.bazel index e2b9f0513682..ff81da89950b 100644 --- a/beacon-chain/core/blocks/BUILD.bazel +++ b/beacon-chain/core/blocks/BUILD.bazel @@ -46,10 +46,12 @@ go_test( "//shared/hashutil:go_default_library", "//shared/params:go_default_library", "//shared/ssz:go_default_library", + "//shared/testutil:go_default_library", "//shared/trieutil:go_default_library", "@com_github_ethereum_go_ethereum//common:go_default_library", "@com_github_ethereum_go_ethereum//core/types:go_default_library", "@com_github_gogo_protobuf//proto:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", + "@com_github_sirupsen_logrus//hooks/test:go_default_library", ], ) diff --git a/beacon-chain/core/blocks/block_operations.go b/beacon-chain/core/blocks/block_operations.go index 416cb91a0c0d..1d522a66608e 100644 --- a/beacon-chain/core/blocks/block_operations.go +++ b/beacon-chain/core/blocks/block_operations.go @@ -578,7 +578,9 @@ func ProcessValidatorDeposits( depositData := deposit.DepositData depositInput, err = helpers.DecodeDepositInput(depositData) if err != nil { - return nil, fmt.Errorf("could not decode deposit input: %v", err) + beaconState = processInvalidDeposit(beaconState) + log.Errorf("could not decode deposit input: %v", err) + continue } if err = verifyDeposit(beaconState, deposit); err != nil { return nil, fmt.Errorf("could not verify deposit #%d: %v", idx, err) @@ -596,7 +598,9 @@ func ProcessValidatorDeposits( depositInput.WithdrawalCredentialsHash32, ) if err != nil { - return nil, fmt.Errorf("could not process deposit into beacon state: %v", err) + beaconState = processInvalidDeposit(beaconState) + log.Errorf("could not process deposit into beacon state: %v", err) + continue } } return beaconState, nil @@ -629,6 +633,13 @@ func verifyDeposit(beaconState *pb.BeaconState, deposit *pb.Deposit) error { return nil } +// we increase the state deposit index, since deposits have to be processed +// in order even if they are invalid +func processInvalidDeposit(bState *pb.BeaconState) *pb.BeaconState { + bState.DepositIndex++ + return bState +} + // ProcessValidatorExits is one of the operations performed // on each processed beacon block to determine which validators // should exit the state's validator registry. diff --git a/beacon-chain/core/blocks/block_operations_test.go b/beacon-chain/core/blocks/block_operations_test.go index afc7da53e726..c55415d353b5 100644 --- a/beacon-chain/core/blocks/block_operations_test.go +++ b/beacon-chain/core/blocks/block_operations_test.go @@ -19,6 +19,9 @@ import ( "github.com/prysmaticlabs/prysm/shared/forkutil" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/ssz" + "github.com/prysmaticlabs/prysm/shared/testutil" + logTest "github.com/sirupsen/logrus/hooks/test" + "github.com/prysmaticlabs/prysm/shared/trieutil" ) @@ -1016,6 +1019,7 @@ func TestProcessValidatorDeposits_ThresholdReached(t *testing.T) { } func TestProcessValidatorDeposits_DepositDataSizeTooSmall(t *testing.T) { + hook := logTest.NewGlobal() data := []byte{1, 2, 3} deposit := &pb.Deposit{ DepositData: data, @@ -1027,16 +1031,15 @@ func TestProcessValidatorDeposits_DepositDataSizeTooSmall(t *testing.T) { } beaconState := &pb.BeaconState{} want := "deposit data slice too small" - if _, err := blocks.ProcessValidatorDeposits( - + _, _ = blocks.ProcessValidatorDeposits( beaconState, block, - ); !strings.Contains(err.Error(), want) { - t.Errorf("Expected error: %s, received %v", want, err) - } + ) + testutil.AssertLogsContain(t, hook, want) } func TestProcessValidatorDeposits_DepositInputDecodingFails(t *testing.T) { + hook := logTest.NewGlobal() data := make([]byte, 16) deposit := &pb.Deposit{ DepositData: data, @@ -1048,13 +1051,12 @@ func TestProcessValidatorDeposits_DepositInputDecodingFails(t *testing.T) { } beaconState := &pb.BeaconState{} want := "ssz decode failed" - if _, err := blocks.ProcessValidatorDeposits( + _, _ = blocks.ProcessValidatorDeposits( beaconState, block, - ); !strings.Contains(err.Error(), want) { - t.Errorf("Expected error: %s, received %v", want, err) - } + ) + testutil.AssertLogsContain(t, hook, want) } func TestProcessValidatorDeposits_MerkleBranchFailsVerification(t *testing.T) { @@ -1111,6 +1113,7 @@ func TestProcessValidatorDeposits_MerkleBranchFailsVerification(t *testing.T) { } func TestProcessValidatorDeposits_ProcessDepositHelperFuncFails(t *testing.T) { + hook := logTest.NewGlobal() // Having mismatched withdrawal credentials will cause the process deposit // validator helper function to fail with error when the public key // currently exists in the validator registry. @@ -1188,13 +1191,12 @@ func TestProcessValidatorDeposits_ProcessDepositHelperFuncFails(t *testing.T) { GenesisTime: uint64(genesisTime), } want := "expected withdrawal credentials to match" - if _, err := blocks.ProcessValidatorDeposits( + _, err = blocks.ProcessValidatorDeposits( beaconState, block, - ); !strings.Contains(err.Error(), want) { - t.Errorf("Expected error: %s, received %v", want, err) - } + ) + testutil.AssertLogsContain(t, hook, want) } func TestProcessValidatorDeposits_IncorrectMerkleIndex(t *testing.T) { @@ -1357,6 +1359,169 @@ func TestProcessValidatorDeposits_ProcessCorrectly(t *testing.T) { } } +func TestProcessValidatorDeposits_InvalidSSZ_DepositIndexIncremented(t *testing.T) { + hook := logTest.NewGlobal() + encodedInput := []byte{'A', 'B', 'C', 'D'} + data := []byte{} + + // We set a deposit value of 1000. + value := make([]byte, 8) + depositValue := uint64(1000) + binary.LittleEndian.PutUint64(value, depositValue) + + // We then serialize a unix time into the timestamp []byte slice + // and ensure it has size of 8 bytes. + timestamp := make([]byte, 8) + + // Set deposit time to 1000 seconds since unix time 0. + depositTime := time.Unix(1000, 0).Unix() + // Set genesis time to unix time 0. + genesisTime := time.Unix(0, 0).Unix() + + currentSlot := 1000 * params.BeaconConfig().SecondsPerSlot + binary.LittleEndian.PutUint64(timestamp, uint64(depositTime)) + + // We then create a serialized deposit data slice of type []byte + // by appending all 3 items above together. + data = append(data, value...) + data = append(data, timestamp...) + data = append(data, encodedInput...) + + deposit := &pb.Deposit{ + DepositData: data, + MerkleTreeIndex: 0, + } + block := &pb.BeaconBlock{ + Body: &pb.BeaconBlockBody{ + Deposits: []*pb.Deposit{deposit}, + }, + } + registry := []*pb.Validator{ + { + Pubkey: []byte{1}, + WithdrawalCredentialsHash32: []byte{1, 2, 3}, + }, + } + balances := []uint64{0} + beaconState := &pb.BeaconState{ + ValidatorRegistry: registry, + ValidatorBalances: balances, + Slot: currentSlot, + GenesisTime: uint64(genesisTime), + DepositIndex: 1, + } + newState, err := blocks.ProcessValidatorDeposits( + + beaconState, + block, + ) + if err != nil { + t.Fatalf("Expected no errors returned but received this %v", err) + } + if newState.DepositIndex != 2 { + t.Errorf( + "Expected state deposit index to equal %d, received %d", + 2, + newState.DepositIndex, + ) + } + testutil.AssertLogsContain(t, hook, "could not decode deposit input") +} + +func TestProcessValidatorDeposits_InvalidWithdrawalCreds_DepositIndexIncremented(t *testing.T) { + hook := logTest.NewGlobal() + depositInput := &pb.DepositInput{ + Pubkey: []byte{1}, + WithdrawalCredentialsHash32: []byte{3, 2, 1}, + ProofOfPossession: []byte{}, + } + wBuf := new(bytes.Buffer) + if err := ssz.Encode(wBuf, depositInput); err != nil { + t.Fatalf("failed to encode deposit input: %v", err) + } + encodedInput := wBuf.Bytes() + data := []byte{} + + // We set a deposit value of 1000. + value := make([]byte, 8) + depositValue := uint64(1000) + binary.LittleEndian.PutUint64(value, depositValue) + + // We then serialize a unix time into the timestamp []byte slice + // and ensure it has size of 8 bytes. + timestamp := make([]byte, 8) + + // Set deposit time to 1000 seconds since unix time 0. + depositTime := time.Unix(1000, 0).Unix() + // Set genesis time to unix time 0. + genesisTime := time.Unix(0, 0).Unix() + + currentSlot := 1000 * params.BeaconConfig().SecondsPerSlot + binary.LittleEndian.PutUint64(timestamp, uint64(depositTime)) + + // We then create a serialized deposit data slice of type []byte + // by appending all 3 items above together. + data = append(data, value...) + data = append(data, timestamp...) + data = append(data, encodedInput...) + + // We then create a merkle branch for the test. + depositTrie, err := trieutil.GenerateTrieFromItems([][]byte{data}, int(params.BeaconConfig().DepositContractTreeDepth)) + if err != nil { + t.Fatalf("Could not generate trie: %v", err) + } + proof, err := depositTrie.MerkleProof(0) + if err != nil { + t.Fatalf("Could not generate proof: %v", err) + } + + deposit := &pb.Deposit{ + DepositData: data, + MerkleProofHash32S: proof, + MerkleTreeIndex: 0, + } + block := &pb.BeaconBlock{ + Body: &pb.BeaconBlockBody{ + Deposits: []*pb.Deposit{deposit}, + }, + } + registry := []*pb.Validator{ + { + Pubkey: []byte{1}, + WithdrawalCredentialsHash32: []byte{1, 2, 3}, + }, + } + balances := []uint64{0} + root := depositTrie.Root() + beaconState := &pb.BeaconState{ + ValidatorRegistry: registry, + ValidatorBalances: balances, + DepositIndex: 0, + LatestEth1Data: &pb.Eth1Data{ + DepositRootHash32: root[:], + BlockHash32: root[:], + }, + Slot: currentSlot, + GenesisTime: uint64(genesisTime), + } + newState, err := blocks.ProcessValidatorDeposits( + beaconState, + block, + ) + if err != nil { + t.Fatalf("Expected no error returned from processing the deposit: %v", err) + } + if newState.DepositIndex != 1 { + t.Errorf( + "Expected state deposit index to equal %d, received %d", + 1, + newState.DepositIndex, + ) + } + + testutil.AssertLogsContain(t, hook, "could not process deposit into beacon state") +} + func TestProcessValidatorExits_ThresholdReached(t *testing.T) { exits := make([]*pb.VoluntaryExit, params.BeaconConfig().MaxVoluntaryExits+1) registry := []*pb.Validator{} diff --git a/beacon-chain/core/validators/validator.go b/beacon-chain/core/validators/validator.go index 65f36a77cad9..3c2dfd9c7769 100644 --- a/beacon-chain/core/validators/validator.go +++ b/beacon-chain/core/validators/validator.go @@ -111,8 +111,6 @@ func ProcessDeposit( var publicKeyExists bool var existingValidatorIdx int - state.DepositIndex++ - existingValidatorIdx, publicKeyExists = validatorIdxMap[bytesutil.ToBytes32(pubkey)] if !publicKeyExists { // If public key does not exist in the registry, we add a new validator @@ -133,7 +131,7 @@ func ProcessDeposit( state.ValidatorRegistry[existingValidatorIdx].WithdrawalCredentialsHash32, withdrawalCredentials, ) { - return nil, fmt.Errorf( + return state, fmt.Errorf( "expected withdrawal credentials to match, received %#x == %#x", state.ValidatorRegistry[existingValidatorIdx].WithdrawalCredentialsHash32, withdrawalCredentials, @@ -141,6 +139,8 @@ func ProcessDeposit( } state.ValidatorBalances[existingValidatorIdx] += amount } + state.DepositIndex++ + return state, nil } From 63998b92aaa10163861ae0751766f85018a17990 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 29 Apr 2019 11:11:52 -0700 Subject: [PATCH 49/54] Add and Use Truncate Hash Library (#2417) * add truncate lib * use trunc * spaces --- beacon-chain/attestation/service.go | 10 +++++----- beacon-chain/blockchain/fork_choice.go | 2 +- beacon-chain/core/state/BUILD.bazel | 1 + beacon-chain/core/state/transition.go | 7 ++++--- beacon-chain/rpc/beacon_server.go | 2 +- beacon-chain/rpc/proposer_server.go | 6 ++++-- beacon-chain/sync/receive_block.go | 15 ++++++++------- beacon-chain/sync/regular_sync.go | 10 +++++----- shared/bytesutil/bytes.go | 8 ++++++++ shared/bytesutil/bytes_test.go | 19 +++++++++++++++++++ 10 files changed, 56 insertions(+), 24 deletions(-) diff --git a/beacon-chain/attestation/service.go b/beacon-chain/attestation/service.go index 570112cc6a38..e405cb51b6a5 100644 --- a/beacon-chain/attestation/service.go +++ b/beacon-chain/attestation/service.go @@ -258,11 +258,11 @@ func (a *Service) updateAttestation(ctx context.Context, headRoot [32]byte, beac } log.WithFields(logrus.Fields{ - "attestation slot": attestation.Data.Slot - params.BeaconConfig().GenesisSlot, - "attestation shard": attestation.Data.Shard, - "committees shard": cachedCommittees.Committees[0].Shard, - "committees list": cachedCommittees.Committees[0].Committee, - "length of committees": len(cachedCommittees.Committees), + "attestationSlot": attestation.Data.Slot - params.BeaconConfig().GenesisSlot, + "attestationShard": attestation.Data.Shard, + "committeesShard": cachedCommittees.Committees[0].Shard, + "committeesList": cachedCommittees.Committees[0].Committee, + "lengthOfCommittees": len(cachedCommittees.Committees), }).Debug("Updating latest attestation") // The participation bitfield from attestation is represented in bytes, diff --git a/beacon-chain/blockchain/fork_choice.go b/beacon-chain/blockchain/fork_choice.go index 3486d43df866..b0eb89c5ec7b 100644 --- a/beacon-chain/blockchain/fork_choice.go +++ b/beacon-chain/blockchain/fork_choice.go @@ -196,7 +196,7 @@ func (c *ChainService) ApplyForkChoiceRule( return fmt.Errorf("could not hash head: %v", err) } log.WithFields(logrus.Fields{ - "headRoot": fmt.Sprintf("0x%x", h), + "headRoot": fmt.Sprintf("#%x", bytesutil.Trunc(h[:])), }).Info("Chain head block and state updated") return nil diff --git a/beacon-chain/core/state/BUILD.bazel b/beacon-chain/core/state/BUILD.bazel index c6cb6461ee85..df4ae5f04b85 100644 --- a/beacon-chain/core/state/BUILD.bazel +++ b/beacon-chain/core/state/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//beacon-chain/core/state/stateutils:go_default_library", "//beacon-chain/core/validators:go_default_library", "//proto/beacon/p2p/v1:go_default_library", + "//shared/bytesutil:go_default_library", "//shared/featureconfig:go_default_library", "//shared/hashutil:go_default_library", "//shared/params:go_default_library", diff --git a/beacon-chain/core/state/transition.go b/beacon-chain/core/state/transition.go index a5f03838e99f..ce3f35179e77 100644 --- a/beacon-chain/core/state/transition.go +++ b/beacon-chain/core/state/transition.go @@ -14,6 +14,7 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" v "github.com/prysmaticlabs/prysm/beacon-chain/core/validators" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/bytesutil" "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/params" @@ -172,9 +173,9 @@ func ProcessBlock( } if config.Logging { - log.WithField("blockRoot", fmt.Sprintf("%#x", r)).Debugf("Verified block slot == state slot") - log.WithField("blockRoot", fmt.Sprintf("%#x", r)).Debugf("Verified and processed block RANDAO") - log.WithField("blockRoot", fmt.Sprintf("%#x", r)).Debugf("Processed ETH1 data") + log.WithField("blockRoot", fmt.Sprintf("%#x", bytesutil.Trunc(r[:]))).Debugf("Verified block slot == state slot") + log.WithField("blockRoot", fmt.Sprintf("%#x", bytesutil.Trunc(r[:]))).Debugf("Verified and processed block RANDAO") + log.WithField("blockRoot", fmt.Sprintf("%#x", bytesutil.Trunc(r[:]))).Debugf("Processed ETH1 data") log.WithField( "attestationsInBlock", len(block.Body.Attestations), ).Info("Block attestations") diff --git a/beacon-chain/rpc/beacon_server.go b/beacon-chain/rpc/beacon_server.go index 2edcce525d18..7536ed3be102 100644 --- a/beacon-chain/rpc/beacon_server.go +++ b/beacon-chain/rpc/beacon_server.go @@ -139,7 +139,7 @@ func (bs *BeaconServer) Eth1Data(ctx context.Context, _ *ptypes.Empty) (*pb.Eth1 // Verify the block from the vote's block hash exists in the eth1.0 chain and fetch its height. blockExists, blockHeight, err := bs.powChainService.BlockExists(ctx, eth1Hash) if err != nil { - log.WithError(err).WithField("blockRoot", fmt.Sprintf("%#x", eth1Hash)). + log.WithError(err).WithField("blockRoot", fmt.Sprintf("%#x", bytesutil.Trunc(eth1Hash[:]))). Debug("Could not verify block with hash in ETH1 chain") continue } diff --git a/beacon-chain/rpc/proposer_server.go b/beacon-chain/rpc/proposer_server.go index 8e2da5554f6a..f6d2d3b6f8a4 100644 --- a/beacon-chain/rpc/proposer_server.go +++ b/beacon-chain/rpc/proposer_server.go @@ -73,7 +73,8 @@ func (ps *ProposerServer) ProposeBlock(ctx context.Context, blk *pbp2p.BeaconBlo if err != nil { return nil, fmt.Errorf("could not tree hash block: %v", err) } - log.WithField("blockRoot", fmt.Sprintf("%#x", h)).Debugf("Block proposal received via RPC") + log.WithField("blockRoot", fmt.Sprintf("%#x", bytesutil.Trunc(h[:]))).Debugf( + "Block proposal received via RPC") beaconState, err := ps.chainService.ReceiveBlock(ctx, blk) if err != nil { return nil, fmt.Errorf("could not process beacon block: %v", err) @@ -81,7 +82,8 @@ func (ps *ProposerServer) ProposeBlock(ctx context.Context, blk *pbp2p.BeaconBlo if err := ps.beaconDB.UpdateChainHead(ctx, blk, beaconState); err != nil { return nil, fmt.Errorf("failed to update chain: %v", err) } - log.WithField("headRoot", fmt.Sprintf("0x%x", h)).Info("Chain head block and state updated") + log.WithField("headRoot", fmt.Sprintf("0x%x", bytesutil.Trunc(h[:]))).Info( + "Chain head block and state updated") if err := ps.beaconDB.SaveHistoricalState(ctx, beaconState); err != nil { log.Errorf("Could not save new historical state: %v", err) diff --git a/beacon-chain/sync/receive_block.go b/beacon-chain/sync/receive_block.go index bf1fd3067255..de045e1b022f 100644 --- a/beacon-chain/sync/receive_block.go +++ b/beacon-chain/sync/receive_block.go @@ -28,7 +28,7 @@ func (rs *RegularSync) receiveBlockAnnounce(msg p2p.Message) error { span.AddAttributes(trace.BoolAttribute("isEvilBlock", isEvilBlock)) if isEvilBlock { - log.WithField("blockRoot", fmt.Sprintf("%#x", h)). + log.WithField("blockRoot", fmt.Sprintf("%#x", bytesutil.Trunc(h[:]))). Debug("Received blacklisted block") return nil } @@ -45,11 +45,11 @@ func (rs *RegularSync) receiveBlockAnnounce(msg p2p.Message) error { span.AddAttributes(trace.BoolAttribute("hasBlock", hasBlock)) if hasBlock { - log.WithField("blockRoot", fmt.Sprintf("%#x", h)).Debug("Already processed") + log.WithField("blockRoot", fmt.Sprintf("%#x", bytesutil.Trunc(h[:]))).Debug("Already processed") return nil } - log.WithField("blockRoot", fmt.Sprintf("%#x", h)).Debug("Received incoming block root, requesting full block data from sender") + log.WithField("blockRoot", fmt.Sprintf("%#x", bytesutil.Trunc(h[:]))).Debug("Received incoming block root, requesting full block data from sender") // Request the full block data from peer that sent the block hash. if err := rs.p2p.Send(ctx, &pb.BeaconBlockRequest{Hash: h[:]}, msg.Peer); err != nil { log.Error(err) @@ -91,7 +91,7 @@ func (rs *RegularSync) processBlockAndFetchAncestors(ctx context.Context, msg p2 } if rs.db.IsEvilBlockHash(blockRoot) { - log.WithField("blockHash", blockRoot).Debug("Skipping blacklisted block") + log.WithField("blockRoot", bytesutil.Trunc(blockRoot[:])).Debug("Skipping blacklisted block") return nil } @@ -121,7 +121,7 @@ func (rs *RegularSync) validateAndProcessBlock( return nil, nil, false, err } - log.WithField("blockRoot", fmt.Sprintf("%#x", blockRoot)). + log.WithField("blockRoot", fmt.Sprintf("%#x", bytesutil.Trunc(blockRoot[:]))). Debug("Processing response to block request") hasBlock := rs.db.HasBlock(blockRoot) if hasBlock { @@ -163,7 +163,8 @@ func (rs *RegularSync) validateAndProcessBlock( return nil, nil, false, nil } - log.WithField("blockRoot", fmt.Sprintf("%#x", blockRoot)).Debug("Sending newly received block to chain service") + log.WithField("blockRoot", fmt.Sprintf("%#x", bytesutil.Trunc(blockRoot[:]))).Debug( + "Sending newly received block to chain service") // We then process the block by passing it through the ChainService and running // a fork choice rule. beaconState, err = rs.chainService.ReceiveBlock(ctx, block) @@ -190,7 +191,7 @@ func (rs *RegularSync) validateAndProcessBlock( forkedBlock.Inc() log.WithFields(logrus.Fields{ "slot": block.Slot, - "root": fmt.Sprintf("%#x", blockRoot)}, + "root": fmt.Sprintf("%#x", bytesutil.Trunc(blockRoot[:]))}, ).Warn("Received Block from a forked chain") if err := rs.db.SaveHistoricalState(ctx, beaconState); err != nil { log.Errorf("Could not save historical state %v", err) diff --git a/beacon-chain/sync/regular_sync.go b/beacon-chain/sync/regular_sync.go index b62a188d70d1..63e45580f6be 100644 --- a/beacon-chain/sync/regular_sync.go +++ b/beacon-chain/sync/regular_sync.go @@ -419,7 +419,7 @@ func (rs *RegularSync) receiveAttestation(msg p2p.Message) error { return err } log.WithFields(logrus.Fields{ - "headRoot": fmt.Sprintf("%#x", attestation.Data.BeaconBlockRootHash32), + "headRoot": fmt.Sprintf("%#x", bytesutil.Trunc(attestation.Data.BeaconBlockRootHash32)), "justifiedEpoch": attestation.Data.JustifiedEpoch - params.BeaconConfig().GenesisEpoch, }).Debug("Received an attestation") @@ -427,7 +427,7 @@ func (rs *RegularSync) receiveAttestation(msg p2p.Message) error { hasAttestation := rs.db.HasAttestation(attestationRoot) span.AddAttributes(trace.BoolAttribute("hasAttestation", hasAttestation)) if hasAttestation { - log.WithField("attestationRoot", fmt.Sprintf("%#x", attestationRoot)). + log.WithField("attestationRoot", fmt.Sprintf("%#x", bytesutil.Trunc(attestationRoot[:]))). Debug("Received, skipping attestation") return nil } @@ -596,13 +596,13 @@ func (rs *RegularSync) handleAttestationRequestByHash(msg p2p.Message) error { } span.AddAttributes(trace.BoolAttribute("hasAttestation", att == nil)) if att == nil { - log.WithField("attestationRoot", fmt.Sprintf("%#x", root)). + log.WithField("attestationRoot", fmt.Sprintf("%#x", bytesutil.Trunc(root[:]))). Debug("Attestation not in db") return nil } log.WithFields(logrus.Fields{ - "attestationRoot": fmt.Sprintf("%#x", root), + "attestationRoot": fmt.Sprintf("%#x", bytesutil.Trunc(root[:])), "peer": msg.Peer}, ).Debug("Sending attestation to peer") if err := rs.p2p.Send(ctx, &pb.AttestationResponse{ @@ -649,7 +649,7 @@ func (rs *RegularSync) handleAttestationAnnouncement(msg p2p.Message) error { func (rs *RegularSync) broadcastCanonicalBlock(ctx context.Context, announce *pb.BeaconBlockAnnounce) { ctx, span := trace.StartSpan(ctx, "beacon-chain.sync.broadcastCanonicalBlock") defer span.End() - log.WithField("blockRoot", fmt.Sprintf("%#x", announce.Hash)). + log.WithField("blockRoot", fmt.Sprintf("%#x", bytesutil.Trunc(announce.Hash))). Debug("Announcing canonical block") rs.p2p.Broadcast(ctx, announce) sentBlockAnnounce.Inc() diff --git a/shared/bytesutil/bytes.go b/shared/bytesutil/bytes.go index acb5486a9eb4..ec71ae595b32 100644 --- a/shared/bytesutil/bytes.go +++ b/shared/bytesutil/bytes.go @@ -97,3 +97,11 @@ func Xor(x []byte, y []byte) []byte { } return result } + +// Trunc truncates the byte slices to 12 bytes. +func Trunc(x []byte) []byte { + if len(x) > 12 { + return x[:12] + } + return x +} diff --git a/shared/bytesutil/bytes_test.go b/shared/bytesutil/bytes_test.go index 984bd8909a1c..1bf3d87b5d24 100644 --- a/shared/bytesutil/bytes_test.go +++ b/shared/bytesutil/bytes_test.go @@ -122,3 +122,22 @@ func TestFromBytes8(t *testing.T) { } } } + +func TestTruncate(t *testing.T) { + tests := []struct { + a []byte + b []byte + }{ + {[]byte{'A', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O'}, + []byte{'A', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M'}}, + {[]byte{'A', 'C', 'D', 'E', 'F'}, + []byte{'A', 'C', 'D', 'E', 'F'}}, + {[]byte{}, []byte{}}, + } + for _, tt := range tests { + b := Trunc(tt.a) + if !bytes.Equal(b, tt.b) { + t.Errorf("Trunc(%d) = %v, want = %d", tt.a, b, tt.b) + } + } +} From aa80c308cef298e5d885eddeda237b22abf72b00 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Mon, 29 Apr 2019 14:50:18 -0500 Subject: [PATCH 50/54] Save Deposit to DB Even If Fails to Deserialize (#2440) * include bad deposits in the db * fmt * Update beacon-chain/powchain/log_processing.go Co-Authored-By: rauljordan * lint * comment --- beacon-chain/powchain/log_processing.go | 22 +++++++++++--------- beacon-chain/powchain/log_processing_test.go | 15 +++++++++++-- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/beacon-chain/powchain/log_processing.go b/beacon-chain/powchain/log_processing.go index c0e8a11f7551..a413a9c7098a 100644 --- a/beacon-chain/powchain/log_processing.go +++ b/beacon-chain/powchain/log_processing.go @@ -76,10 +76,11 @@ func (w *Web3Service) ProcessDepositLog(depositLog gethTypes.Log) { // We then decode the deposit input in order to create a deposit object // we can store in our persistent DB. + validData := true depositInput, err := helpers.DecodeDepositInput(depositData) if err != nil { log.Errorf("Could not decode deposit input %v", err) - return + validData = false } deposit := &pb.Deposit{ @@ -87,14 +88,11 @@ func (w *Web3Service) ProcessDepositLog(depositLog gethTypes.Log) { MerkleTreeIndex: index, } - validData := true - // Make sure duplicates are rejected pre-chainstart. - if !w.chainStarted { + if !w.chainStarted && validData { var pubkey = fmt.Sprintf("#%x", depositInput.Pubkey) if w.beaconDB.PubkeyInChainstart(w.ctx, pubkey) { log.Warnf("Pubkey %#x has already been submitted for chainstart", pubkey) - validData = false } else { w.beaconDB.MarkPubkeyForChainstart(w.ctx, pubkey) } @@ -103,17 +101,21 @@ func (w *Web3Service) ProcessDepositLog(depositLog gethTypes.Log) { // We always store all historical deposits in the DB. w.beaconDB.InsertDeposit(w.ctx, deposit, big.NewInt(int64(depositLog.BlockNumber))) + if !w.chainStarted { + w.chainStartDeposits = append(w.chainStartDeposits, depositData) + } else { + w.beaconDB.InsertPendingDeposit(w.ctx, deposit, big.NewInt(int64(depositLog.BlockNumber))) + } if validData { - if !w.chainStarted { - w.chainStartDeposits = append(w.chainStartDeposits, depositData) - } else { - w.beaconDB.InsertPendingDeposit(w.ctx, deposit, big.NewInt(int64(depositLog.BlockNumber))) - } log.WithFields(logrus.Fields{ "publicKey": fmt.Sprintf("%#x", depositInput.Pubkey), "merkleTreeIndex": index, }).Info("Deposit registered from deposit contract") validDepositsCount.Inc() + } else { + log.WithFields(logrus.Fields{ + "merkleTreeIndex": index, + }).Info("Invalid deposit registered in deposit contract") } } diff --git a/beacon-chain/powchain/log_processing_test.go b/beacon-chain/powchain/log_processing_test.go index 4ef751136ee6..fde3852f8851 100644 --- a/beacon-chain/powchain/log_processing_test.go +++ b/beacon-chain/powchain/log_processing_test.go @@ -90,6 +90,7 @@ func TestProcessDepositLog_OK(t *testing.T) { } func TestProcessDepositLog_InsertsPendingDeposit(t *testing.T) { + hook := logTest.NewGlobal() testAcc, err := setup() if err != nil { t.Fatalf("Unable to set up simulated backend %v", err) @@ -124,10 +125,17 @@ func TestProcessDepositLog_InsertsPendingDeposit(t *testing.T) { } testAcc.txOpts.Value = amount32Eth + badData := []byte("bad data") if _, err := testAcc.contract.Deposit(testAcc.txOpts, serializedData.Bytes()); err != nil { t.Fatalf("Could not deposit to deposit contract %v", err) } + // A deposit with bad data should also be correctly processed and added to the + // db in the pending deposits bucket. + if _, err := testAcc.contract.Deposit(testAcc.txOpts, badData); err != nil { + t.Fatalf("Could not deposit to deposit contract %v", err) + } + testAcc.backend.Commit() query := ethereum.FilterQuery{ @@ -144,10 +152,13 @@ func TestProcessDepositLog_InsertsPendingDeposit(t *testing.T) { web3Service.chainStarted = true web3Service.ProcessDepositLog(logs[0]) + web3Service.ProcessDepositLog(logs[1]) pendingDeposits := web3Service.beaconDB.PendingDeposits(context.Background(), nil /*blockNum*/) - if len(pendingDeposits) != 1 { + if len(pendingDeposits) != 2 { t.Errorf("Unexpected number of deposits. Wanted 1 deposit, got %+v", pendingDeposits) } + testutil.AssertLogsContain(t, hook, "Invalid deposit registered in deposit contract") + hook.Reset() } func TestUnpackDepositLogData_OK(t *testing.T) { @@ -300,7 +311,7 @@ func TestProcessChainStartLog_8DuplicatePubkeys(t *testing.T) { } cachedDeposits := web3Service.ChainStartDeposits() - if len(cachedDeposits) != 1 { + if len(cachedDeposits) != depositsReqForChainStart { t.Errorf( "Did not cache the chain start deposits correctly, received %d, wanted %d", len(cachedDeposits), From bff774de32be8d4989555f09d9cf62d00f090a04 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 29 Apr 2019 13:03:28 -0700 Subject: [PATCH 51/54] Operation service prunes invalid attestation (#2439) * prune atts > one epoch old * use VerifyAttestation instead * added test to prune invalid att * space * added ctx * fixed existing tests * gaz * gazelle --- beacon-chain/core/blocks/block_operations.go | 1 + beacon-chain/operations/BUILD.bazel | 2 + beacon-chain/operations/service.go | 24 ++++- beacon-chain/operations/service_test.go | 103 +++++++++++++++---- beacon-chain/rpc/proposer_server.go | 2 +- beacon-chain/rpc/service.go | 2 +- beacon-chain/rpc/service_test.go | 2 +- 7 files changed, 111 insertions(+), 25 deletions(-) diff --git a/beacon-chain/core/blocks/block_operations.go b/beacon-chain/core/blocks/block_operations.go index 1d522a66608e..652f7fe502e8 100644 --- a/beacon-chain/core/blocks/block_operations.go +++ b/beacon-chain/core/blocks/block_operations.go @@ -499,6 +499,7 @@ func VerifyAttestation(beaconState *pb.BeaconState, att *pb.Attestation, verifyS } crosslinkFromAttestation := att.Data.LatestCrosslink crosslinkFromState := beaconState.LatestCrosslinks[shard] + if !(reflect.DeepEqual(crosslinkFromState, crosslink) || reflect.DeepEqual(crosslinkFromState, crosslinkFromAttestation)) { return fmt.Errorf( diff --git a/beacon-chain/operations/BUILD.bazel b/beacon-chain/operations/BUILD.bazel index ee9e4c2d41c2..eb6ad12c6236 100644 --- a/beacon-chain/operations/BUILD.bazel +++ b/beacon-chain/operations/BUILD.bazel @@ -6,6 +6,7 @@ go_library( importpath = "github.com/prysmaticlabs/prysm/beacon-chain/operations", visibility = ["//beacon-chain:__subpackages__"], deps = [ + "//beacon-chain/core/blocks:go_default_library", "//beacon-chain/db:go_default_library", "//proto/beacon/p2p/v1:go_default_library", "//shared/event:go_default_library", @@ -26,6 +27,7 @@ go_test( deps = [ "//beacon-chain/internal:go_default_library", "//proto/beacon/p2p/v1:go_default_library", + "//shared/hashutil:go_default_library", "//shared/params:go_default_library", "//shared/testutil:go_default_library", "@com_github_gogo_protobuf//proto:go_default_library", diff --git a/beacon-chain/operations/service.go b/beacon-chain/operations/service.go index a7aeced6adea..fa7cd8889fd1 100644 --- a/beacon-chain/operations/service.go +++ b/beacon-chain/operations/service.go @@ -7,6 +7,7 @@ import ( "sort" "github.com/gogo/protobuf/proto" + "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks" "github.com/prysmaticlabs/prysm/beacon-chain/db" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/event" @@ -112,21 +113,36 @@ func (s *Service) IncomingProcessedBlockFeed() *event.Feed { // PendingAttestations returns the attestations that have not seen on the beacon chain, the attestations are // returns in slot ascending order and up to MaxAttestations capacity. The attestations get // deleted in DB after they have been retrieved. -func (s *Service) PendingAttestations() ([]*pb.Attestation, error) { +func (s *Service) PendingAttestations(ctx context.Context) ([]*pb.Attestation, error) { var attestations []*pb.Attestation attestationsFromDB, err := s.beaconDB.Attestations() if err != nil { return nil, fmt.Errorf("could not retrieve attestations from DB") } + state, err := s.beaconDB.HeadState(ctx) + if err != nil { + return nil, fmt.Errorf("could not retrieve attestations from DB") + } sort.Slice(attestationsFromDB, func(i, j int) bool { return attestationsFromDB[i].Data.Slot < attestationsFromDB[j].Data.Slot }) - for i := range attestationsFromDB { + var validAttsCount uint64 + for _, att := range attestationsFromDB { + // Delete the attestation if it fails to verify using head state, + // we don't want to pass the attestation to the proposer. + if err := blocks.VerifyAttestation(state, att, false /* verify signature */); err != nil { + if err := s.beaconDB.DeleteAttestation(att); err != nil { + return nil, err + } + continue + } + + validAttsCount++ // Stop the max attestation number per beacon block is reached. - if uint64(i) == params.BeaconConfig().MaxAttestations { + if validAttsCount == params.BeaconConfig().MaxAttestations { break } - attestations = append(attestations, attestationsFromDB[i]) + attestations = append(attestations, att) } return attestations, nil } diff --git a/beacon-chain/operations/service_test.go b/beacon-chain/operations/service_test.go index 13ecb6a3e74f..10e04d8f025e 100644 --- a/beacon-chain/operations/service_test.go +++ b/beacon-chain/operations/service_test.go @@ -10,6 +10,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/prysmaticlabs/prysm/beacon-chain/internal" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/hashutil" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/testutil" "github.com/sirupsen/logrus" @@ -126,29 +127,80 @@ func TestRetrieveAttestations_OK(t *testing.T) { defer internal.TeardownDB(t, beaconDB) service := NewOpsPoolService(context.Background(), &Config{BeaconDB: beaconDB}) - // Save 140 attestations for test. During 1st retrieval we should get slot:0 - slot:128 attestations, - // 2nd retrieval we should get slot:128 - slot:140 attestations. - // Max attestation config value is set to 128. + // Save 140 attestations for test. During 1st retrieval we should get slot:1 - slot:61 attestations. + // The 1st retrieval is set at slot 64. origAttestations := make([]*pb.Attestation, 140) for i := 0; i < len(origAttestations); i++ { origAttestations[i] = &pb.Attestation{ Data: &pb.AttestationData{ - Slot: uint64(i), - Shard: uint64(i), + Slot: params.BeaconConfig().GenesisSlot + uint64(i), + CrosslinkDataRootHash32: params.BeaconConfig().ZeroHash[:], + }, + } + if err := service.beaconDB.SaveAttestation(context.Background(), origAttestations[i]); err != nil { + t.Fatalf("Failed to save attestation: %v", err) + } + } + if err := beaconDB.SaveState(context.Background(), &pb.BeaconState{ + Slot: params.BeaconConfig().GenesisSlot + 64, + LatestCrosslinks: []*pb.Crosslink{{ + Epoch: params.BeaconConfig().GenesisEpoch, + CrosslinkDataRootHash32: params.BeaconConfig().ZeroHash[:]}}}); err != nil { + t.Fatal(err) + } + // Test we can retrieve attestations from slot1 - slot61. + attestations, err := service.PendingAttestations(context.Background()) + if err != nil { + t.Fatalf("Could not retrieve attestations: %v", err) + } + + if !reflect.DeepEqual(attestations, origAttestations[1:61]) { + t.Error("Retrieved attestations did not match") + } +} + +func TestRetrieveAttestations_PruneInvalidAtts(t *testing.T) { + beaconDB := internal.SetupDB(t) + defer internal.TeardownDB(t, beaconDB) + service := NewOpsPoolService(context.Background(), &Config{BeaconDB: beaconDB}) + + // Save 140 attestations for slots 0 to 139. + origAttestations := make([]*pb.Attestation, 140) + for i := 0; i < len(origAttestations); i++ { + origAttestations[i] = &pb.Attestation{ + Data: &pb.AttestationData{ + Slot: params.BeaconConfig().GenesisSlot + uint64(i), + CrosslinkDataRootHash32: params.BeaconConfig().ZeroHash[:], }, } if err := service.beaconDB.SaveAttestation(context.Background(), origAttestations[i]); err != nil { t.Fatalf("Failed to save attestation: %v", err) } } - // Test we can retrieve attestations from slot0 - slot127 (Max attestation amount). - attestations, err := service.PendingAttestations() + + // At slot 200 only attestations up to from slot 137 to 139 are valid attestations. + if err := beaconDB.SaveState(context.Background(), &pb.BeaconState{ + Slot: params.BeaconConfig().GenesisSlot + 200, + LatestCrosslinks: []*pb.Crosslink{{ + Epoch: params.BeaconConfig().GenesisEpoch + 2, + CrosslinkDataRootHash32: params.BeaconConfig().ZeroHash[:]}}}); err != nil { + t.Fatal(err) + } + attestations, err := service.PendingAttestations(context.Background()) if err != nil { t.Fatalf("Could not retrieve attestations: %v", err) } - if !reflect.DeepEqual(attestations, origAttestations[0:params.BeaconConfig().MaxAttestations]) { - t.Errorf("Retrieved attestations did not match prev generated attestations for the first %d", - params.BeaconConfig().MaxAttestations) + if !reflect.DeepEqual(attestations, origAttestations[137:]) { + t.Error("Incorrect pruned attestations") + } + + // Verify the invalid attestations are deleted. + hash, err := hashutil.HashProto(origAttestations[136]) + if err != nil { + t.Fatal(err) + } + if service.beaconDB.HasAttestation(hash) { + t.Error("Invalid attestation is not deleted") } } @@ -161,16 +213,23 @@ func TestRemoveProcessedAttestations_Ok(t *testing.T) { for i := 0; i < len(attestations); i++ { attestations[i] = &pb.Attestation{ Data: &pb.AttestationData{ - Slot: uint64(i), - Shard: uint64(i), + Slot: params.BeaconConfig().GenesisSlot + uint64(i), + CrosslinkDataRootHash32: params.BeaconConfig().ZeroHash[:], }, } if err := s.beaconDB.SaveAttestation(context.Background(), attestations[i]); err != nil { t.Fatalf("Failed to save attestation: %v", err) } } + if err := db.SaveState(context.Background(), &pb.BeaconState{ + Slot: params.BeaconConfig().GenesisSlot + 15, + LatestCrosslinks: []*pb.Crosslink{{ + Epoch: params.BeaconConfig().GenesisEpoch, + CrosslinkDataRootHash32: params.BeaconConfig().ZeroHash[:]}}}); err != nil { + t.Fatal(err) + } - retrievedAtts, err := s.PendingAttestations() + retrievedAtts, err := s.PendingAttestations(context.Background()) if err != nil { t.Fatalf("Could not retrieve attestations: %v", err) } @@ -182,7 +241,7 @@ func TestRemoveProcessedAttestations_Ok(t *testing.T) { t.Fatalf("Could not remove pending attestations: %v", err) } - retrievedAtts, _ = s.PendingAttestations() + retrievedAtts, _ = s.PendingAttestations(context.Background()) if len(retrievedAtts) != 0 { t.Errorf("Attestation pool should be empty but got a length of %d", len(retrievedAtts)) } @@ -233,8 +292,8 @@ func TestReceiveBlkRemoveOps_Ok(t *testing.T) { for i := 0; i < len(attestations); i++ { attestations[i] = &pb.Attestation{ Data: &pb.AttestationData{ - Slot: uint64(i), - Shard: uint64(i), + Slot: params.BeaconConfig().GenesisSlot + uint64(i), + CrosslinkDataRootHash32: params.BeaconConfig().ZeroHash[:], }, } if err := s.beaconDB.SaveAttestation(context.Background(), attestations[i]); err != nil { @@ -242,7 +301,15 @@ func TestReceiveBlkRemoveOps_Ok(t *testing.T) { } } - atts, _ := s.PendingAttestations() + if err := db.SaveState(context.Background(), &pb.BeaconState{ + Slot: params.BeaconConfig().GenesisSlot + 15, + LatestCrosslinks: []*pb.Crosslink{{ + Epoch: params.BeaconConfig().GenesisEpoch, + CrosslinkDataRootHash32: params.BeaconConfig().ZeroHash[:]}}}); err != nil { + t.Fatal(err) + } + + atts, _ := s.PendingAttestations(context.Background()) if len(atts) != len(attestations) { t.Errorf("Attestation pool should be %d but got a length of %d", len(attestations), len(atts)) @@ -259,7 +326,7 @@ func TestReceiveBlkRemoveOps_Ok(t *testing.T) { t.Error(err) } - atts, _ = s.PendingAttestations() + atts, _ = s.PendingAttestations(context.Background()) if len(atts) != 0 { t.Errorf("Attestation pool should be empty but got a length of %d", len(atts)) } diff --git a/beacon-chain/rpc/proposer_server.go b/beacon-chain/rpc/proposer_server.go index f6d2d3b6f8a4..efffd63f634c 100644 --- a/beacon-chain/rpc/proposer_server.go +++ b/beacon-chain/rpc/proposer_server.go @@ -101,7 +101,7 @@ func (ps *ProposerServer) PendingAttestations(ctx context.Context, req *pb.Pendi if err != nil { return nil, fmt.Errorf("could not retrieve beacon state: %v", err) } - atts, err := ps.operationService.PendingAttestations() + atts, err := ps.operationService.PendingAttestations(ctx) if err != nil { return nil, fmt.Errorf("could not retrieve pending attestations from operations service: %v", err) } diff --git a/beacon-chain/rpc/service.go b/beacon-chain/rpc/service.go index a67176a0336d..cc9a2d2afc8c 100644 --- a/beacon-chain/rpc/service.go +++ b/beacon-chain/rpc/service.go @@ -39,7 +39,7 @@ type chainService interface { } type operationService interface { - PendingAttestations() ([]*pbp2p.Attestation, error) + PendingAttestations(ctx context.Context) ([]*pbp2p.Attestation, error) HandleAttestations(context.Context, proto.Message) error IncomingAttFeed() *event.Feed } diff --git a/beacon-chain/rpc/service_test.go b/beacon-chain/rpc/service_test.go index 7cd09f2ff0d1..94720e48286e 100644 --- a/beacon-chain/rpc/service_test.go +++ b/beacon-chain/rpc/service_test.go @@ -47,7 +47,7 @@ func (ms *mockOperationService) HandleAttestations(_ context.Context, _ proto.Me return nil } -func (ms *mockOperationService) PendingAttestations() ([]*pb.Attestation, error) { +func (ms *mockOperationService) PendingAttestations(_ context.Context) ([]*pb.Attestation, error) { if ms.pendingAttestations != nil { return ms.pendingAttestations, nil } From 22fa59566551885b6c39a0fb207e250697195064 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 29 Apr 2019 13:45:04 -0700 Subject: [PATCH 52/54] Prune Atts One Epoch Older (#2442) * prune atts > one epoch old * use VerifyAttestation instead * added test to prune invalid att * space * added ctx * fixed existing tests * gaz * gazelle * update filter for pruning atts * gaz --- beacon-chain/operations/BUILD.bazel | 1 - beacon-chain/operations/service.go | 7 +++---- beacon-chain/operations/service_test.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/beacon-chain/operations/BUILD.bazel b/beacon-chain/operations/BUILD.bazel index eb6ad12c6236..9226c5fdc93b 100644 --- a/beacon-chain/operations/BUILD.bazel +++ b/beacon-chain/operations/BUILD.bazel @@ -6,7 +6,6 @@ go_library( importpath = "github.com/prysmaticlabs/prysm/beacon-chain/operations", visibility = ["//beacon-chain:__subpackages__"], deps = [ - "//beacon-chain/core/blocks:go_default_library", "//beacon-chain/db:go_default_library", "//proto/beacon/p2p/v1:go_default_library", "//shared/event:go_default_library", diff --git a/beacon-chain/operations/service.go b/beacon-chain/operations/service.go index fa7cd8889fd1..649411f4d0db 100644 --- a/beacon-chain/operations/service.go +++ b/beacon-chain/operations/service.go @@ -7,7 +7,6 @@ import ( "sort" "github.com/gogo/protobuf/proto" - "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks" "github.com/prysmaticlabs/prysm/beacon-chain/db" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/event" @@ -128,9 +127,9 @@ func (s *Service) PendingAttestations(ctx context.Context) ([]*pb.Attestation, e }) var validAttsCount uint64 for _, att := range attestationsFromDB { - // Delete the attestation if it fails to verify using head state, - // we don't want to pass the attestation to the proposer. - if err := blocks.VerifyAttestation(state, att, false /* verify signature */); err != nil { + // Delete the attestation if the attestation is one epoch older than head state, + // we don't want to pass these attestations to RPC for proposer to include. + if att.Data.Slot+params.BeaconConfig().SlotsPerEpoch <= state.Slot { if err := s.beaconDB.DeleteAttestation(att); err != nil { return nil, err } diff --git a/beacon-chain/operations/service_test.go b/beacon-chain/operations/service_test.go index 10e04d8f025e..ce00d5a14230 100644 --- a/beacon-chain/operations/service_test.go +++ b/beacon-chain/operations/service_test.go @@ -154,7 +154,7 @@ func TestRetrieveAttestations_OK(t *testing.T) { t.Fatalf("Could not retrieve attestations: %v", err) } - if !reflect.DeepEqual(attestations, origAttestations[1:61]) { + if !reflect.DeepEqual(attestations, origAttestations[1:128]) { t.Error("Retrieved attestations did not match") } } From 3f742d79c0eb6737b0f264f86cd38516c13266f1 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Mon, 29 Apr 2019 18:21:54 -0400 Subject: [PATCH 53/54] Use new port for testnet site, redirect prylabs.net to alpha.prylabs.net for now (#2441) --- k8s/beacon-chain/testnet-site.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/k8s/beacon-chain/testnet-site.yaml b/k8s/beacon-chain/testnet-site.yaml index ae0cc7ec8d1b..10eb02dd11da 100644 --- a/k8s/beacon-chain/testnet-site.yaml +++ b/k8s/beacon-chain/testnet-site.yaml @@ -26,14 +26,14 @@ spec: - name: site image: gcr.io/prysmaticlabs/prysm-testnet-site:latest ports: - - containerPort: 80 + - containerPort: 4000 name: http livenessProbe: httpGet: - path: /healthz - port: 80 + path: / + port: 4000 initialDelaySeconds: 3 - periodSeconds: 3 + periodSeconds: 15 resources: requests: cpu: "100m" @@ -68,7 +68,7 @@ spec: version: alpha ports: - port: 80 - targetPort: 80 + targetPort: 4000 name: http-nginx type: ClusterIP --- @@ -111,4 +111,8 @@ spec: port: number: 80 host: testnet-site-alpha.beacon-chain.svc.cluster.local - + - match: + - uri: + prefix: / + redirect: + authority: alpha.prylabs.net From 7a7b4e55c9d783e145130410f53ef3b306c465b5 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 29 Apr 2019 16:25:31 -0700 Subject: [PATCH 54/54] prevent validator exit reentrance (#2443) --- beacon-chain/core/validators/validator.go | 5 ++--- beacon-chain/core/validators/validator_test.go | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/beacon-chain/core/validators/validator.go b/beacon-chain/core/validators/validator.go index 3c2dfd9c7769..969df113b495 100644 --- a/beacon-chain/core/validators/validator.go +++ b/beacon-chain/core/validators/validator.go @@ -206,11 +206,10 @@ func InitiateValidatorExit(state *pb.BeaconState, idx uint64) *pb.BeaconState { func ExitValidator(state *pb.BeaconState, idx uint64) *pb.BeaconState { validator := state.ValidatorRegistry[idx] - exitEpoch := entryExitEffectEpoch(helpers.CurrentEpoch(state)) - if validator.ExitEpoch <= exitEpoch { + if validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch { return state } - validator.ExitEpoch = exitEpoch + validator.ExitEpoch = entryExitEffectEpoch(helpers.CurrentEpoch(state)) return state } diff --git a/beacon-chain/core/validators/validator_test.go b/beacon-chain/core/validators/validator_test.go index 8961adfe2d60..e0ab5fe3f41d 100644 --- a/beacon-chain/core/validators/validator_test.go +++ b/beacon-chain/core/validators/validator_test.go @@ -430,14 +430,14 @@ func TestExitValidator_OK(t *testing.T) { func TestExitValidator_AlreadyExited(t *testing.T) { state := &pb.BeaconState{ - Slot: 1, + Slot: params.BeaconConfig().GenesisEpoch + 1000, ValidatorRegistry: []*pb.Validator{ {ExitEpoch: params.BeaconConfig().ActivationExitDelay}, }, } state = ExitValidator(state, 0) - if len(state.ValidatorRegistry) != 1 { - t.Error("Expected validator to have failed exiting") + if state.ValidatorRegistry[0].ExitEpoch != params.BeaconConfig().ActivationExitDelay { + t.Error("Expected exited validator to stay exited") } }