From 17a8ae817b90d4cc4b4b1cfe999ef8efb2a27c57 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Thu, 18 Apr 2024 11:53:23 -0400 Subject: [PATCH 01/62] Add validator manager callbacks --- chains/manager.go | 8 ++-- network/network.go | 2 +- node/overridden_manager.go | 8 +++- .../avalanche/bootstrap/bootstrapper_test.go | 2 +- .../snowman/bootstrap/bootstrapper_test.go | 6 +-- .../snowman/syncer/state_syncer_test.go | 30 ++++++------- snow/engine/snowman/transitive.go | 2 +- snow/engine/snowman/transitive_test.go | 2 +- snow/networking/handler/health_test.go | 2 +- snow/validators/manager.go | 34 +++++++++++--- snow/validators/manager_test.go | 8 ++-- snow/validators/set.go | 45 ++++++++++++++----- snow/validators/set_test.go | 34 +++++++------- vms/platformvm/vm.go | 4 +- vms/platformvm/vm_test.go | 2 +- 15 files changed, 119 insertions(+), 70 deletions(-) diff --git a/chains/manager.go b/chains/manager.go index f2c79547c205..b5397c7c692e 100644 --- a/chains/manager.go +++ b/chains/manager.go @@ -761,7 +761,7 @@ func (m *manager) createAvalancheChain( if err != nil { return nil, fmt.Errorf("error creating peer tracker: %w", err) } - vdrs.RegisterCallbackListener(ctx.SubnetID, connectedValidators) + vdrs.RegisterSetCallbackListener(ctx.SubnetID, connectedValidators) // Asynchronously passes messages from the network to the consensus engine h, err := handler.New( @@ -781,7 +781,7 @@ func (m *manager) createAvalancheChain( connectedBeacons := tracker.NewPeers() startupTracker := tracker.NewStartup(connectedBeacons, (3*bootstrapWeight+3)/4) - vdrs.RegisterCallbackListener(ctx.SubnetID, startupTracker) + vdrs.RegisterSetCallbackListener(ctx.SubnetID, startupTracker) snowGetHandler, err := snowgetter.New( vmWrappingProposerVM, @@ -1093,7 +1093,7 @@ func (m *manager) createSnowmanChain( if err != nil { return nil, fmt.Errorf("error creating peer tracker: %w", err) } - vdrs.RegisterCallbackListener(ctx.SubnetID, connectedValidators) + vdrs.RegisterSetCallbackListener(ctx.SubnetID, connectedValidators) // Asynchronously passes messages from the network to the consensus engine h, err := handler.New( @@ -1113,7 +1113,7 @@ func (m *manager) createSnowmanChain( connectedBeacons := tracker.NewPeers() startupTracker := tracker.NewStartup(connectedBeacons, (3*bootstrapWeight+3)/4) - beacons.RegisterCallbackListener(ctx.SubnetID, startupTracker) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startupTracker) snowGetHandler, err := snowgetter.New( vm, diff --git a/network/network.go b/network/network.go index 4e462e933a42..506e51908586 100644 --- a/network/network.go +++ b/network/network.go @@ -239,7 +239,7 @@ func NewNetwork( if err != nil { return nil, fmt.Errorf("initializing ip tracker failed with: %w", err) } - config.Validators.RegisterCallbackListener(constants.PrimaryNetworkID, ipTracker) + config.Validators.RegisterSetCallbackListener(constants.PrimaryNetworkID, ipTracker) // Track all default bootstrappers to ensure their current IPs are gossiped // like validator IPs. diff --git a/node/overridden_manager.go b/node/overridden_manager.go index 4dd49b65eab6..484fe05da758 100644 --- a/node/overridden_manager.go +++ b/node/overridden_manager.go @@ -72,8 +72,12 @@ func (o *overriddenManager) GetMap(ids.ID) map[ids.NodeID]*validators.GetValidat return o.manager.GetMap(o.subnetID) } -func (o *overriddenManager) RegisterCallbackListener(_ ids.ID, listener validators.SetCallbackListener) { - o.manager.RegisterCallbackListener(o.subnetID, listener) +func (o *overriddenManager) RegisterCallbackListener(listener validators.ManagerCallbackListener) { + o.manager.RegisterCallbackListener(listener) +} + +func (o *overriddenManager) RegisterSetCallbackListener(_ ids.ID, listener validators.SetCallbackListener) { + o.manager.RegisterSetCallbackListener(o.subnetID, listener) } func (o *overriddenManager) String() string { diff --git a/snow/engine/avalanche/bootstrap/bootstrapper_test.go b/snow/engine/avalanche/bootstrap/bootstrapper_test.go index 8d8c107383e9..812a343dd171 100644 --- a/snow/engine/avalanche/bootstrap/bootstrapper_test.go +++ b/snow/engine/avalanche/bootstrap/bootstrapper_test.go @@ -81,7 +81,7 @@ func newConfig(t *testing.T) (Config, ids.NodeID, *common.SenderTest, *vertex.Te totalWeight, err := vdrs.TotalWeight(constants.PrimaryNetworkID) require.NoError(err) startupTracker := tracker.NewStartup(peerTracker, totalWeight/2+1) - vdrs.RegisterCallbackListener(constants.PrimaryNetworkID, startupTracker) + vdrs.RegisterSetCallbackListener(constants.PrimaryNetworkID, startupTracker) avaGetHandler, err := getter.New(manager, sender, ctx.Log, time.Second, 2000, ctx.AvalancheRegisterer) require.NoError(err) diff --git a/snow/engine/snowman/bootstrap/bootstrapper_test.go b/snow/engine/snowman/bootstrap/bootstrapper_test.go index d5db4be5ae93..4407ddf432d6 100644 --- a/snow/engine/snowman/bootstrap/bootstrapper_test.go +++ b/snow/engine/snowman/bootstrap/bootstrapper_test.go @@ -71,7 +71,7 @@ func newConfig(t *testing.T) (Config, ids.NodeID, *common.SenderTest, *block.Tes totalWeight, err := vdrs.TotalWeight(ctx.SubnetID) require.NoError(err) startupTracker := tracker.NewStartup(peerTracker, totalWeight/2+1) - vdrs.RegisterCallbackListener(ctx.SubnetID, startupTracker) + vdrs.RegisterSetCallbackListener(ctx.SubnetID, startupTracker) require.NoError(startupTracker.Connected(context.Background(), peer, version.CurrentApp)) @@ -113,7 +113,7 @@ func TestBootstrapperStartsOnlyIfEnoughStakeIsConnected(t *testing.T) { peerTracker := tracker.NewPeers() startupTracker := tracker.NewStartup(peerTracker, startupAlpha) - peers.RegisterCallbackListener(ctx.SubnetID, startupTracker) + peers.RegisterSetCallbackListener(ctx.SubnetID, startupTracker) snowGetHandler, err := getter.New(vm, sender, ctx.Log, time.Second, 2000, ctx.Registerer) require.NoError(err) @@ -626,7 +626,7 @@ func TestBootstrapNoParseOnNew(t *testing.T) { totalWeight, err := peers.TotalWeight(ctx.SubnetID) require.NoError(err) startupTracker := tracker.NewStartup(peerTracker, totalWeight/2+1) - peers.RegisterCallbackListener(ctx.SubnetID, startupTracker) + peers.RegisterSetCallbackListener(ctx.SubnetID, startupTracker) require.NoError(startupTracker.Connected(context.Background(), peer, version.CurrentApp)) snowGetHandler, err := getter.New(vm, sender, ctx.Log, time.Second, 2000, ctx.Registerer) diff --git a/snow/engine/snowman/syncer/state_syncer_test.go b/snow/engine/snowman/syncer/state_syncer_test.go index 1ec1e67021bc..2ee745bb5c64 100644 --- a/snow/engine/snowman/syncer/state_syncer_test.go +++ b/snow/engine/snowman/syncer/state_syncer_test.go @@ -116,7 +116,7 @@ func TestStateSyncingStartsOnlyIfEnoughStakeIsConnected(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, _, sender := buildTestsObjects(t, ctx, startup, beacons, alpha) @@ -159,7 +159,7 @@ func TestStateSyncLocalSummaryIsIncludedAmongFrontiersIfAvailable(t *testing.T) peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, _ := buildTestsObjects(t, ctx, startup, beacons, (totalWeight+1)/2) @@ -197,7 +197,7 @@ func TestStateSyncNotFoundOngoingSummaryIsNotIncludedAmongFrontiers(t *testing.T peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, _ := buildTestsObjects(t, ctx, startup, beacons, (totalWeight+1)/2) @@ -228,7 +228,7 @@ func TestBeaconsAreReachedForFrontiersUponStartup(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, _, sender := buildTestsObjects(t, ctx, startup, beacons, (totalWeight+1)/2) @@ -267,7 +267,7 @@ func TestUnRequestedStateSummaryFrontiersAreDropped(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, sender := buildTestsObjects(t, ctx, startup, beacons, (totalWeight+1)/2) @@ -357,7 +357,7 @@ func TestMalformedStateSummaryFrontiersAreDropped(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, sender := buildTestsObjects(t, ctx, startup, beacons, (totalWeight+1)/2) @@ -426,7 +426,7 @@ func TestLateResponsesFromUnresponsiveFrontiersAreNotRecorded(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, sender := buildTestsObjects(t, ctx, startup, beacons, (totalWeight+1)/2) @@ -509,7 +509,7 @@ func TestStateSyncIsRestartedIfTooManyFrontierSeedersTimeout(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, sender := buildTestsObjects(t, ctx, startup, beacons, (totalWeight+1)/2) @@ -598,7 +598,7 @@ func TestVoteRequestsAreSentAsAllFrontierBeaconsResponded(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, sender := buildTestsObjects(t, ctx, startup, beacons, (totalWeight+1)/2) @@ -669,7 +669,7 @@ func TestUnRequestedVotesAreDropped(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, sender := buildTestsObjects(t, ctx, startup, beacons, (totalWeight+1)/2) @@ -786,7 +786,7 @@ func TestVotesForUnknownSummariesAreDropped(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, sender := buildTestsObjects(t, ctx, startup, beacons, (totalWeight+1)/2) @@ -890,7 +890,7 @@ func TestStateSummaryIsPassedToVMAsMajorityOfVotesIsCastedForIt(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, sender := buildTestsObjects(t, ctx, startup, beacons, alpha) @@ -1035,7 +1035,7 @@ func TestVotingIsRestartedIfMajorityIsNotReachedDueToTimeouts(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, sender := buildTestsObjects(t, ctx, startup, beacons, alpha) @@ -1141,7 +1141,7 @@ func TestStateSyncIsStoppedIfEnoughVotesAreCastedWithNoClearMajority(t *testing. peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, fullVM, sender := buildTestsObjects(t, ctx, startup, beacons, alpha) @@ -1286,7 +1286,7 @@ func TestStateSyncIsDoneOnceVMNotifies(t *testing.T) { peers := tracker.NewPeers() startup := tracker.NewStartup(peers, startupAlpha) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) syncer, _, _ := buildTestsObjects(t, ctx, startup, beacons, (totalWeight+1)/2) diff --git a/snow/engine/snowman/transitive.go b/snow/engine/snowman/transitive.go index 43a00dbcb9c2..07cffdca565e 100644 --- a/snow/engine/snowman/transitive.go +++ b/snow/engine/snowman/transitive.go @@ -109,7 +109,7 @@ func New(config Config) (*Transitive, error) { } acceptedFrontiers := tracker.NewAccepted() - config.Validators.RegisterCallbackListener(config.Ctx.SubnetID, acceptedFrontiers) + config.Validators.RegisterSetCallbackListener(config.Ctx.SubnetID, acceptedFrontiers) factory := poll.NewEarlyTermNoTraversalFactory( config.Params.AlphaPreference, diff --git a/snow/engine/snowman/transitive_test.go b/snow/engine/snowman/transitive_test.go index bd5966f797b9..f7e74412075e 100644 --- a/snow/engine/snowman/transitive_test.go +++ b/snow/engine/snowman/transitive_test.go @@ -43,7 +43,7 @@ func setup(t *testing.T, engCfg Config) (ids.NodeID, validators.Manager, *common require.NoError(vals.AddStaker(engCfg.Ctx.SubnetID, vdr, nil, ids.Empty, 1)) require.NoError(engCfg.ConnectedValidators.Connected(context.Background(), vdr, version.CurrentApp)) - vals.RegisterCallbackListener(engCfg.Ctx.SubnetID, engCfg.ConnectedValidators) + vals.RegisterSetCallbackListener(engCfg.Ctx.SubnetID, engCfg.ConnectedValidators) sender := &common.SenderTest{T: t} engCfg.Sender = sender diff --git a/snow/networking/handler/health_test.go b/snow/networking/handler/health_test.go index adeb3430f277..a66d207b867c 100644 --- a/snow/networking/handler/health_test.go +++ b/snow/networking/handler/health_test.go @@ -61,7 +61,7 @@ func TestHealthCheckSubnet(t *testing.T) { require.NoError(err) peerTracker := commontracker.NewPeers() - vdrs.RegisterCallbackListener(ctx.SubnetID, peerTracker) + vdrs.RegisterSetCallbackListener(ctx.SubnetID, peerTracker) sb := subnets.New( ctx.NodeID, diff --git a/snow/validators/manager.go b/snow/validators/manager.go index 5844c1e7f185..1b5be8ca1874 100644 --- a/snow/validators/manager.go +++ b/snow/validators/manager.go @@ -24,6 +24,12 @@ var ( ErrMissingValidators = errors.New("missing validators") ) +type ManagerCallbackListener interface { + OnValidatorAdded(subnetID ids.ID, nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) + OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, weight uint64) + OnValidatorWeightChanged(subnetID ids.ID, nodeID ids.NodeID, oldWeight, newWeight uint64) +} + type SetCallbackListener interface { OnValidatorAdded(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) OnValidatorRemoved(nodeID ids.NodeID, weight uint64) @@ -88,9 +94,13 @@ type Manager interface { // Map of the validators in this subnet GetMap(subnetID ids.ID) map[ids.NodeID]*GetValidatorOutput - // When a validator's weight changes, or a validator is added/removed, - // this listener is called. - RegisterCallbackListener(subnetID ids.ID, listener SetCallbackListener) + // When a validator is added, removed, or its weight changes, the listener + // will be notified of the event. + RegisterCallbackListener(listener ManagerCallbackListener) + + // When a validator is added, removed, or its weight changes on [subnetID], + // the listener will be notified of the event. + RegisterSetCallbackListener(subnetID ids.ID, listener SetCallbackListener) } // NewManager returns a new, empty manager @@ -105,7 +115,8 @@ type manager struct { // Key: Subnet ID // Value: The validators that validate the subnet - subnetToVdrs map[ids.ID]*vdrSet + subnetToVdrs map[ids.ID]*vdrSet + callbackListeners []ManagerCallbackListener } func (m *manager) AddStaker(subnetID ids.ID, nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) error { @@ -118,7 +129,7 @@ func (m *manager) AddStaker(subnetID ids.ID, nodeID ids.NodeID, pk *bls.PublicKe set, exists := m.subnetToVdrs[subnetID] if !exists { - set = newSet() + set = newSet(subnetID, m.callbackListeners) m.subnetToVdrs[subnetID] = set } @@ -264,13 +275,22 @@ func (m *manager) GetMap(subnetID ids.ID) map[ids.NodeID]*GetValidatorOutput { return set.Map() } -func (m *manager) RegisterCallbackListener(subnetID ids.ID, listener SetCallbackListener) { +func (m *manager) RegisterCallbackListener(listener ManagerCallbackListener) { + m.lock.Lock() + defer m.lock.Unlock() + + for _, set := range m.subnetToVdrs { + set.RegisterManagerCallbackListener(listener) + } +} + +func (m *manager) RegisterSetCallbackListener(subnetID ids.ID, listener SetCallbackListener) { m.lock.Lock() defer m.lock.Unlock() set, exists := m.subnetToVdrs[subnetID] if !exists { - set = newSet() + set = newSet(subnetID, m.callbackListeners) m.subnetToVdrs[subnetID] = set } diff --git a/snow/validators/manager_test.go b/snow/validators/manager_test.go index 781d2e784e1d..8be3797994fb 100644 --- a/snow/validators/manager_test.go +++ b/snow/validators/manager_test.go @@ -421,7 +421,7 @@ func TestAddCallback(t *testing.T) { m := NewManager() subnetID := ids.GenerateTestID() callCount := 0 - m.RegisterCallbackListener(subnetID, &callbackListener{ + m.RegisterSetCallbackListener(subnetID, &callbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { require.Equal(nodeID0, nodeID) @@ -452,7 +452,7 @@ func TestAddWeightCallback(t *testing.T) { require.NoError(m.AddStaker(subnetID, nodeID0, nil, txID0, weight0)) callCount := 0 - m.RegisterCallbackListener(subnetID, &callbackListener{ + m.RegisterSetCallbackListener(subnetID, &callbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { require.Equal(nodeID0, nodeID) @@ -490,7 +490,7 @@ func TestRemoveWeightCallback(t *testing.T) { require.NoError(m.AddStaker(subnetID, nodeID0, nil, txID0, weight0)) callCount := 0 - m.RegisterCallbackListener(subnetID, &callbackListener{ + m.RegisterSetCallbackListener(subnetID, &callbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { require.Equal(nodeID0, nodeID) @@ -527,7 +527,7 @@ func TestValidatorRemovedCallback(t *testing.T) { require.NoError(m.AddStaker(subnetID, nodeID0, nil, txID0, weight0)) callCount := 0 - m.RegisterCallbackListener(subnetID, &callbackListener{ + m.RegisterSetCallbackListener(subnetID, &callbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { require.Equal(nodeID0, nodeID) diff --git a/snow/validators/set.go b/snow/validators/set.go index 5e7c81a2310e..e9bb235f995b 100644 --- a/snow/validators/set.go +++ b/snow/validators/set.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "math/big" + "slices" "strings" "sync" @@ -25,15 +26,19 @@ var ( ) // newSet returns a new, empty set of validators. -func newSet() *vdrSet { +func newSet(subnetID ids.ID, callbackListeners []ManagerCallbackListener) *vdrSet { return &vdrSet{ - vdrs: make(map[ids.NodeID]*Validator), - sampler: sampler.NewWeightedWithoutReplacement(), - totalWeight: new(big.Int), + subnetID: subnetID, + vdrs: make(map[ids.NodeID]*Validator), + totalWeight: new(big.Int), + sampler: sampler.NewWeightedWithoutReplacement(), + managerCallbackListeners: slices.Clone(callbackListeners), } } type vdrSet struct { + subnetID ids.ID + lock sync.RWMutex vdrs map[ids.NodeID]*Validator vdrSlice []*Validator @@ -43,7 +48,8 @@ type vdrSet struct { samplerInitialized bool sampler sampler.WeightedWithoutReplacement - callbackListeners []SetCallbackListener + managerCallbackListeners []ManagerCallbackListener + setCallbackListeners []SetCallbackListener } func (s *vdrSet) Add(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) error { @@ -218,7 +224,7 @@ func (s *vdrSet) HasCallbackRegistered() bool { s.lock.RLock() defer s.lock.RUnlock() - return len(s.callbackListeners) > 0 + return len(s.setCallbackListeners) > 0 } func (s *vdrSet) Map() map[ids.NodeID]*GetValidatorOutput { @@ -305,11 +311,21 @@ func (s *vdrSet) prefixedString(prefix string) string { return sb.String() } +func (s *vdrSet) RegisterManagerCallbackListener(callbackListener ManagerCallbackListener) { + s.lock.Lock() + defer s.lock.Unlock() + + s.managerCallbackListeners = append(s.managerCallbackListeners, callbackListener) + for _, vdr := range s.vdrSlice { + callbackListener.OnValidatorAdded(s.subnetID, vdr.NodeID, vdr.PublicKey, vdr.TxID, vdr.Weight) + } +} + func (s *vdrSet) RegisterCallbackListener(callbackListener SetCallbackListener) { s.lock.Lock() defer s.lock.Unlock() - s.callbackListeners = append(s.callbackListeners, callbackListener) + s.setCallbackListeners = append(s.setCallbackListeners, callbackListener) for _, vdr := range s.vdrSlice { callbackListener.OnValidatorAdded(vdr.NodeID, vdr.PublicKey, vdr.TxID, vdr.Weight) } @@ -317,21 +333,30 @@ func (s *vdrSet) RegisterCallbackListener(callbackListener SetCallbackListener) // Assumes [s.lock] is held func (s *vdrSet) callWeightChangeCallbacks(node ids.NodeID, oldWeight, newWeight uint64) { - for _, callbackListener := range s.callbackListeners { + for _, callbackListener := range s.managerCallbackListeners { + callbackListener.OnValidatorWeightChanged(s.subnetID, node, oldWeight, newWeight) + } + for _, callbackListener := range s.setCallbackListeners { callbackListener.OnValidatorWeightChanged(node, oldWeight, newWeight) } } // Assumes [s.lock] is held func (s *vdrSet) callValidatorAddedCallbacks(node ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { - for _, callbackListener := range s.callbackListeners { + for _, callbackListener := range s.managerCallbackListeners { + callbackListener.OnValidatorAdded(s.subnetID, node, pk, txID, weight) + } + for _, callbackListener := range s.setCallbackListeners { callbackListener.OnValidatorAdded(node, pk, txID, weight) } } // Assumes [s.lock] is held func (s *vdrSet) callValidatorRemovedCallbacks(node ids.NodeID, weight uint64) { - for _, callbackListener := range s.callbackListeners { + for _, callbackListener := range s.managerCallbackListeners { + callbackListener.OnValidatorRemoved(s.subnetID, node, weight) + } + for _, callbackListener := range s.setCallbackListeners { callbackListener.OnValidatorRemoved(node, weight) } } diff --git a/snow/validators/set_test.go b/snow/validators/set_test.go index 4554f930fa37..8e6b1af53ae6 100644 --- a/snow/validators/set_test.go +++ b/snow/validators/set_test.go @@ -20,7 +20,7 @@ import ( func TestSetAddDuplicate(t *testing.T) { require := require.New(t) - s := newSet() + s := newSet(ids.Empty, nil) nodeID := ids.GenerateTestNodeID() require.NoError(s.Add(nodeID, nil, ids.Empty, 1)) @@ -32,7 +32,7 @@ func TestSetAddDuplicate(t *testing.T) { func TestSetAddOverflow(t *testing.T) { require := require.New(t) - s := newSet() + s := newSet(ids.Empty, nil) require.NoError(s.Add(ids.GenerateTestNodeID(), nil, ids.Empty, 1)) require.NoError(s.Add(ids.GenerateTestNodeID(), nil, ids.Empty, math.MaxUint64)) @@ -44,7 +44,7 @@ func TestSetAddOverflow(t *testing.T) { func TestSetAddWeightOverflow(t *testing.T) { require := require.New(t) - s := newSet() + s := newSet(ids.Empty, nil) require.NoError(s.Add(ids.GenerateTestNodeID(), nil, ids.Empty, 1)) @@ -60,7 +60,7 @@ func TestSetAddWeightOverflow(t *testing.T) { func TestSetGetWeight(t *testing.T) { require := require.New(t) - s := newSet() + s := newSet(ids.Empty, nil) nodeID := ids.GenerateTestNodeID() require.Zero(s.GetWeight(nodeID)) @@ -83,7 +83,7 @@ func TestSetSubsetWeight(t *testing.T) { subset := set.Of(nodeID0, nodeID1) - s := newSet() + s := newSet(ids.Empty, nil) require.NoError(s.Add(nodeID0, nil, ids.Empty, weight0)) require.NoError(s.Add(nodeID1, nil, ids.Empty, weight1)) @@ -98,7 +98,7 @@ func TestSetSubsetWeight(t *testing.T) { func TestSetRemoveWeightMissingValidator(t *testing.T) { require := require.New(t) - s := newSet() + s := newSet(ids.Empty, nil) require.NoError(s.Add(ids.GenerateTestNodeID(), nil, ids.Empty, 1)) @@ -109,7 +109,7 @@ func TestSetRemoveWeightMissingValidator(t *testing.T) { func TestSetRemoveWeightUnderflow(t *testing.T) { require := require.New(t) - s := newSet() + s := newSet(ids.Empty, nil) require.NoError(s.Add(ids.GenerateTestNodeID(), nil, ids.Empty, 1)) @@ -127,7 +127,7 @@ func TestSetRemoveWeightUnderflow(t *testing.T) { func TestSetGet(t *testing.T) { require := require.New(t) - s := newSet() + s := newSet(ids.Empty, nil) nodeID := ids.GenerateTestNodeID() _, ok := s.Get(nodeID) @@ -164,7 +164,7 @@ func TestSetGet(t *testing.T) { func TestSetLen(t *testing.T) { require := require.New(t) - s := newSet() + s := newSet(ids.Empty, nil) setLen := s.Len() require.Zero(setLen) @@ -195,7 +195,7 @@ func TestSetLen(t *testing.T) { func TestSetMap(t *testing.T) { require := require.New(t) - s := newSet() + s := newSet(ids.Empty, nil) m := s.Map() require.Empty(m) @@ -278,7 +278,7 @@ func TestSetWeight(t *testing.T) { vdr1 := ids.BuildTestNodeID([]byte{2}) weight1 := uint64(123) - s := newSet() + s := newSet(ids.Empty, nil) require.NoError(s.Add(vdr0, nil, ids.Empty, weight0)) require.NoError(s.Add(vdr1, nil, ids.Empty, weight1)) @@ -292,7 +292,7 @@ func TestSetWeight(t *testing.T) { func TestSetSample(t *testing.T) { require := require.New(t) - s := newSet() + s := newSet(ids.Empty, nil) sampled, err := s.Sample(0) require.NoError(err) @@ -337,7 +337,7 @@ func TestSetString(t *testing.T) { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, }) - s := newSet() + s := newSet(ids.Empty, nil) require.NoError(s.Add(nodeID0, nil, ids.Empty, 1)) require.NoError(s.Add(nodeID1, nil, ids.Empty, math.MaxInt64-1)) @@ -392,7 +392,7 @@ func TestSetAddCallback(t *testing.T) { txID0 := ids.GenerateTestID() weight0 := uint64(1) - s := newSet() + s := newSet(ids.Empty, nil) callCount := 0 require.False(s.HasCallbackRegistered()) s.RegisterCallbackListener(&callbackListener{ @@ -418,7 +418,7 @@ func TestSetAddWeightCallback(t *testing.T) { weight0 := uint64(1) weight1 := uint64(93) - s := newSet() + s := newSet(ids.Empty, nil) require.NoError(s.Add(nodeID0, nil, txID0, weight0)) callCount := 0 @@ -452,7 +452,7 @@ func TestSetRemoveWeightCallback(t *testing.T) { weight0 := uint64(93) weight1 := uint64(92) - s := newSet() + s := newSet(ids.Empty, nil) require.NoError(s.Add(nodeID0, nil, txID0, weight0)) callCount := 0 @@ -485,7 +485,7 @@ func TestSetValidatorRemovedCallback(t *testing.T) { txID0 := ids.GenerateTestID() weight0 := uint64(93) - s := newSet() + s := newSet(ids.Empty, nil) require.NoError(s.Add(nodeID0, nil, txID0, weight0)) callCount := 0 diff --git a/vms/platformvm/vm.go b/vms/platformvm/vm.go index 214e7246ce37..4458259eacee 100644 --- a/vms/platformvm/vm.go +++ b/vms/platformvm/vm.go @@ -353,7 +353,7 @@ func (vm *VM) onNormalOperationsStarted() error { } vl := validators.NewLogger(vm.ctx.Log, constants.PrimaryNetworkID, vm.ctx.NodeID) - vm.Validators.RegisterCallbackListener(constants.PrimaryNetworkID, vl) + vm.Validators.RegisterSetCallbackListener(constants.PrimaryNetworkID, vl) for subnetID := range vm.TrackedSubnets { vdrIDs := vm.Validators.GetValidatorIDs(subnetID) @@ -362,7 +362,7 @@ func (vm *VM) onNormalOperationsStarted() error { } vl := validators.NewLogger(vm.ctx.Log, subnetID, vm.ctx.NodeID) - vm.Validators.RegisterCallbackListener(subnetID, vl) + vm.Validators.RegisterSetCallbackListener(subnetID, vl) } if err := vm.state.Commit(); err != nil { diff --git a/vms/platformvm/vm_test.go b/vms/platformvm/vm_test.go index 7fd32ef8a21e..a43619144323 100644 --- a/vms/platformvm/vm_test.go +++ b/vms/platformvm/vm_test.go @@ -1459,7 +1459,7 @@ func TestBootstrapPartiallyAccepted(t *testing.T) { totalWeight, err := beacons.TotalWeight(ctx.SubnetID) require.NoError(err) startup := tracker.NewStartup(peers, (totalWeight+1)/2) - beacons.RegisterCallbackListener(ctx.SubnetID, startup) + beacons.RegisterSetCallbackListener(ctx.SubnetID, startup) // The engine handles consensus snowGetHandler, err := snowgetter.New( From 01aaf16905af11690a2b7559232c923ac735644b Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Thu, 18 Apr 2024 13:23:49 -0400 Subject: [PATCH 02/62] Add tests --- snow/validators/manager.go | 1 + snow/validators/manager_test.go | 375 ++++++++++++++++++++++++-------- snow/validators/set_test.go | 74 +++---- 3 files changed, 317 insertions(+), 133 deletions(-) diff --git a/snow/validators/manager.go b/snow/validators/manager.go index 1b5be8ca1874..fcf37934112d 100644 --- a/snow/validators/manager.go +++ b/snow/validators/manager.go @@ -279,6 +279,7 @@ func (m *manager) RegisterCallbackListener(listener ManagerCallbackListener) { m.lock.Lock() defer m.lock.Unlock() + m.callbackListeners = append(m.callbackListeners, listener) for _, set := range m.subnetToVdrs { set.RegisterManagerCallbackListener(listener) } diff --git a/snow/validators/manager_test.go b/snow/validators/manager_test.go index 8be3797994fb..cf23d49d39be 100644 --- a/snow/validators/manager_test.go +++ b/snow/validators/manager_test.go @@ -17,6 +17,39 @@ import ( safemath "github.com/ava-labs/avalanchego/utils/math" ) +var _ ManagerCallbackListener = (*managerCallbackListener)(nil) + +type managerCallbackListener struct { + t *testing.T + onAdd func(ids.ID, ids.NodeID, *bls.PublicKey, ids.ID, uint64) + onWeight func(ids.ID, ids.NodeID, uint64, uint64) + onRemoved func(ids.ID, ids.NodeID, uint64) +} + +func (c *managerCallbackListener) OnValidatorAdded(subnetID ids.ID, nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { + if c.onAdd != nil { + c.onAdd(subnetID, nodeID, pk, txID, weight) + } else { + c.t.Fail() + } +} + +func (c *managerCallbackListener) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, weight uint64) { + if c.onRemoved != nil { + c.onRemoved(subnetID, nodeID, weight) + } else { + c.t.Fail() + } +} + +func (c *managerCallbackListener) OnValidatorWeightChanged(subnetID ids.ID, nodeID ids.NodeID, oldWeight, newWeight uint64) { + if c.onWeight != nil { + c.onWeight(subnetID, nodeID, oldWeight, newWeight) + } else { + c.t.Fail() + } +} + func TestAddZeroWeight(t *testing.T) { require := require.New(t) @@ -411,142 +444,292 @@ func TestString(t *testing.T) { func TestAddCallback(t *testing.T) { require := require.New(t) - nodeID0 := ids.BuildTestNodeID([]byte{1}) - sk0, err := bls.NewSecretKey() + expectedSK, err := bls.NewSecretKey() require.NoError(err) - pk0 := bls.PublicFromSecretKey(sk0) - txID0 := ids.GenerateTestID() - weight0 := uint64(1) - m := NewManager() - subnetID := ids.GenerateTestID() - callCount := 0 - m.RegisterSetCallbackListener(subnetID, &callbackListener{ + var ( + expectedNodeID = ids.GenerateTestNodeID() + expectedPK = bls.PublicFromSecretKey(expectedSK) + expectedTxID = ids.GenerateTestID() + expectedWeight uint64 = 1 + expectedSubnetID0 = ids.GenerateTestID() + expectedSubnetID1 = ids.GenerateTestID() + + m = NewManager() + managerCallCount = 0 + setCallCount = 0 + ) + m.RegisterCallbackListener(&managerCallbackListener{ + t: t, + onAdd: func(subnetID ids.ID, nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { + require.Contains([]ids.ID{expectedSubnetID0, expectedSubnetID1}, subnetID) + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedPK, pk) + require.Equal(expectedTxID, txID) + require.Equal(expectedWeight, weight) + managerCallCount++ + }, + }) + m.RegisterSetCallbackListener(expectedSubnetID0, &setCallbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { - require.Equal(nodeID0, nodeID) - require.Equal(pk0, pk) - require.Equal(txID0, txID) - require.Equal(weight0, weight) - callCount++ + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedPK, pk) + require.Equal(expectedTxID, txID) + require.Equal(expectedWeight, weight) + setCallCount++ }, }) - require.NoError(m.AddStaker(subnetID, nodeID0, pk0, txID0, weight0)) - // setup another subnetID - subnetID2 := ids.GenerateTestID() - require.NoError(m.AddStaker(subnetID2, nodeID0, nil, txID0, weight0)) - // should not be called for subnetID2 - require.Equal(1, callCount) + require.NoError(m.AddStaker(expectedSubnetID0, expectedNodeID, expectedPK, expectedTxID, expectedWeight)) + require.Equal(1, managerCallCount) // should be called for expectedSubnetID0 + require.Equal(1, setCallCount) // should be called for expectedSubnetID0 + + require.NoError(m.AddStaker(expectedSubnetID1, expectedNodeID, expectedPK, expectedTxID, expectedWeight)) + require.Equal(2, managerCallCount) // should be called for expectedSubnetID1 + require.Equal(1, setCallCount) // should not be called for expectedSubnetID1 } func TestAddWeightCallback(t *testing.T) { require := require.New(t) - nodeID0 := ids.BuildTestNodeID([]byte{1}) - txID0 := ids.GenerateTestID() - weight0 := uint64(1) - weight1 := uint64(93) - - m := NewManager() - subnetID := ids.GenerateTestID() - require.NoError(m.AddStaker(subnetID, nodeID0, nil, txID0, weight0)) + expectedSK, err := bls.NewSecretKey() + require.NoError(err) - callCount := 0 - m.RegisterSetCallbackListener(subnetID, &callbackListener{ + var ( + expectedNodeID = ids.GenerateTestNodeID() + expectedPK = bls.PublicFromSecretKey(expectedSK) + expectedTxID = ids.GenerateTestID() + expectedOldWeight uint64 = 1 + expectedAddedWeight uint64 = 10 + expectedNewWeight = expectedOldWeight + expectedAddedWeight + expectedSubnetID0 = ids.GenerateTestID() + expectedSubnetID1 = ids.GenerateTestID() + + m = NewManager() + managerAddCallCount = 0 + managerChangeCallCount = 0 + setAddCallCount = 0 + setChangeCallCount = 0 + ) + + require.NoError(m.AddStaker(expectedSubnetID0, expectedNodeID, expectedPK, expectedTxID, expectedOldWeight)) + + m.RegisterCallbackListener(&managerCallbackListener{ + t: t, + onAdd: func(subnetID ids.ID, nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { + require.Contains([]ids.ID{expectedSubnetID0, expectedSubnetID1}, subnetID) + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedPK, pk) + require.Equal(expectedTxID, txID) + require.Equal(expectedOldWeight, weight) + managerAddCallCount++ + }, + onWeight: func(subnetID ids.ID, nodeID ids.NodeID, oldWeight, newWeight uint64) { + require.Contains([]ids.ID{expectedSubnetID0, expectedSubnetID1}, subnetID) + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedOldWeight, oldWeight) + require.Equal(expectedNewWeight, newWeight) + managerChangeCallCount++ + }, + }) + m.RegisterSetCallbackListener(expectedSubnetID0, &setCallbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { - require.Equal(nodeID0, nodeID) - require.Nil(pk) - require.Equal(txID0, txID) - require.Equal(weight0, weight) - callCount++ + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedPK, pk) + require.Equal(expectedTxID, txID) + require.Equal(expectedOldWeight, weight) + setAddCallCount++ }, onWeight: func(nodeID ids.NodeID, oldWeight, newWeight uint64) { - require.Equal(nodeID0, nodeID) - require.Equal(weight0, oldWeight) - require.Equal(weight0+weight1, newWeight) - callCount++ + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedOldWeight, oldWeight) + require.Equal(expectedNewWeight, newWeight) + setChangeCallCount++ }, }) - require.NoError(m.AddWeight(subnetID, nodeID0, weight1)) - // setup another subnetID - subnetID2 := ids.GenerateTestID() - require.NoError(m.AddStaker(subnetID2, nodeID0, nil, txID0, weight0)) - require.NoError(m.AddWeight(subnetID2, nodeID0, weight1)) - // should not be called for subnetID2 - require.Equal(2, callCount) + require.Equal(1, managerAddCallCount) + require.Zero(managerChangeCallCount) + require.Equal(1, setAddCallCount) + require.Zero(setChangeCallCount) + + require.NoError(m.AddWeight(expectedSubnetID0, expectedNodeID, expectedAddedWeight)) + require.Equal(1, managerAddCallCount) + require.Equal(1, managerChangeCallCount) + require.Equal(1, setAddCallCount) + require.Equal(1, setChangeCallCount) + + require.NoError(m.AddStaker(expectedSubnetID1, expectedNodeID, expectedPK, expectedTxID, expectedOldWeight)) + require.Equal(2, managerAddCallCount) + require.Equal(1, managerChangeCallCount) + require.Equal(1, setAddCallCount) + require.Equal(1, setChangeCallCount) + + require.NoError(m.AddWeight(expectedSubnetID1, expectedNodeID, expectedAddedWeight)) + require.Equal(2, managerAddCallCount) + require.Equal(2, managerChangeCallCount) + require.Equal(1, setAddCallCount) + require.Equal(1, setChangeCallCount) } func TestRemoveWeightCallback(t *testing.T) { require := require.New(t) - nodeID0 := ids.BuildTestNodeID([]byte{1}) - txID0 := ids.GenerateTestID() - weight0 := uint64(93) - weight1 := uint64(92) - - m := NewManager() - subnetID := ids.GenerateTestID() - require.NoError(m.AddStaker(subnetID, nodeID0, nil, txID0, weight0)) + expectedSK, err := bls.NewSecretKey() + require.NoError(err) - callCount := 0 - m.RegisterSetCallbackListener(subnetID, &callbackListener{ + var ( + expectedNodeID = ids.GenerateTestNodeID() + expectedPK = bls.PublicFromSecretKey(expectedSK) + expectedTxID = ids.GenerateTestID() + expectedNewWeight uint64 = 1 + expectedRemovedWeight uint64 = 10 + expectedOldWeight = expectedNewWeight + expectedRemovedWeight + expectedSubnetID0 = ids.GenerateTestID() + expectedSubnetID1 = ids.GenerateTestID() + + m = NewManager() + managerAddCallCount = 0 + managerChangeCallCount = 0 + setAddCallCount = 0 + setChangeCallCount = 0 + ) + + require.NoError(m.AddStaker(expectedSubnetID0, expectedNodeID, expectedPK, expectedTxID, expectedOldWeight)) + + m.RegisterCallbackListener(&managerCallbackListener{ + t: t, + onAdd: func(subnetID ids.ID, nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { + require.Contains([]ids.ID{expectedSubnetID0, expectedSubnetID1}, subnetID) + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedPK, pk) + require.Equal(expectedTxID, txID) + require.Equal(expectedOldWeight, weight) + managerAddCallCount++ + }, + onWeight: func(subnetID ids.ID, nodeID ids.NodeID, oldWeight, newWeight uint64) { + require.Contains([]ids.ID{expectedSubnetID0, expectedSubnetID1}, subnetID) + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedOldWeight, oldWeight) + require.Equal(expectedNewWeight, newWeight) + managerChangeCallCount++ + }, + }) + m.RegisterSetCallbackListener(expectedSubnetID0, &setCallbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { - require.Equal(nodeID0, nodeID) - require.Nil(pk) - require.Equal(txID0, txID) - require.Equal(weight0, weight) - callCount++ + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedPK, pk) + require.Equal(expectedTxID, txID) + require.Equal(expectedOldWeight, weight) + setAddCallCount++ }, onWeight: func(nodeID ids.NodeID, oldWeight, newWeight uint64) { - require.Equal(nodeID0, nodeID) - require.Equal(weight0, oldWeight) - require.Equal(weight0-weight1, newWeight) - callCount++ + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedOldWeight, oldWeight) + require.Equal(expectedNewWeight, newWeight) + setChangeCallCount++ }, }) - require.NoError(m.RemoveWeight(subnetID, nodeID0, weight1)) - // setup another subnetID - subnetID2 := ids.GenerateTestID() - require.NoError(m.AddStaker(subnetID2, nodeID0, nil, txID0, weight0)) - require.NoError(m.RemoveWeight(subnetID2, nodeID0, weight1)) - // should not be called for subnetID2 - require.Equal(2, callCount) + require.Equal(1, managerAddCallCount) + require.Zero(managerChangeCallCount) + require.Equal(1, setAddCallCount) + require.Zero(setChangeCallCount) + + require.NoError(m.RemoveWeight(expectedSubnetID0, expectedNodeID, expectedRemovedWeight)) + require.Equal(1, managerAddCallCount) + require.Equal(1, managerChangeCallCount) + require.Equal(1, setAddCallCount) + require.Equal(1, setChangeCallCount) + + require.NoError(m.AddStaker(expectedSubnetID1, expectedNodeID, expectedPK, expectedTxID, expectedOldWeight)) + require.Equal(2, managerAddCallCount) + require.Equal(1, managerChangeCallCount) + require.Equal(1, setAddCallCount) + require.Equal(1, setChangeCallCount) + + require.NoError(m.RemoveWeight(expectedSubnetID1, expectedNodeID, expectedRemovedWeight)) + require.Equal(2, managerAddCallCount) + require.Equal(2, managerChangeCallCount) + require.Equal(1, setAddCallCount) + require.Equal(1, setChangeCallCount) } -func TestValidatorRemovedCallback(t *testing.T) { +func TestRemoveCallback(t *testing.T) { require := require.New(t) - nodeID0 := ids.BuildTestNodeID([]byte{1}) - txID0 := ids.GenerateTestID() - weight0 := uint64(93) + expectedSK, err := bls.NewSecretKey() + require.NoError(err) - m := NewManager() - subnetID := ids.GenerateTestID() - require.NoError(m.AddStaker(subnetID, nodeID0, nil, txID0, weight0)) + var ( + expectedNodeID = ids.GenerateTestNodeID() + expectedPK = bls.PublicFromSecretKey(expectedSK) + expectedTxID = ids.GenerateTestID() + expectedWeight uint64 = 1 + expectedSubnetID0 = ids.GenerateTestID() + expectedSubnetID1 = ids.GenerateTestID() + + m = NewManager() + managerAddCallCount = 0 + managerRemoveCallCount = 0 + setAddCallCount = 0 + setRemoveCallCount = 0 + ) - callCount := 0 - m.RegisterSetCallbackListener(subnetID, &callbackListener{ + require.NoError(m.AddStaker(expectedSubnetID0, expectedNodeID, expectedPK, expectedTxID, expectedWeight)) + + m.RegisterCallbackListener(&managerCallbackListener{ + t: t, + onAdd: func(subnetID ids.ID, nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { + require.Contains([]ids.ID{expectedSubnetID0, expectedSubnetID1}, subnetID) + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedPK, pk) + require.Equal(expectedTxID, txID) + require.Equal(expectedWeight, weight) + managerAddCallCount++ + }, + onRemoved: func(subnetID ids.ID, nodeID ids.NodeID, weight uint64) { + require.Contains([]ids.ID{expectedSubnetID0, expectedSubnetID1}, subnetID) + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedWeight, weight) + managerRemoveCallCount++ + }, + }) + m.RegisterSetCallbackListener(expectedSubnetID0, &setCallbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { - require.Equal(nodeID0, nodeID) - require.Nil(pk) - require.Equal(txID0, txID) - require.Equal(weight0, weight) - callCount++ + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedPK, pk) + require.Equal(expectedTxID, txID) + require.Equal(expectedWeight, weight) + setAddCallCount++ }, onRemoved: func(nodeID ids.NodeID, weight uint64) { - require.Equal(nodeID0, nodeID) - require.Equal(weight0, weight) - callCount++ + require.Equal(expectedNodeID, nodeID) + require.Equal(expectedWeight, weight) + setRemoveCallCount++ }, }) - require.NoError(m.RemoveWeight(subnetID, nodeID0, weight0)) - // setup another subnetID - subnetID2 := ids.GenerateTestID() - require.NoError(m.AddStaker(subnetID2, nodeID0, nil, txID0, weight0)) - require.NoError(m.AddWeight(subnetID2, nodeID0, weight0)) - // should not be called for subnetID2 - require.Equal(2, callCount) + require.Equal(1, managerAddCallCount) + require.Zero(managerRemoveCallCount) + require.Equal(1, setAddCallCount) + require.Zero(setRemoveCallCount) + + require.NoError(m.RemoveWeight(expectedSubnetID0, expectedNodeID, expectedWeight)) + require.Equal(1, managerAddCallCount) + require.Equal(1, managerRemoveCallCount) + require.Equal(1, setAddCallCount) + require.Equal(1, setRemoveCallCount) + + require.NoError(m.AddStaker(expectedSubnetID1, expectedNodeID, expectedPK, expectedTxID, expectedWeight)) + require.Equal(2, managerAddCallCount) + require.Equal(1, managerRemoveCallCount) + require.Equal(1, setAddCallCount) + require.Equal(1, setRemoveCallCount) + + require.NoError(m.RemoveWeight(expectedSubnetID1, expectedNodeID, expectedWeight)) + require.Equal(2, managerAddCallCount) + require.Equal(2, managerRemoveCallCount) + require.Equal(1, setAddCallCount) + require.Equal(1, setRemoveCallCount) } diff --git a/snow/validators/set_test.go b/snow/validators/set_test.go index 8e6b1af53ae6..480f9dba4f8e 100644 --- a/snow/validators/set_test.go +++ b/snow/validators/set_test.go @@ -17,6 +17,39 @@ import ( safemath "github.com/ava-labs/avalanchego/utils/math" ) +var _ SetCallbackListener = (*setCallbackListener)(nil) + +type setCallbackListener struct { + t *testing.T + onAdd func(ids.NodeID, *bls.PublicKey, ids.ID, uint64) + onWeight func(ids.NodeID, uint64, uint64) + onRemoved func(ids.NodeID, uint64) +} + +func (c *setCallbackListener) OnValidatorAdded(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { + if c.onAdd != nil { + c.onAdd(nodeID, pk, txID, weight) + } else { + c.t.Fail() + } +} + +func (c *setCallbackListener) OnValidatorRemoved(nodeID ids.NodeID, weight uint64) { + if c.onRemoved != nil { + c.onRemoved(nodeID, weight) + } else { + c.t.Fail() + } +} + +func (c *setCallbackListener) OnValidatorWeightChanged(nodeID ids.NodeID, oldWeight, newWeight uint64) { + if c.onWeight != nil { + c.onWeight(nodeID, oldWeight, newWeight) + } else { + c.t.Fail() + } +} + func TestSetAddDuplicate(t *testing.T) { require := require.New(t) @@ -349,39 +382,6 @@ func TestSetString(t *testing.T) { require.Equal(expected, result) } -var _ SetCallbackListener = (*callbackListener)(nil) - -type callbackListener struct { - t *testing.T - onAdd func(ids.NodeID, *bls.PublicKey, ids.ID, uint64) - onWeight func(ids.NodeID, uint64, uint64) - onRemoved func(ids.NodeID, uint64) -} - -func (c *callbackListener) OnValidatorAdded(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { - if c.onAdd != nil { - c.onAdd(nodeID, pk, txID, weight) - } else { - c.t.Fail() - } -} - -func (c *callbackListener) OnValidatorRemoved(nodeID ids.NodeID, weight uint64) { - if c.onRemoved != nil { - c.onRemoved(nodeID, weight) - } else { - c.t.Fail() - } -} - -func (c *callbackListener) OnValidatorWeightChanged(nodeID ids.NodeID, oldWeight, newWeight uint64) { - if c.onWeight != nil { - c.onWeight(nodeID, oldWeight, newWeight) - } else { - c.t.Fail() - } -} - func TestSetAddCallback(t *testing.T) { require := require.New(t) @@ -395,7 +395,7 @@ func TestSetAddCallback(t *testing.T) { s := newSet(ids.Empty, nil) callCount := 0 require.False(s.HasCallbackRegistered()) - s.RegisterCallbackListener(&callbackListener{ + s.RegisterCallbackListener(&setCallbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { require.Equal(nodeID0, nodeID) @@ -423,7 +423,7 @@ func TestSetAddWeightCallback(t *testing.T) { callCount := 0 require.False(s.HasCallbackRegistered()) - s.RegisterCallbackListener(&callbackListener{ + s.RegisterCallbackListener(&setCallbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { require.Equal(nodeID0, nodeID) @@ -457,7 +457,7 @@ func TestSetRemoveWeightCallback(t *testing.T) { callCount := 0 require.False(s.HasCallbackRegistered()) - s.RegisterCallbackListener(&callbackListener{ + s.RegisterCallbackListener(&setCallbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { require.Equal(nodeID0, nodeID) @@ -490,7 +490,7 @@ func TestSetValidatorRemovedCallback(t *testing.T) { callCount := 0 require.False(s.HasCallbackRegistered()) - s.RegisterCallbackListener(&callbackListener{ + s.RegisterCallbackListener(&setCallbackListener{ t: t, onAdd: func(nodeID ids.NodeID, pk *bls.PublicKey, txID ids.ID, weight uint64) { require.Equal(nodeID0, nodeID) From 542ac269f3e54f13ffeb851a419c27e736136957 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Thu, 18 Apr 2024 14:44:22 -0400 Subject: [PATCH 03/62] Expose all validator sets to IP tracking --- network/ip_tracker.go | 17 ++++-- network/ip_tracker_test.go | 113 +++++++++++++++++++------------------ network/network.go | 2 +- 3 files changed, 71 insertions(+), 61 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 8bca76d7d755..6fe7293686f6 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -14,6 +14,7 @@ import ( "github.com/ava-labs/avalanchego/snow/validators" "github.com/ava-labs/avalanchego/utils" "github.com/ava-labs/avalanchego/utils/bloom" + "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/crypto/bls" "github.com/ava-labs/avalanchego/utils/ips" "github.com/ava-labs/avalanchego/utils/logging" @@ -38,7 +39,7 @@ const ( newTimestamp = 2 ) -var _ validators.SetCallbackListener = (*ipTracker)(nil) +var _ validators.ManagerCallbackListener = (*ipTracker)(nil) func newIPTracker( log logging.Logger, @@ -260,7 +261,11 @@ func (i *ipTracker) Disconnected(nodeID ids.NodeID) { i.removeGossipableIP(nodeID) } -func (i *ipTracker) OnValidatorAdded(nodeID ids.NodeID, _ *bls.PublicKey, _ ids.ID, _ uint64) { +func (i *ipTracker) OnValidatorAdded(subnetID ids.ID, nodeID ids.NodeID, _ *bls.PublicKey, _ ids.ID, _ uint64) { + if subnetID != constants.PrimaryNetworkID { + return + } + i.lock.Lock() defer i.lock.Unlock() @@ -302,9 +307,13 @@ func (i *ipTracker) addGossipableID(nodeID ids.NodeID) { i.addGossipableIP(connectedIP) } -func (*ipTracker) OnValidatorWeightChanged(ids.NodeID, uint64, uint64) {} +func (*ipTracker) OnValidatorWeightChanged(ids.ID, ids.NodeID, uint64, uint64) {} + +func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uint64) { + if subnetID != constants.PrimaryNetworkID { + return + } -func (i *ipTracker) OnValidatorRemoved(nodeID ids.NodeID, _ uint64) { i.lock.Lock() defer i.lock.Unlock() diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index be9ea59a51ee..f27eec276720 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -12,6 +12,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/bloom" + "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/ips" "github.com/ava-labs/avalanchego/utils/logging" ) @@ -93,13 +94,13 @@ func TestIPTracker_ManuallyTrack(t *testing.T) { name: "non-connected validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.manuallyTracked.Add(ip.NodeID) return tracker }(), @@ -109,14 +110,14 @@ func TestIPTracker_ManuallyTrack(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.manuallyTracked.Add(ip.NodeID) return tracker }(), @@ -179,13 +180,13 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { name: "non-connected validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.manuallyTracked.Add(ip.NodeID) tracker.manuallyGossipable.Add(ip.NodeID) return tracker @@ -196,14 +197,14 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.manuallyTracked.Add(ip.NodeID) tracker.manuallyGossipable.Add(ip.NodeID) return tracker @@ -239,14 +240,14 @@ func TestIPTracker_AddIP(t *testing.T) { name: "first known IP", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), ip: ip, expectedUpdated: true, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.mostRecentTrackedIPs[ip.NodeID] = ip tracker.bloomAdditions[ip.NodeID] = 1 return tracker @@ -256,7 +257,7 @@ func TestIPTracker_AddIP(t *testing.T) { name: "older IP", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) return tracker }(), @@ -264,7 +265,7 @@ func TestIPTracker_AddIP(t *testing.T) { expectedUpdated: false, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) return tracker }(), @@ -273,7 +274,7 @@ func TestIPTracker_AddIP(t *testing.T) { name: "same IP", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker }(), @@ -281,7 +282,7 @@ func TestIPTracker_AddIP(t *testing.T) { expectedUpdated: false, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker }(), @@ -290,7 +291,7 @@ func TestIPTracker_AddIP(t *testing.T) { name: "disconnected newer IP", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker }(), @@ -298,7 +299,7 @@ func TestIPTracker_AddIP(t *testing.T) { expectedUpdated: true, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) tracker.mostRecentTrackedIPs[newerIP.NodeID] = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 @@ -309,7 +310,7 @@ func TestIPTracker_AddIP(t *testing.T) { name: "connected newer IP", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) return tracker }(), @@ -317,7 +318,7 @@ func TestIPTracker_AddIP(t *testing.T) { expectedUpdated: true, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) tracker.mostRecentTrackedIPs[newerIP.NodeID] = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 @@ -359,13 +360,13 @@ func TestIPTracker_Connected(t *testing.T) { name: "first known IP", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), ip: ip, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.mostRecentTrackedIPs[ip.NodeID] = ip tracker.bloomAdditions[ip.NodeID] = 1 tracker.connected[ip.NodeID] = ip @@ -380,14 +381,14 @@ func TestIPTracker_Connected(t *testing.T) { name: "connected with older IP", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) return tracker }(), ip: ip, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) tracker.connected[ip.NodeID] = ip return tracker @@ -397,14 +398,14 @@ func TestIPTracker_Connected(t *testing.T) { name: "connected with newer IP", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker }(), ip: newerIP, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) tracker.mostRecentTrackedIPs[newerIP.NodeID] = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 @@ -420,14 +421,14 @@ func TestIPTracker_Connected(t *testing.T) { name: "connected with same IP", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker }(), ip: ip, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) tracker.connected[ip.NodeID] = ip tracker.gossipableIndices[ip.NodeID] = 0 @@ -485,14 +486,14 @@ func TestIPTracker_Disconnected(t *testing.T) { name: "latest gossipable", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) delete(tracker.connected, ip.NodeID) delete(tracker.gossipableIndices, ip.NodeID) @@ -504,18 +505,18 @@ func TestIPTracker_Disconnected(t *testing.T) { name: "non-latest gossipable", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) tracker.Connected(otherIP) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) tracker.Connected(otherIP) delete(tracker.connected, ip.NodeID) tracker.gossipableIndices = map[ids.NodeID]int{ @@ -630,7 +631,7 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.OnValidatorAdded(test.nodeID, nil, ids.Empty, 0) + test.initialState.OnValidatorAdded(constants.PrimaryNetworkID, test.nodeID, nil, ids.Empty, 0) requireEqual(t, test.expectedState, test.initialState) requireMetricsConsistent(t, test.initialState) }) @@ -649,7 +650,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker }(), @@ -657,7 +658,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) tracker.gossipableIDs.Remove(ip.NodeID) return tracker @@ -668,7 +669,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) return tracker }(), @@ -676,7 +677,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) delete(tracker.gossipableIndices, ip.NodeID) tracker.gossipableIPs = tracker.gossipableIPs[:0] @@ -689,7 +690,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyGossip(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) return tracker }(), @@ -697,7 +698,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyGossip(ip.NodeID) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) return tracker }(), @@ -706,14 +707,14 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { name: "not gossipable", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) delete(tracker.mostRecentTrackedIPs, ip.NodeID) tracker.trackedIDs.Remove(ip.NodeID) @@ -725,14 +726,14 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { name: "latest gossipable", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) delete(tracker.mostRecentTrackedIPs, ip.NodeID) tracker.trackedIDs.Remove(ip.NodeID) @@ -746,18 +747,18 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { name: "non-latest gossipable", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) tracker.Connected(otherIP) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) tracker.Connected(otherIP) delete(tracker.mostRecentTrackedIPs, ip.NodeID) tracker.trackedIDs.Remove(ip.NodeID) @@ -774,7 +775,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.OnValidatorRemoved(test.nodeID, 0) + test.initialState.OnValidatorRemoved(constants.PrimaryNetworkID, test.nodeID, 0) requireEqual(t, test.expectedState, test.initialState) requireMetricsConsistent(t, test.initialState) }) @@ -787,8 +788,8 @@ func TestIPTracker_GetGossipableIPs(t *testing.T) { tracker := newTestIPTracker(t) tracker.Connected(ip) tracker.Connected(otherIP) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) gossipableIPs := tracker.GetGossipableIPs(ids.EmptyNodeID, bloom.EmptyFilter, nil, 2) require.ElementsMatch([]*ips.ClaimedIPPort{ip, otherIP}, gossipableIPs) @@ -816,8 +817,8 @@ func TestIPTracker_BloomFiltersEverything(t *testing.T) { tracker := newTestIPTracker(t) tracker.Connected(ip) tracker.Connected(otherIP) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) bloomBytes, salt := tracker.Bloom() readFilter, err := bloom.Parse(bloomBytes) @@ -837,7 +838,7 @@ func TestIPTracker_BloomGrows(t *testing.T) { { name: "Add Validator", add: func(tracker *ipTracker) { - tracker.OnValidatorAdded(ids.GenerateTestNodeID(), nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ids.GenerateTestNodeID(), nil, ids.Empty, 0) }, }, { @@ -870,11 +871,11 @@ func TestIPTracker_BloomResetsDynamically(t *testing.T) { tracker := newTestIPTracker(t) tracker.Connected(ip) - tracker.OnValidatorAdded(ip.NodeID, nil, ids.Empty, 0) - tracker.OnValidatorRemoved(ip.NodeID, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorRemoved(constants.PrimaryNetworkID, ip.NodeID, 0) tracker.maxBloomCount = 1 tracker.Connected(otherIP) - tracker.OnValidatorAdded(otherIP.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) requireMetricsConsistent(t, tracker) bloomBytes, salt := tracker.Bloom() diff --git a/network/network.go b/network/network.go index 506e51908586..051550d7b0dd 100644 --- a/network/network.go +++ b/network/network.go @@ -239,7 +239,7 @@ func NewNetwork( if err != nil { return nil, fmt.Errorf("initializing ip tracker failed with: %w", err) } - config.Validators.RegisterSetCallbackListener(constants.PrimaryNetworkID, ipTracker) + config.Validators.RegisterCallbackListener(ipTracker) // Track all default bootstrappers to ensure their current IPs are gossiped // like validator IPs. From 6d2c138d481382d21f885febbedcc25973f8f693 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Wed, 24 Apr 2024 19:25:09 -0400 Subject: [PATCH 04/62] wip --- network/ip_tracker.go | 20 +++++-- network/ip_tracker_test.go | 106 +++++++++++++++++------------------ network/metrics.go | 20 ++++--- network/network.go | 22 ++++++-- network/peer/network.go | 2 + network/peer/peer.go | 44 ++++++++++----- network/peer/test_network.go | 3 +- 7 files changed, 130 insertions(+), 87 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 6fe7293686f6..db38f232a3db 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -42,6 +42,7 @@ const ( var _ validators.ManagerCallbackListener = (*ipTracker)(nil) func newIPTracker( + trackedSubnets set.Set[ids.ID], log logging.Logger, namespace string, registerer prometheus.Registerer, @@ -52,7 +53,8 @@ func newIPTracker( return nil, err } tracker := &ipTracker{ - log: log, + trackedSubnets: trackedSubnets, + log: log, numTrackedIPs: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Name: "tracked_ips", @@ -80,6 +82,7 @@ func newIPTracker( } type ipTracker struct { + trackedSubnets set.Set[ids.ID] log logging.Logger numTrackedIPs prometheus.Gauge numGossipableIPs prometheus.Gauge @@ -96,7 +99,7 @@ type ipTracker struct { // mostRecentTrackedIPs tracks the most recent IP of each node whose // connection is desired. // - // An IP is tracked if one of the following conditions are met: + // An IP is tracked if any of the following conditions are met: // - The node was manually tracked // - The node was manually requested to be gossiped // - The node is a validator @@ -149,7 +152,11 @@ func (i *ipTracker) ManuallyTrack(nodeID ids.NodeID) { // // In order to avoid persistent network gossip, it's important for nodes in the // network to agree upon manually gossiped nodeIDs. -func (i *ipTracker) ManuallyGossip(nodeID ids.NodeID) { +func (i *ipTracker) ManuallyGossip(subnetID ids.ID, nodeID ids.NodeID) { + if subnetID != constants.PrimaryNetworkID { + return + } + i.lock.Lock() defer i.lock.Unlock() @@ -215,7 +222,9 @@ func (i *ipTracker) GetIP(nodeID ids.NodeID) (*ips.ClaimedIPPort, bool) { // Connected is called when a connection is established. The peer should have // provided [ip] during the handshake. -func (i *ipTracker) Connected(ip *ips.ClaimedIPPort) { +func (i *ipTracker) Connected(ip *ips.ClaimedIPPort, trackedSubnets set.Set[ids.ID]) { + _ = trackedSubnets + i.lock.Lock() defer i.lock.Unlock() @@ -394,11 +403,14 @@ func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) { // the number of eligible IPs to return low, it's possible that every IP will be // iterated over while handling this call. func (i *ipTracker) GetGossipableIPs( + trackedSubnets set.Set[ids.ID], exceptNodeID ids.NodeID, exceptIPs *bloom.ReadFilter, salt []byte, maxNumIPs int, ) []*ips.ClaimedIPPort { + _ = trackedSubnets + var ( uniform = sampler.NewUniform() ips = make([]*ips.ClaimedIPPort, 0, maxNumIPs) diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index f27eec276720..f8db59d29d05 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -18,7 +18,7 @@ import ( ) func newTestIPTracker(t *testing.T) *ipTracker { - tracker, err := newIPTracker(logging.NoLog{}, "", prometheus.NewRegistry()) + tracker, err := newIPTracker(nil, logging.NoLog{}, "", prometheus.NewRegistry()) require.NoError(t, err) return tracker } @@ -76,13 +76,13 @@ func TestIPTracker_ManuallyTrack(t *testing.T) { name: "connected non-validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.manuallyTracked.Add(ip.NodeID) tracker.mostRecentTrackedIPs[ip.NodeID] = ip tracker.trackedIDs.Add(ip.NodeID) @@ -109,14 +109,14 @@ func TestIPTracker_ManuallyTrack(t *testing.T) { name: "connected validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.manuallyTracked.Add(ip.NodeID) return tracker @@ -156,13 +156,13 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { name: "connected non-validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.manuallyTracked.Add(ip.NodeID) tracker.manuallyGossipable.Add(ip.NodeID) tracker.mostRecentTrackedIPs[ip.NodeID] = ip @@ -196,14 +196,14 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { name: "connected validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.manuallyTracked.Add(ip.NodeID) tracker.manuallyGossipable.Add(ip.NodeID) @@ -213,7 +213,7 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.ManuallyGossip(test.nodeID) + test.initialState.ManuallyGossip(constants.PrimaryNetworkID, test.nodeID) requireEqual(t, test.expectedState, test.initialState) requireMetricsConsistent(t, test.initialState) }) @@ -311,7 +311,7 @@ func TestIPTracker_AddIP(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) return tracker }(), ip: newerIP, @@ -319,7 +319,7 @@ func TestIPTracker_AddIP(t *testing.T) { expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.mostRecentTrackedIPs[newerIP.NodeID] = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 delete(tracker.gossipableIndices, newerIP.NodeID) @@ -441,7 +441,7 @@ func TestIPTracker_Connected(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.Connected(test.ip) + test.initialState.Connected(test.ip, nil) requireEqual(t, test.expectedState, test.initialState) requireMetricsConsistent(t, test.initialState) }) @@ -459,7 +459,7 @@ func TestIPTracker_Disconnected(t *testing.T) { name: "not tracked", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) return tracker }(), nodeID: ip.NodeID, @@ -469,14 +469,14 @@ func TestIPTracker_Disconnected(t *testing.T) { name: "not gossipable", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.ManuallyTrack(ip.NodeID) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.ManuallyTrack(ip.NodeID) delete(tracker.connected, ip.NodeID) return tracker @@ -487,14 +487,14 @@ func TestIPTracker_Disconnected(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) delete(tracker.connected, ip.NodeID) delete(tracker.gossipableIndices, ip.NodeID) tracker.gossipableIPs = tracker.gossipableIPs[:0] @@ -506,18 +506,18 @@ func TestIPTracker_Disconnected(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP) + tracker.Connected(otherIP, nil) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP) + tracker.Connected(otherIP, nil) delete(tracker.connected, ip.NodeID) tracker.gossipableIndices = map[ids.NodeID]int{ otherIP.NodeID: 0, @@ -567,7 +567,7 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.Connected(ip) + tracker.Connected(ip, nil) require.True(t, tracker.AddIP(newerIP)) return tracker }(), @@ -575,7 +575,7 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.Connected(ip) + tracker.Connected(ip, nil) require.True(t, tracker.AddIP(newerIP)) tracker.gossipableIDs.Add(ip.NodeID) return tracker @@ -585,13 +585,13 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { name: "manually gossiped", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.ManuallyGossip(ip.NodeID) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.ManuallyGossip(ip.NodeID) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) return tracker }(), }, @@ -610,13 +610,13 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { name: "connected", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.mostRecentTrackedIPs[ip.NodeID] = ip tracker.trackedIDs.Add(ip.NodeID) tracker.bloomAdditions[ip.NodeID] = 1 @@ -670,7 +670,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) return tracker }(), nodeID: ip.NodeID, @@ -678,7 +678,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) delete(tracker.gossipableIndices, ip.NodeID) tracker.gossipableIPs = tracker.gossipableIPs[:0] tracker.gossipableIDs.Remove(ip.NodeID) @@ -689,17 +689,17 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { name: "manually gossiped", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.ManuallyGossip(ip.NodeID) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.ManuallyGossip(ip.NodeID) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) return tracker }(), }, @@ -727,14 +727,14 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) delete(tracker.mostRecentTrackedIPs, ip.NodeID) tracker.trackedIDs.Remove(ip.NodeID) delete(tracker.gossipableIndices, ip.NodeID) @@ -748,18 +748,18 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP) + tracker.Connected(otherIP, nil) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP) + tracker.Connected(otherIP, nil) delete(tracker.mostRecentTrackedIPs, ip.NodeID) tracker.trackedIDs.Remove(ip.NodeID) tracker.gossipableIndices = map[ids.NodeID]int{ @@ -786,18 +786,18 @@ func TestIPTracker_GetGossipableIPs(t *testing.T) { require := require.New(t) tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.Connected(otherIP) + tracker.Connected(ip, nil) + tracker.Connected(otherIP, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - gossipableIPs := tracker.GetGossipableIPs(ids.EmptyNodeID, bloom.EmptyFilter, nil, 2) + gossipableIPs := tracker.GetGossipableIPs(nil, ids.EmptyNodeID, bloom.EmptyFilter, nil, 2) require.ElementsMatch([]*ips.ClaimedIPPort{ip, otherIP}, gossipableIPs) - gossipableIPs = tracker.GetGossipableIPs(ip.NodeID, bloom.EmptyFilter, nil, 2) + gossipableIPs = tracker.GetGossipableIPs(nil, ip.NodeID, bloom.EmptyFilter, nil, 2) require.Equal([]*ips.ClaimedIPPort{otherIP}, gossipableIPs) - gossipableIPs = tracker.GetGossipableIPs(ids.EmptyNodeID, bloom.FullFilter, nil, 2) + gossipableIPs = tracker.GetGossipableIPs(nil, ids.EmptyNodeID, bloom.FullFilter, nil, 2) require.Empty(gossipableIPs) filter, err := bloom.New(8, 1024) @@ -807,7 +807,7 @@ func TestIPTracker_GetGossipableIPs(t *testing.T) { readFilter, err := bloom.Parse(filter.Marshal()) require.NoError(err) - gossipableIPs = tracker.GetGossipableIPs(ip.NodeID, readFilter, nil, 2) + gossipableIPs = tracker.GetGossipableIPs(nil, ip.NodeID, readFilter, nil, 2) require.Equal([]*ips.ClaimedIPPort{otherIP}, gossipableIPs) } @@ -815,8 +815,8 @@ func TestIPTracker_BloomFiltersEverything(t *testing.T) { require := require.New(t) tracker := newTestIPTracker(t) - tracker.Connected(ip) - tracker.Connected(otherIP) + tracker.Connected(ip, nil) + tracker.Connected(otherIP, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) @@ -824,7 +824,7 @@ func TestIPTracker_BloomFiltersEverything(t *testing.T) { readFilter, err := bloom.Parse(bloomBytes) require.NoError(err) - gossipableIPs := tracker.GetGossipableIPs(ids.EmptyNodeID, readFilter, salt, 2) + gossipableIPs := tracker.GetGossipableIPs(nil, ids.EmptyNodeID, readFilter, salt, 2) require.Empty(gossipableIPs) require.NoError(tracker.ResetBloom()) @@ -870,11 +870,11 @@ func TestIPTracker_BloomResetsDynamically(t *testing.T) { require := require.New(t) tracker := newTestIPTracker(t) - tracker.Connected(ip) + tracker.Connected(ip, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.OnValidatorRemoved(constants.PrimaryNetworkID, ip.NodeID, 0) tracker.maxBloomCount = 1 - tracker.Connected(otherIP) + tracker.Connected(otherIP, nil) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) requireMetricsConsistent(t, tracker) @@ -893,7 +893,7 @@ func TestIPTracker_PreventBloomFilterAddition(t *testing.T) { newestIP := newerTestIP(newerIP) tracker := newTestIPTracker(t) - tracker.ManuallyGossip(ip.NodeID) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) require.True(tracker.AddIP(ip)) require.True(tracker.AddIP(newerIP)) require.True(tracker.AddIP(newestIP)) @@ -910,7 +910,7 @@ func TestIPTracker_ShouldVerifyIP(t *testing.T) { require.False(tracker.ShouldVerifyIP(ip)) tracker.ManuallyTrack(ip.NodeID) require.True(tracker.ShouldVerifyIP(ip)) - tracker.ManuallyGossip(ip.NodeID) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) require.True(tracker.ShouldVerifyIP(ip)) require.True(tracker.AddIP(ip)) require.False(tracker.ShouldVerifyIP(ip)) diff --git a/network/metrics.go b/network/metrics.go index e2a3a363b403..cb8a023b8448 100644 --- a/network/metrics.go +++ b/network/metrics.go @@ -12,11 +12,12 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/network/peer" "github.com/ava-labs/avalanchego/utils" - "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/set" ) type metrics struct { + trackedSubnets set.Set[ids.ID] + numTracked prometheus.Gauge numPeers prometheus.Gauge numSubnetPeers *prometheus.GaugeVec @@ -41,8 +42,9 @@ type metrics struct { peerConnectedStartTimesSum float64 } -func newMetrics(namespace string, registerer prometheus.Registerer, initialSubnetIDs set.Set[ids.ID]) (*metrics, error) { +func newMetrics(namespace string, registerer prometheus.Registerer, trackedSubnets set.Set[ids.ID]) (*metrics, error) { m := &metrics{ + trackedSubnets: trackedSubnets, numPeers: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Name: "peers", @@ -169,11 +171,7 @@ func newMetrics(namespace string, registerer prometheus.Registerer, initialSubne ) // init subnet tracker metrics with tracked subnets - for subnetID := range initialSubnetIDs { - // no need to track primary network ID - if subnetID == constants.PrimaryNetworkID { - continue - } + for subnetID := range trackedSubnets { // initialize to 0 subnetIDStr := subnetID.String() m.numSubnetPeers.WithLabelValues(subnetIDStr).Set(0) @@ -190,7 +188,9 @@ func (m *metrics) markConnected(peer peer.Peer) { trackedSubnets := peer.TrackedSubnets() for subnetID := range trackedSubnets { - m.numSubnetPeers.WithLabelValues(subnetID.String()).Inc() + if m.trackedSubnets.Contains(subnetID) { + m.numSubnetPeers.WithLabelValues(subnetID.String()).Inc() + } } m.lock.Lock() @@ -207,7 +207,9 @@ func (m *metrics) markDisconnected(peer peer.Peer) { trackedSubnets := peer.TrackedSubnets() for subnetID := range trackedSubnets { - m.numSubnetPeers.WithLabelValues(subnetID.String()).Dec() + if m.trackedSubnets.Contains(subnetID) { + m.numSubnetPeers.WithLabelValues(subnetID.String()).Dec() + } } m.lock.Lock() diff --git a/network/network.go b/network/network.go index 051550d7b0dd..3d5b8225cba9 100644 --- a/network/network.go +++ b/network/network.go @@ -235,7 +235,8 @@ func NewNetwork( return nil, fmt.Errorf("initializing network metrics failed with: %w", err) } - ipTracker, err := newIPTracker(log, config.Namespace, metricsRegisterer) + // TODO: Should pass in [config.TrackedSubnets] rather than [nil] + ipTracker, err := newIPTracker(nil, log, config.Namespace, metricsRegisterer) if err != nil { return nil, fmt.Errorf("initializing ip tracker failed with: %w", err) } @@ -244,7 +245,7 @@ func NewNetwork( // Track all default bootstrappers to ensure their current IPs are gossiped // like validator IPs. for _, bootstrapper := range genesis.GetBootstrappers(config.NetworkID) { - ipTracker.ManuallyGossip(bootstrapper.ID) + ipTracker.ManuallyGossip(constants.PrimaryNetworkID, bootstrapper.ID) } // Track all recent validators to optimistically connect to them before the // P-chain has finished syncing. @@ -454,14 +455,17 @@ func (n *network) Connected(nodeID ids.NodeID) { peerIP.Timestamp, peerIP.TLSSignature, ) - n.ipTracker.Connected(newIP) + trackedSubnets := peer.TrackedSubnets() + n.ipTracker.Connected(newIP, trackedSubnets) n.metrics.markConnected(peer) peerVersion := peer.Version() n.router.Connected(nodeID, peerVersion, constants.PrimaryNetworkID) - for subnetID := range peer.TrackedSubnets() { - n.router.Connected(nodeID, peerVersion, subnetID) + for subnetID := range trackedSubnets { + if n.peerConfig.MySubnets.Contains(subnetID) { + n.router.Connected(nodeID, peerVersion, subnetID) + } } } @@ -509,8 +513,14 @@ func (n *network) KnownPeers() ([]byte, []byte) { return n.ipTracker.Bloom() } -func (n *network) Peers(except ids.NodeID, knownPeers *bloom.ReadFilter, salt []byte) []*ips.ClaimedIPPort { +func (n *network) Peers( + trackedSubnets set.Set[ids.ID], + except ids.NodeID, + knownPeers *bloom.ReadFilter, + salt []byte, +) []*ips.ClaimedIPPort { return n.ipTracker.GetGossipableIPs( + trackedSubnets, except, knownPeers, salt, diff --git a/network/peer/network.go b/network/peer/network.go index b8fb01814546..eb1f5fb7ec68 100644 --- a/network/peer/network.go +++ b/network/peer/network.go @@ -7,6 +7,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/bloom" "github.com/ava-labs/avalanchego/utils/ips" + "github.com/ava-labs/avalanchego/utils/set" ) // Network defines the interface that is used by a peer to help establish a well @@ -34,6 +35,7 @@ type Network interface { // Peers returns peers that are not known. Peers( + trackedSubnets set.Set[ids.ID], peerID ids.NodeID, knownPeers *bloom.ReadFilter, peerSalt []byte, diff --git a/network/peer/peer.go b/network/peer/peer.go index 1774d3f24f97..c4059fff7f27 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -31,9 +31,14 @@ import ( "github.com/ava-labs/avalanchego/version" ) -// maxBloomSaltLen restricts the allowed size of the bloom salt to prevent -// excessively expensive bloom filter contains checks. -const maxBloomSaltLen = 32 +const ( + // maxBloomSaltLen restricts the allowed size of the bloom salt to prevent + // excessively expensive bloom filter contains checks. + maxBloomSaltLen = 32 + // maxNumTrackedSubnets limits how many subnets a peer can track to prevent + // excessive memory usage. + maxNumTrackedSubnets = 16 +) var ( errClosed = errors.New("closed") @@ -263,9 +268,8 @@ func (p *peer) Info() Info { publicIPStr = p.ip.IPPort.String() } - uptimes := make(map[ids.ID]json.Uint32, p.trackedSubnets.Len()) - - for subnetID := range p.trackedSubnets { + uptimes := make(map[ids.ID]json.Uint32, p.MySubnets.Len()) + for subnetID := range p.MySubnets { uptime, exist := p.ObservedUptime(subnetID) if !exist { continue @@ -802,8 +806,12 @@ func (p *peer) getUptimes() (uint32, []*p2p.SubnetUptime) { primaryUptime = 0 } - subnetUptimes := make([]*p2p.SubnetUptime, 0, p.trackedSubnets.Len()) - for subnetID := range p.trackedSubnets { + subnetUptimes := make([]*p2p.SubnetUptime, 0, p.MySubnets.Len()) + for subnetID := range p.MySubnets { + if !p.trackedSubnets.Contains(subnetID) { + continue + } + subnetUptime, err := p.UptimeCalculator.CalculateUptimePercent(p.id, subnetID) if err != nil { p.Log.Debug("failed to get peer uptime percentage", @@ -947,6 +955,17 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { } } + if numTrackedSubnets := len(msg.TrackedSubnets); numTrackedSubnets > maxNumTrackedSubnets { + p.Log.Debug("message with invalid field", + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.HandshakeOp), + zap.String("field", "TrackedSubnets"), + zap.Int("numTrackedSubnets", numTrackedSubnets), + ) + p.StartClose() + return + } + // handle subnet IDs for _, subnetIDBytes := range msg.TrackedSubnets { subnetID, err := ids.ToID(subnetIDBytes) @@ -958,10 +977,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { p.StartClose() return } - // add only if we also track this subnet - if p.MySubnets.Contains(subnetID) { - p.trackedSubnets.Add(subnetID) - } + p.trackedSubnets.Add(subnetID) } for _, acp := range msg.SupportedAcps { @@ -1101,7 +1117,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { p.gotHandshake.Set(true) - peerIPs := p.Network.Peers(p.id, knownPeers, salt) + peerIPs := p.Network.Peers(p.trackedSubnets, p.id, knownPeers, salt) // We bypass throttling here to ensure that the handshake message is // acknowledged correctly. @@ -1158,7 +1174,7 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { return } - peerIPs := p.Network.Peers(p.id, filter, salt) + peerIPs := p.Network.Peers(p.trackedSubnets, p.id, filter, salt) if len(peerIPs) == 0 { p.Log.Debug("skipping sending of empty peer list", zap.Stringer("nodeID", p.id), diff --git a/network/peer/test_network.go b/network/peer/test_network.go index 01a341ae9abc..796c4cfc5440 100644 --- a/network/peer/test_network.go +++ b/network/peer/test_network.go @@ -7,6 +7,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/bloom" "github.com/ava-labs/avalanchego/utils/ips" + "github.com/ava-labs/avalanchego/utils/set" ) var TestNetwork Network = testNetwork{} @@ -29,6 +30,6 @@ func (testNetwork) KnownPeers() ([]byte, []byte) { return bloom.EmptyFilter.Marshal(), nil } -func (testNetwork) Peers(ids.NodeID, *bloom.ReadFilter, []byte) []*ips.ClaimedIPPort { +func (testNetwork) Peers(set.Set[ids.ID], ids.NodeID, *bloom.ReadFilter, []byte) []*ips.ClaimedIPPort { return nil } From 496426d78adbf6f62d64ae2bce3fa25ccde3f2b7 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Wed, 24 Apr 2024 19:27:01 -0400 Subject: [PATCH 05/62] wip --- network/network.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/network/network.go b/network/network.go index 3d5b8225cba9..2551d692034a 100644 --- a/network/network.go +++ b/network/network.go @@ -235,8 +235,7 @@ func NewNetwork( return nil, fmt.Errorf("initializing network metrics failed with: %w", err) } - // TODO: Should pass in [config.TrackedSubnets] rather than [nil] - ipTracker, err := newIPTracker(nil, log, config.Namespace, metricsRegisterer) + ipTracker, err := newIPTracker(config.TrackedSubnets, log, config.Namespace, metricsRegisterer) if err != nil { return nil, fmt.Errorf("initializing ip tracker failed with: %w", err) } From a84939381ae0c5376744025edadaf874884af4e7 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Wed, 24 Apr 2024 19:33:08 -0400 Subject: [PATCH 06/62] nit --- network/peer/peer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/network/peer/peer.go b/network/peer/peer.go index c4059fff7f27..d46c2b09f02e 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -977,7 +977,9 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { p.StartClose() return } - p.trackedSubnets.Add(subnetID) + if subnetID != constants.PrimaryNetworkID { + p.trackedSubnets.Add(subnetID) + } } for _, acp := range msg.SupportedAcps { From b9a10cdab9afdb60309f48d60d1c390862711fab Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Fri, 26 Apr 2024 11:48:35 -0400 Subject: [PATCH 07/62] wip --- network/ip_tracker.go | 158 ++++++++++++++++++++++++++----------- network/ip_tracker_test.go | 20 +++-- 2 files changed, 126 insertions(+), 52 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index db38f232a3db..11a2b3fcfab3 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -65,10 +65,13 @@ func newIPTracker( Name: "gossipable_ips", Help: "Number of IPs this node is willing to gossip", }), - bloomMetrics: bloomMetrics, + bloomMetrics: bloomMetrics, + tracked: make(map[ids.NodeID]*trackedNode), + bloomAdditions: make(map[ids.NodeID]int), + connected: make(map[ids.NodeID]*connectedNode), + subnet: make(map[ids.ID]*gossipableSubnet), + mostRecentTrackedIPs: make(map[ids.NodeID]*ips.ClaimedIPPort), - bloomAdditions: make(map[ids.NodeID]int), - connected: make(map[ids.NodeID]*ips.ClaimedIPPort), gossipableIndices: make(map[ids.NodeID]int), } err = utils.Err( @@ -81,6 +84,41 @@ func newIPTracker( return tracker, tracker.resetBloom() } +// A node is tracked if any of the following conditions are met: +// - The node was manually tracked +// - The node is a validator on any subnet +type trackedNode struct { + // manuallyTracked tracks if this node's connection was manually requested. + manuallyTracked bool + // subnets contains all the subnets that this node is a validator of. + subnets set.Set[ids.ID] + // ip is the most recently known IP of this node. + ip *ips.ClaimedIPPort +} + +type connectedNode struct { + // trackedSubnets contains all the subnets that this node is syncing. + trackedSubnets set.Set[ids.ID] + // ip this node claimed when connecting to use. The IP is not necessarily + // the same IP as in mostRecentTrackedIPs. + ip *ips.ClaimedIPPort +} + +type gossipableSubnet struct { + // manuallyGossipable contains the nodeIDs of all nodes whose IP was + // manually configured to be gossiped for this subnet. + manuallyGossipable set.Set[ids.NodeID] + + // An IP is marked as gossipable if all of the following conditions are met: + // - The node is a validator or was manually requested to be gossiped + // - The node is connected + // - The node reported that they are syncing this subnet + // - The IP the node connected with is its latest IP + gossipableIndices map[ids.NodeID]int + gossipableIPs []*ips.ClaimedIPPort + gossipableIDs set.Set[ids.NodeID] +} + type ipTracker struct { trackedSubnets set.Set[ids.ID] log logging.Logger @@ -88,14 +126,31 @@ type ipTracker struct { numGossipableIPs prometheus.Gauge bloomMetrics *bloom.Metrics - lock sync.RWMutex + lock sync.RWMutex + tracked map[ids.NodeID]*trackedNode + + // The bloom filter contains the most recent tracked IPs to avoid + // unnecessary IP gossip. + bloom *bloom.Filter + // To prevent validators from causing the bloom filter to have too many + // false positives, we limit each validator to maxIPEntriesPerValidator in + // the bloom filter. + bloomAdditions map[ids.NodeID]int // Number of IPs added to the bloom + bloomSalt []byte + maxBloomCount int + + // Connected tracks the information of currently connected peers, including + // tracked and untracked nodes. + connected map[ids.NodeID]*connectedNode + subnet map[ids.ID]*gossipableSubnet + + // TODO: Remove everything below this line + // manuallyTracked contains the nodeIDs of all nodes whose connection was // manually requested. manuallyTracked set.Set[ids.NodeID] - // manuallyGossipable contains the nodeIDs of all nodes whose IP was - // manually configured to be gossiped. - manuallyGossipable set.Set[ids.NodeID] + // TODO: Should include what subnets are causing the IPs to be tracked // mostRecentTrackedIPs tracks the most recent IP of each node whose // connection is desired. // @@ -107,21 +162,12 @@ type ipTracker struct { // trackedIDs contains the nodeIDs of all nodes whose connection is desired. trackedIDs set.Set[ids.NodeID] - // The bloom filter contains the most recent tracked IPs to avoid - // unnecessary IP gossip. - bloom *bloom.Filter - // To prevent validators from causing the bloom filter to have too many - // false positives, we limit each validator to maxIPEntriesPerValidator in - // the bloom filter. - bloomAdditions map[ids.NodeID]int // Number of IPs added to the bloom - bloomSalt []byte - maxBloomCount int - - // Connected tracks the IP of currently connected peers, including tracked - // and untracked nodes. The IP is not necessarily the same IP as in - // mostRecentTrackedIPs. - connected map[ids.NodeID]*ips.ClaimedIPPort + // TODO: Should be by subnet + // manuallyGossipable contains the nodeIDs of all nodes whose IP was + // manually configured to be gossiped. + manuallyGossipable set.Set[ids.NodeID] + // TODO: Should be by subnet // An IP is marked as gossipable if all of the following conditions are met: // - The node is a validator or was manually requested to be gossiped // - The node is connected @@ -143,8 +189,7 @@ func (i *ipTracker) ManuallyTrack(nodeID ids.NodeID) { i.lock.Lock() defer i.lock.Unlock() - i.addTrackableID(nodeID) - i.manuallyTracked.Add(nodeID) + i.addTrackableID(nodeID, nil) } // ManuallyGossip marks the provided nodeID as being desirable to connect to and @@ -160,8 +205,9 @@ func (i *ipTracker) ManuallyGossip(subnetID ids.ID, nodeID ids.NodeID) { i.lock.Lock() defer i.lock.Unlock() - i.addTrackableID(nodeID) - i.manuallyTracked.Add(nodeID) + if subnetID == constants.PrimaryNetworkID || i.trackedSubnets.Contains(subnetID) { + i.addTrackableID(nodeID, nil) + } i.addGossipableID(nodeID) i.manuallyGossipable.Add(nodeID) @@ -186,13 +232,13 @@ func (i *ipTracker) ShouldVerifyIP(ip *ips.ClaimedIPPort) bool { i.lock.RLock() defer i.lock.RUnlock() - if !i.trackedIDs.Contains(ip.NodeID) { + node, ok := i.tracked[ip.NodeID] + if !ok { return false } - prevIP, ok := i.mostRecentTrackedIPs[ip.NodeID] - return !ok || // This would be the first IP - prevIP.Timestamp < ip.Timestamp // This would be a newer IP + return node.ip == nil || // This would be the first IP + node.ip.Timestamp < ip.Timestamp // This would be a newer IP } // AddIP attempts to update the node's IP to the provided IP. This function @@ -216,31 +262,35 @@ func (i *ipTracker) GetIP(nodeID ids.NodeID) (*ips.ClaimedIPPort, bool) { i.lock.RLock() defer i.lock.RUnlock() - ip, ok := i.mostRecentTrackedIPs[nodeID] - return ip, ok + node, ok := i.tracked[nodeID] + if !ok { + return nil, false + } + return node.ip, node.ip != nil } // Connected is called when a connection is established. The peer should have // provided [ip] during the handshake. func (i *ipTracker) Connected(ip *ips.ClaimedIPPort, trackedSubnets set.Set[ids.ID]) { - _ = trackedSubnets - i.lock.Lock() defer i.lock.Unlock() - i.connected[ip.NodeID] = ip + i.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: trackedSubnets, + ip: ip, + } if i.addIP(ip) >= sameTimestamp && i.gossipableIDs.Contains(ip.NodeID) { i.addGossipableIP(ip) } } func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) int { - if !i.trackedIDs.Contains(ip.NodeID) { + node, ok := i.tracked[ip.NodeID] + if !ok { return untrackedTimestamp } - prevIP, ok := i.mostRecentTrackedIPs[ip.NodeID] - if !ok { + if node.ip == nil { // This is the first IP we've heard from the validator, so it is the // most recent. i.updateMostRecentTrackedIP(ip) @@ -249,10 +299,10 @@ func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) int { return newTimestamp } - if prevIP.Timestamp > ip.Timestamp { + if node.ip.Timestamp > ip.Timestamp { return olderTimestamp // This IP is old than the previously known IP. } - if prevIP.Timestamp == ip.Timestamp { + if node.ip.Timestamp == ip.Timestamp { return sameTimestamp // This IP is equal to the previously known IP. } @@ -278,24 +328,37 @@ func (i *ipTracker) OnValidatorAdded(subnetID ids.ID, nodeID ids.NodeID, _ *bls. i.lock.Lock() defer i.lock.Unlock() - i.addTrackableID(nodeID) + i.addTrackableID(nodeID, &subnetID) i.addGossipableID(nodeID) } -func (i *ipTracker) addTrackableID(nodeID ids.NodeID) { - if i.trackedIDs.Contains(nodeID) { +func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { + nodeTracker, previouslyTracked := i.tracked[nodeID] + if !previouslyTracked { + nodeTracker = &trackedNode{} + } + + nodeTracker.manuallyTracked = nodeTracker.manuallyTracked || subnetID == nil + if subnetID == nil { + nodeTracker.manuallyTracked = true + i.manuallyTracked.Add(nodeID) + } else { + nodeTracker.subnets.Add(*subnetID) + } + + if previouslyTracked { return } i.trackedIDs.Add(nodeID) - ip, connected := i.connected[nodeID] + node, connected := i.connected[nodeID] if !connected { return } // Because we previously weren't tracking this nodeID, the IP from the // connection is guaranteed to be the most up-to-date IP that we know. - i.updateMostRecentTrackedIP(ip) + i.updateMostRecentTrackedIP(node.ip) } func (i *ipTracker) addGossipableID(nodeID ids.NodeID) { @@ -304,16 +367,16 @@ func (i *ipTracker) addGossipableID(nodeID ids.NodeID) { } i.gossipableIDs.Add(nodeID) - connectedIP, connected := i.connected[nodeID] + node, connected := i.connected[nodeID] if !connected { return } - if updatedIP, ok := i.mostRecentTrackedIPs[nodeID]; !ok || connectedIP.Timestamp != updatedIP.Timestamp { + if updatedIP, ok := i.mostRecentTrackedIPs[nodeID]; !ok || node.ip.Timestamp != updatedIP.Timestamp { return } - i.addGossipableIP(connectedIP) + i.addGossipableIP(node.ip) } func (*ipTracker) OnValidatorWeightChanged(ids.ID, ids.NodeID, uint64, uint64) {} @@ -343,6 +406,7 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin } func (i *ipTracker) updateMostRecentTrackedIP(ip *ips.ClaimedIPPort) { + i.tracked[ip.NodeID].ip = ip i.mostRecentTrackedIPs[ip.NodeID] = ip i.numTrackedIPs.Set(float64(len(i.mostRecentTrackedIPs))) diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index f8db59d29d05..4c8f621667a2 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -352,7 +352,9 @@ func TestIPTracker_Connected(t *testing.T) { ip: ip, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.connected[ip.NodeID] = ip + tracker.connected[ip.NodeID] = &connectedNode{ + ip: ip, + } return tracker }(), }, @@ -369,7 +371,9 @@ func TestIPTracker_Connected(t *testing.T) { tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.mostRecentTrackedIPs[ip.NodeID] = ip tracker.bloomAdditions[ip.NodeID] = 1 - tracker.connected[ip.NodeID] = ip + tracker.connected[ip.NodeID] = &connectedNode{ + ip: ip, + } tracker.gossipableIndices[ip.NodeID] = 0 tracker.gossipableIPs = []*ips.ClaimedIPPort{ ip, @@ -390,7 +394,9 @@ func TestIPTracker_Connected(t *testing.T) { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) - tracker.connected[ip.NodeID] = ip + tracker.connected[ip.NodeID] = &connectedNode{ + ip: ip, + } return tracker }(), }, @@ -409,7 +415,9 @@ func TestIPTracker_Connected(t *testing.T) { require.True(t, tracker.AddIP(ip)) tracker.mostRecentTrackedIPs[newerIP.NodeID] = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 - tracker.connected[newerIP.NodeID] = newerIP + tracker.connected[newerIP.NodeID] = &connectedNode{ + ip: newerIP, + } tracker.gossipableIndices[newerIP.NodeID] = 0 tracker.gossipableIPs = []*ips.ClaimedIPPort{ newerIP, @@ -430,7 +438,9 @@ func TestIPTracker_Connected(t *testing.T) { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) - tracker.connected[ip.NodeID] = ip + tracker.connected[ip.NodeID] = &connectedNode{ + ip: ip, + } tracker.gossipableIndices[ip.NodeID] = 0 tracker.gossipableIPs = []*ips.ClaimedIPPort{ ip, From 0bb9d82233ab94576cca8bfab1fbc315d42b4e0b Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 29 Apr 2024 15:10:48 -0400 Subject: [PATCH 08/62] Remove subnet filter from Peer.TrackedSubnets() --- network/metrics.go | 24 +++++++++++++----------- network/network.go | 8 ++++++-- network/peer/peer.go | 42 ++++++++++++++++++++++++++++++------------ 3 files changed, 49 insertions(+), 25 deletions(-) diff --git a/network/metrics.go b/network/metrics.go index e2a3a363b403..ac1df146ccd7 100644 --- a/network/metrics.go +++ b/network/metrics.go @@ -12,11 +12,12 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/network/peer" "github.com/ava-labs/avalanchego/utils" - "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/set" ) type metrics struct { + trackedSubnets set.Set[ids.ID] + numTracked prometheus.Gauge numPeers prometheus.Gauge numSubnetPeers *prometheus.GaugeVec @@ -41,8 +42,9 @@ type metrics struct { peerConnectedStartTimesSum float64 } -func newMetrics(namespace string, registerer prometheus.Registerer, initialSubnetIDs set.Set[ids.ID]) (*metrics, error) { +func newMetrics(namespace string, registerer prometheus.Registerer, trackedSubnets set.Set[ids.ID]) (*metrics, error) { m := &metrics{ + trackedSubnets: trackedSubnets, numPeers: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Name: "peers", @@ -169,11 +171,7 @@ func newMetrics(namespace string, registerer prometheus.Registerer, initialSubne ) // init subnet tracker metrics with tracked subnets - for subnetID := range initialSubnetIDs { - // no need to track primary network ID - if subnetID == constants.PrimaryNetworkID { - continue - } + for subnetID := range trackedSubnets { // initialize to 0 subnetIDStr := subnetID.String() m.numSubnetPeers.WithLabelValues(subnetIDStr).Set(0) @@ -189,8 +187,10 @@ func (m *metrics) markConnected(peer peer.Peer) { m.connected.Inc() trackedSubnets := peer.TrackedSubnets() - for subnetID := range trackedSubnets { - m.numSubnetPeers.WithLabelValues(subnetID.String()).Inc() + for subnetID := range m.trackedSubnets { + if trackedSubnets.Contains(subnetID) { + m.numSubnetPeers.WithLabelValues(subnetID.String()).Inc() + } } m.lock.Lock() @@ -206,8 +206,10 @@ func (m *metrics) markDisconnected(peer peer.Peer) { m.disconnected.Inc() trackedSubnets := peer.TrackedSubnets() - for subnetID := range trackedSubnets { - m.numSubnetPeers.WithLabelValues(subnetID.String()).Dec() + for subnetID := range m.trackedSubnets { + if trackedSubnets.Contains(subnetID) { + m.numSubnetPeers.WithLabelValues(subnetID.String()).Dec() + } } m.lock.Lock() diff --git a/network/network.go b/network/network.go index 506e51908586..1f0692b29e8b 100644 --- a/network/network.go +++ b/network/network.go @@ -460,8 +460,12 @@ func (n *network) Connected(nodeID ids.NodeID) { peerVersion := peer.Version() n.router.Connected(nodeID, peerVersion, constants.PrimaryNetworkID) - for subnetID := range peer.TrackedSubnets() { - n.router.Connected(nodeID, peerVersion, subnetID) + + trackedSubnets := peer.TrackedSubnets() + for subnetID := range n.peerConfig.MySubnets { + if trackedSubnets.Contains(subnetID) { + n.router.Connected(nodeID, peerVersion, subnetID) + } } } diff --git a/network/peer/peer.go b/network/peer/peer.go index 1774d3f24f97..818a5d389a38 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -31,9 +31,14 @@ import ( "github.com/ava-labs/avalanchego/version" ) -// maxBloomSaltLen restricts the allowed size of the bloom salt to prevent -// excessively expensive bloom filter contains checks. -const maxBloomSaltLen = 32 +const ( + // maxBloomSaltLen restricts the allowed size of the bloom salt to prevent + // excessively expensive bloom filter contains checks. + maxBloomSaltLen = 32 + // maxNumTrackedSubnets limits how many subnets a peer can track to prevent + // excessive memory usage. + maxNumTrackedSubnets = 16 +) var ( errClosed = errors.New("closed") @@ -131,8 +136,8 @@ type peer struct { // version is the claimed version the peer is running that we received in // the Handshake message. version *version.Application - // trackedSubnets is the subset of subnetIDs the peer sent us in the Handshake - // message that we are also tracking. + // trackedSubnets are the subnetIDs the peer sent us in the Handshake + // message. trackedSubnets set.Set[ids.ID] // options of ACPs provided in the Handshake message. supportedACPs set.Set[uint32] @@ -263,9 +268,8 @@ func (p *peer) Info() Info { publicIPStr = p.ip.IPPort.String() } - uptimes := make(map[ids.ID]json.Uint32, p.trackedSubnets.Len()) - - for subnetID := range p.trackedSubnets { + uptimes := make(map[ids.ID]json.Uint32, p.MySubnets.Len()) + for subnetID := range p.MySubnets { uptime, exist := p.ObservedUptime(subnetID) if !exist { continue @@ -802,8 +806,12 @@ func (p *peer) getUptimes() (uint32, []*p2p.SubnetUptime) { primaryUptime = 0 } - subnetUptimes := make([]*p2p.SubnetUptime, 0, p.trackedSubnets.Len()) - for subnetID := range p.trackedSubnets { + subnetUptimes := make([]*p2p.SubnetUptime, 0, p.MySubnets.Len()) + for subnetID := range p.MySubnets { + if !p.trackedSubnets.Contains(subnetID) { + continue + } + subnetUptime, err := p.UptimeCalculator.CalculateUptimePercent(p.id, subnetID) if err != nil { p.Log.Debug("failed to get peer uptime percentage", @@ -948,6 +956,17 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { } // handle subnet IDs + if numTrackedSubnets := len(msg.TrackedSubnets); numTrackedSubnets > maxNumTrackedSubnets { + p.Log.Debug("message with invalid field", + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.HandshakeOp), + zap.String("field", "TrackedSubnets"), + zap.Int("numTrackedSubnets", numTrackedSubnets), + ) + p.StartClose() + return + } + for _, subnetIDBytes := range msg.TrackedSubnets { subnetID, err := ids.ToID(subnetIDBytes) if err != nil { @@ -958,8 +977,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { p.StartClose() return } - // add only if we also track this subnet - if p.MySubnets.Contains(subnetID) { + if subnetID != constants.PrimaryNetworkID { p.trackedSubnets.Add(subnetID) } } From 7c3b2284b7dba6d63b7444fe041e92f8efae1318 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 29 Apr 2024 16:49:45 -0400 Subject: [PATCH 09/62] refactor test peer creation --- network/peer/peer_test.go | 253 +++++++++++++------------------------- 1 file changed, 86 insertions(+), 167 deletions(-) diff --git a/network/peer/peer_test.go b/network/peer/peer_test.go index 96d6ddb7b98b..17b13c32cfca 100644 --- a/network/peer/peer_test.go +++ b/network/peer/peer_test.go @@ -39,7 +39,6 @@ type testPeer struct { type rawTestPeer struct { config *Config - conn net.Conn cert *staking.Certificate nodeID ids.NodeID inboundMsgChan <-chan message.InboundMessage @@ -60,27 +59,10 @@ func newMessageCreator(t *testing.T) message.Creator { return mc } -func makeRawTestPeers(t *testing.T, trackedSubnets set.Set[ids.ID]) (*rawTestPeer, *rawTestPeer) { +func newConfig(t *testing.T) Config { t.Helper() require := require.New(t) - conn0, conn1 := net.Pipe() - - tlsCert0, err := staking.NewTLSCert() - require.NoError(err) - cert0, err := staking.ParseCertificate(tlsCert0.Leaf.Raw) - require.NoError(err) - - tlsCert1, err := staking.NewTLSCert() - require.NoError(err) - cert1, err := staking.ParseCertificate(tlsCert1.Leaf.Raw) - require.NoError(err) - - nodeID0 := ids.NodeIDFromCert(cert0) - nodeID1 := ids.NodeIDFromCert(cert1) - - mc := newMessageCreator(t) - metrics, err := NewMetrics( logging.NoLog{}, "", @@ -96,14 +78,17 @@ func makeRawTestPeers(t *testing.T, trackedSubnets set.Set[ids.ID]) (*rawTestPee ) require.NoError(err) - sharedConfig := Config{ + return Config{ + ReadBufferSize: constants.DefaultNetworkPeerReadBufferSize, + WriteBufferSize: constants.DefaultNetworkPeerWriteBufferSize, Metrics: metrics, - MessageCreator: mc, + MessageCreator: newMessageCreator(t), Log: logging.NoLog{}, InboundMsgThrottler: throttling.NewNoInboundThrottler(), + Network: TestNetwork, + Router: nil, VersionCompatibility: version.GetCompatibility(constants.LocalID), - MySubnets: trackedSubnets, - UptimeCalculator: uptime.NoOpCalculator, + MySubnets: nil, Beacons: validators.NewManager(), Validators: validators.NewManager(), NetworkID: constants.LocalID, @@ -111,141 +96,91 @@ func makeRawTestPeers(t *testing.T, trackedSubnets set.Set[ids.ID]) (*rawTestPee PongTimeout: constants.DefaultPingPongTimeout, MaxClockDifference: time.Minute, ResourceTracker: resourceTracker, + UptimeCalculator: uptime.NoOpCalculator, + IPSigner: nil, } - peerConfig0 := sharedConfig - peerConfig1 := sharedConfig - - ip0 := ips.NewDynamicIPPort(net.IPv6loopback, 1) - tls0 := tlsCert0.PrivateKey.(crypto.Signer) - bls0, err := bls.NewSecretKey() - require.NoError(err) +} - peerConfig0.IPSigner = NewIPSigner(ip0, tls0, bls0) +func newRawTestPeer(t *testing.T, config Config) *rawTestPeer { + t.Helper() + require := require.New(t) - peerConfig0.Network = TestNetwork - inboundMsgChan0 := make(chan message.InboundMessage) - peerConfig0.Router = router.InboundHandlerFunc(func(_ context.Context, msg message.InboundMessage) { - inboundMsgChan0 <- msg - }) + tlsCert, err := staking.NewTLSCert() + require.NoError(err) + cert, err := staking.ParseCertificate(tlsCert.Leaf.Raw) + require.NoError(err) + nodeID := ids.NodeIDFromCert(cert) - ip1 := ips.NewDynamicIPPort(net.IPv6loopback, 2) - tls1 := tlsCert1.PrivateKey.(crypto.Signer) - bls1, err := bls.NewSecretKey() + ip := ips.NewDynamicIPPort(net.IPv6loopback, 1) + tls := tlsCert.PrivateKey.(crypto.Signer) + bls, err := bls.NewSecretKey() require.NoError(err) - peerConfig1.IPSigner = NewIPSigner(ip1, tls1, bls1) + config.IPSigner = NewIPSigner(ip, tls, bls) - peerConfig1.Network = TestNetwork - inboundMsgChan1 := make(chan message.InboundMessage) - peerConfig1.Router = router.InboundHandlerFunc(func(_ context.Context, msg message.InboundMessage) { - inboundMsgChan1 <- msg + inboundMsgChan := make(chan message.InboundMessage) + config.Router = router.InboundHandlerFunc(func(_ context.Context, msg message.InboundMessage) { + inboundMsgChan <- msg }) - peer0 := &rawTestPeer{ - config: &peerConfig0, - conn: conn0, - cert: cert0, - nodeID: nodeID0, - inboundMsgChan: inboundMsgChan0, + return &rawTestPeer{ + config: &config, + cert: cert, + nodeID: nodeID, + inboundMsgChan: inboundMsgChan, } - peer1 := &rawTestPeer{ - config: &peerConfig1, - conn: conn1, - cert: cert1, - nodeID: nodeID1, - inboundMsgChan: inboundMsgChan1, - } - return peer0, peer1 } -func makeTestPeers(t *testing.T, trackedSubnets set.Set[ids.ID]) (*testPeer, *testPeer) { - rawPeer0, rawPeer1 := makeRawTestPeers(t, trackedSubnets) - - peer0 := &testPeer{ - Peer: Start( - rawPeer0.config, - rawPeer0.conn, - rawPeer1.cert, - rawPeer1.nodeID, - NewThrottledMessageQueue( - rawPeer0.config.Metrics, - rawPeer1.nodeID, - logging.NoLog{}, - throttling.NewNoOutboundThrottler(), - ), - ), - inboundMsgChan: rawPeer0.inboundMsgChan, - } - peer1 := &testPeer{ +func startTestPeer(self *rawTestPeer, peer *rawTestPeer, conn net.Conn) *testPeer { + return &testPeer{ Peer: Start( - rawPeer1.config, - rawPeer1.conn, - rawPeer0.cert, - rawPeer0.nodeID, + self.config, + conn, + peer.cert, + peer.nodeID, NewThrottledMessageQueue( - rawPeer1.config.Metrics, - rawPeer0.nodeID, + self.config.Metrics, + peer.nodeID, logging.NoLog{}, throttling.NewNoOutboundThrottler(), ), ), - inboundMsgChan: rawPeer1.inboundMsgChan, + inboundMsgChan: self.inboundMsgChan, } +} + +func startTestPeers(rawPeer0 *rawTestPeer, rawPeer1 *rawTestPeer) (*testPeer, *testPeer) { + conn0, conn1 := net.Pipe() + peer0 := startTestPeer(rawPeer0, rawPeer1, conn0) + peer1 := startTestPeer(rawPeer1, rawPeer0, conn1) return peer0, peer1 } -func makeReadyTestPeers(t *testing.T, trackedSubnets set.Set[ids.ID]) (*testPeer, *testPeer) { +func awaitReady(t *testing.T, peers ...Peer) { t.Helper() require := require.New(t) - peer0, peer1 := makeTestPeers(t, trackedSubnets) - - require.NoError(peer0.AwaitReady(context.Background())) - require.True(peer0.Ready()) - - require.NoError(peer1.AwaitReady(context.Background())) - require.True(peer1.Ready()) - - return peer0, peer1 + for _, peer := range peers { + require.NoError(peer.AwaitReady(context.Background())) + require.True(peer.Ready()) + } } func TestReady(t *testing.T) { require := require.New(t) - rawPeer0, rawPeer1 := makeRawTestPeers(t, set.Set[ids.ID]{}) - peer0 := Start( - rawPeer0.config, - rawPeer0.conn, - rawPeer1.cert, - rawPeer1.nodeID, - NewThrottledMessageQueue( - rawPeer0.config.Metrics, - rawPeer1.nodeID, - logging.NoLog{}, - throttling.NewNoOutboundThrottler(), - ), - ) + config := newConfig(t) - require.False(peer0.Ready()) + rawPeer0 := newRawTestPeer(t, config) + rawPeer1 := newRawTestPeer(t, config) - peer1 := Start( - rawPeer1.config, - rawPeer1.conn, - rawPeer0.cert, - rawPeer0.nodeID, - NewThrottledMessageQueue( - rawPeer1.config.Metrics, - rawPeer0.nodeID, - logging.NoLog{}, - throttling.NewNoOutboundThrottler(), - ), - ) + conn0, conn1 := net.Pipe() - require.NoError(peer0.AwaitReady(context.Background())) - require.True(peer0.Ready()) + peer0 := startTestPeer(rawPeer0, rawPeer1, conn0) + require.False(peer0.Ready()) - require.NoError(peer1.AwaitReady(context.Background())) - require.True(peer1.Ready()) + peer1 := startTestPeer(rawPeer1, rawPeer0, conn1) + awaitReady(t, peer0, peer1) peer0.StartClose() require.NoError(peer0.AwaitClosed(context.Background())) @@ -255,10 +190,15 @@ func TestReady(t *testing.T) { func TestSend(t *testing.T) { require := require.New(t) - peer0, peer1 := makeReadyTestPeers(t, set.Set[ids.ID]{}) - mc := newMessageCreator(t) + sharedConfig := newConfig(t) - outboundGetMsg, err := mc.Get(ids.Empty, 1, time.Second, ids.Empty) + rawPeer0 := newRawTestPeer(t, sharedConfig) + rawPeer1 := newRawTestPeer(t, sharedConfig) + + peer0, peer1 := startTestPeers(rawPeer0, rawPeer1) + awaitReady(t, peer0, peer1) + + outboundGetMsg, err := sharedConfig.MessageCreator.Get(ids.Empty, 1, time.Second, ids.Empty) require.NoError(err) require.True(peer0.Send(context.Background(), outboundGetMsg)) @@ -275,9 +215,8 @@ func TestPingUptimes(t *testing.T) { trackedSubnetID := ids.GenerateTestID() untrackedSubnetID := ids.GenerateTestID() - trackedSubnets := set.Of(trackedSubnetID) - - mc := newMessageCreator(t) + sharedConfig := newConfig(t) + sharedConfig.MySubnets = set.Of(trackedSubnetID) testCases := []struct { name string @@ -288,7 +227,7 @@ func TestPingUptimes(t *testing.T) { { name: "primary network only", msg: func() message.OutboundMessage { - pingMsg, err := mc.Ping(1, nil) + pingMsg, err := sharedConfig.MessageCreator.Ping(1, nil) require.NoError(t, err) return pingMsg }(), @@ -305,7 +244,7 @@ func TestPingUptimes(t *testing.T) { { name: "primary network and subnet", msg: func() message.OutboundMessage { - pingMsg, err := mc.Ping( + pingMsg, err := sharedConfig.MessageCreator.Ping( 1, []*p2p.SubnetUptime{ { @@ -330,7 +269,7 @@ func TestPingUptimes(t *testing.T) { { name: "primary network and non tracked subnet", msg: func() message.OutboundMessage { - pingMsg, err := mc.Ping( + pingMsg, err := sharedConfig.MessageCreator.Ping( 1, []*p2p.SubnetUptime{ { @@ -352,9 +291,13 @@ func TestPingUptimes(t *testing.T) { }, } - // Note: we reuse peers across tests because makeReadyTestPeers takes awhile - // to run. - peer0, peer1 := makeReadyTestPeers(t, trackedSubnets) + // Note: we reuse peers across tests because newRawTestPeer takes awhile to + // run. + rawPeer0 := newRawTestPeer(t, sharedConfig) + rawPeer1 := newRawTestPeer(t, sharedConfig) + + peer0, peer1 := startTestPeers(rawPeer0, rawPeer1) + awaitReady(t, peer0, peer1) defer func() { peer1.StartClose() peer0.StartClose() @@ -390,7 +333,11 @@ func TestPingUptimes(t *testing.T) { func TestInvalidBLSKeyDisconnects(t *testing.T) { require := require.New(t) - rawPeer0, rawPeer1 := makeRawTestPeers(t, nil) + sharedConfig := newConfig(t) + + rawPeer0 := newRawTestPeer(t, sharedConfig) + rawPeer1 := newRawTestPeer(t, sharedConfig) + require.NoError(rawPeer0.config.Validators.AddStaker( constants.PrimaryNetworkID, rawPeer1.nodeID, @@ -408,36 +355,8 @@ func TestInvalidBLSKeyDisconnects(t *testing.T) { ids.GenerateTestID(), 1, )) - peer0 := &testPeer{ - Peer: Start( - rawPeer0.config, - rawPeer0.conn, - rawPeer1.cert, - rawPeer1.nodeID, - NewThrottledMessageQueue( - rawPeer0.config.Metrics, - rawPeer1.nodeID, - logging.NoLog{}, - throttling.NewNoOutboundThrottler(), - ), - ), - inboundMsgChan: rawPeer0.inboundMsgChan, - } - peer1 := &testPeer{ - Peer: Start( - rawPeer1.config, - rawPeer1.conn, - rawPeer0.cert, - rawPeer0.nodeID, - NewThrottledMessageQueue( - rawPeer1.config.Metrics, - rawPeer0.nodeID, - logging.NoLog{}, - throttling.NewNoOutboundThrottler(), - ), - ), - inboundMsgChan: rawPeer1.inboundMsgChan, - } + + peer0, peer1 := startTestPeers(rawPeer0, rawPeer1) // Because peer1 thinks that peer0 is using the wrong BLS key, they should // disconnect from each other. From 4a5cfaaa9671f42923e6f1f038e096bda735b9d2 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 29 Apr 2024 16:59:45 -0400 Subject: [PATCH 10/62] Add tracked subnets test --- network/peer/peer_test.go | 70 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/network/peer/peer_test.go b/network/peer/peer_test.go index 17b13c32cfca..cb09e1409f15 100644 --- a/network/peer/peer_test.go +++ b/network/peer/peer_test.go @@ -329,6 +329,76 @@ func TestPingUptimes(t *testing.T) { } } +func TestTrackedSubnets(t *testing.T) { + sharedConfig := newConfig(t) + rawPeer0 := newRawTestPeer(t, sharedConfig) + rawPeer1 := newRawTestPeer(t, sharedConfig) + + tests := []struct { + name string + trackedSubnets set.Set[ids.ID] + shouldDisconnect bool + }{ + { + name: "primary network only", + trackedSubnets: nil, + shouldDisconnect: false, + }, + { + name: "single subnet", + trackedSubnets: set.Of(ids.GenerateTestID()), + shouldDisconnect: false, + }, + { + name: "max subnets", + trackedSubnets: func() set.Set[ids.ID] { + trackedSubnets := set.NewSet[ids.ID](maxNumTrackedSubnets) + for i := 0; i < maxNumTrackedSubnets; i++ { + trackedSubnets.Add(ids.GenerateTestID()) + } + return trackedSubnets + }(), + shouldDisconnect: false, + }, + { + name: "too many subnets", + trackedSubnets: func() set.Set[ids.ID] { + trackedSubnets := set.NewSet[ids.ID](maxNumTrackedSubnets + 1) + for i := 0; i < maxNumTrackedSubnets+1; i++ { + trackedSubnets.Add(ids.GenerateTestID()) + } + return trackedSubnets + }(), + shouldDisconnect: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require := require.New(t) + + rawPeer0.config.MySubnets = test.trackedSubnets + peer0, peer1 := startTestPeers(rawPeer0, rawPeer1) + if test.shouldDisconnect { + require.NoError(peer0.AwaitClosed(context.Background())) + require.NoError(peer1.AwaitClosed(context.Background())) + return + } + + defer func() { + peer1.StartClose() + peer0.StartClose() + require.NoError(peer0.AwaitClosed(context.Background())) + require.NoError(peer1.AwaitClosed(context.Background())) + }() + + awaitReady(t, peer0, peer1) + require.Empty(peer0.TrackedSubnets()) + require.Equal(test.trackedSubnets, peer1.TrackedSubnets()) + }) + } +} + // Test that a peer using the wrong BLS key is disconnected from. func TestInvalidBLSKeyDisconnects(t *testing.T) { require := require.New(t) From 255a2c741586498023aa27d4a35099d12715154c Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 29 Apr 2024 17:51:18 -0400 Subject: [PATCH 11/62] Update p2p messages --- message/mock_outbound_message_builder.go | 16 +- message/outbound_msg_builder.go | 8 +- network/network.go | 1 + network/peer/config.go | 1 + network/peer/peer.go | 7 +- proto/p2p/p2p.proto | 2 + proto/pb/p2p/p2p.pb.go | 378 ++++++++++++----------- 7 files changed, 224 insertions(+), 189 deletions(-) diff --git a/message/mock_outbound_message_builder.go b/message/mock_outbound_message_builder.go index cff8ed554caf..646abe780329 100644 --- a/message/mock_outbound_message_builder.go +++ b/message/mock_outbound_message_builder.go @@ -253,18 +253,18 @@ func (mr *MockOutboundMsgBuilderMockRecorder) GetAncestors(arg0, arg1, arg2, arg } // GetPeerList mocks base method. -func (m *MockOutboundMsgBuilder) GetPeerList(arg0, arg1 []byte) (OutboundMessage, error) { +func (m *MockOutboundMsgBuilder) GetPeerList(arg0, arg1 []byte, arg2 bool) (OutboundMessage, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPeerList", arg0, arg1) + ret := m.ctrl.Call(m, "GetPeerList", arg0, arg1, arg2) ret0, _ := ret[0].(OutboundMessage) ret1, _ := ret[1].(error) return ret0, ret1 } // GetPeerList indicates an expected call of GetPeerList. -func (mr *MockOutboundMsgBuilderMockRecorder) GetPeerList(arg0, arg1 any) *gomock.Call { +func (mr *MockOutboundMsgBuilderMockRecorder) GetPeerList(arg0, arg1, arg2 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPeerList", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).GetPeerList), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPeerList", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).GetPeerList), arg0, arg1, arg2) } // GetStateSummaryFrontier mocks base method. @@ -283,18 +283,18 @@ func (mr *MockOutboundMsgBuilderMockRecorder) GetStateSummaryFrontier(arg0, arg1 } // Handshake mocks base method. -func (m *MockOutboundMsgBuilder) Handshake(arg0 uint32, arg1 uint64, arg2 ips.IPPort, arg3 string, arg4, arg5, arg6 uint32, arg7 uint64, arg8, arg9 []byte, arg10 []ids.ID, arg11, arg12 []uint32, arg13, arg14 []byte) (OutboundMessage, error) { +func (m *MockOutboundMsgBuilder) Handshake(arg0 uint32, arg1 uint64, arg2 ips.IPPort, arg3 string, arg4, arg5, arg6 uint32, arg7 uint64, arg8, arg9 []byte, arg10 []ids.ID, arg11, arg12 []uint32, arg13, arg14 []byte, arg15 bool) (OutboundMessage, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Handshake", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14) + ret := m.ctrl.Call(m, "Handshake", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14, arg15) ret0, _ := ret[0].(OutboundMessage) ret1, _ := ret[1].(error) return ret0, ret1 } // Handshake indicates an expected call of Handshake. -func (mr *MockOutboundMsgBuilderMockRecorder) Handshake(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14 any) *gomock.Call { +func (mr *MockOutboundMsgBuilderMockRecorder) Handshake(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14, arg15 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Handshake", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).Handshake), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Handshake", reflect.TypeOf((*MockOutboundMsgBuilder)(nil).Handshake), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11, arg12, arg13, arg14, arg15) } // PeerList mocks base method. diff --git a/message/outbound_msg_builder.go b/message/outbound_msg_builder.go index 1b02d8fa74f5..9a3ecaa2da1e 100644 --- a/message/outbound_msg_builder.go +++ b/message/outbound_msg_builder.go @@ -34,11 +34,13 @@ type OutboundMsgBuilder interface { objectedACPs []uint32, knownPeersFilter []byte, knownPeersSalt []byte, + requestAllSubnetIPs bool, ) (OutboundMessage, error) GetPeerList( knownPeersFilter []byte, knownPeersSalt []byte, + requestAllSubnetIPs bool, ) (OutboundMessage, error) PeerList( @@ -241,6 +243,7 @@ func (b *outMsgBuilder) Handshake( objectedACPs []uint32, knownPeersFilter []byte, knownPeersSalt []byte, + requestAllSubnetIPs bool, ) (OutboundMessage, error) { subnetIDBytes := make([][]byte, len(trackedSubnets)) encodeIDs(trackedSubnets, subnetIDBytes) @@ -267,7 +270,8 @@ func (b *outMsgBuilder) Handshake( Filter: knownPeersFilter, Salt: knownPeersSalt, }, - IpBlsSig: ipBLSSig, + IpBlsSig: ipBLSSig, + AllSubnets: requestAllSubnetIPs, }, }, }, @@ -279,6 +283,7 @@ func (b *outMsgBuilder) Handshake( func (b *outMsgBuilder) GetPeerList( knownPeersFilter []byte, knownPeersSalt []byte, + requestAllSubnetIPs bool, ) (OutboundMessage, error) { return b.builder.createOutbound( &p2p.Message{ @@ -288,6 +293,7 @@ func (b *outMsgBuilder) GetPeerList( Filter: knownPeersFilter, Salt: knownPeersSalt, }, + AllSubnets: requestAllSubnetIPs, }, }, }, diff --git a/network/network.go b/network/network.go index b876ce7bd63f..0fb1d5fab36d 100644 --- a/network/network.go +++ b/network/network.go @@ -263,6 +263,7 @@ func NewNetwork( Network: nil, // This is set below. Router: router, VersionCompatibility: version.GetCompatibility(config.NetworkID), + MyNodeID: config.MyNodeID, MySubnets: config.TrackedSubnets, Beacons: config.Beacons, Validators: config.Validators, diff --git a/network/peer/config.go b/network/peer/config.go index 3eb8319216d7..9a86941f5038 100644 --- a/network/peer/config.go +++ b/network/peer/config.go @@ -33,6 +33,7 @@ type Config struct { Network Network Router router.InboundHandler VersionCompatibility version.Compatibility + MyNodeID ids.NodeID MySubnets set.Set[ids.ID] Beacons validators.Manager Validators validators.Manager diff --git a/network/peer/peer.go b/network/peer/peer.go index 53487232e9f5..1150255009a8 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -545,6 +545,7 @@ func (p *peer) writeMessages() { p.ObjectedACPs, knownPeersFilter, knownPeersSalt, + p.Validators.GetWeight(constants.PrimaryNetworkID, p.MyNodeID) != 0, ) if err != nil { p.Log.Error("failed to create message", @@ -637,7 +638,11 @@ func (p *peer) sendNetworkMessages() { select { case <-p.getPeerListChan: knownPeersFilter, knownPeersSalt := p.Config.Network.KnownPeers() - msg, err := p.Config.MessageCreator.GetPeerList(knownPeersFilter, knownPeersSalt) + msg, err := p.Config.MessageCreator.GetPeerList( + knownPeersFilter, + knownPeersSalt, + p.Validators.GetWeight(constants.PrimaryNetworkID, p.MyNodeID) != 0, + ) if err != nil { p.Log.Error("failed to create get peer list message", zap.Stringer("nodeID", p.id), diff --git a/proto/p2p/p2p.proto b/proto/p2p/p2p.proto index 53c0c84de303..6e43e8da2aba 100644 --- a/proto/p2p/p2p.proto +++ b/proto/p2p/p2p.proto @@ -112,6 +112,7 @@ message Handshake { // Signature of the peer IP port pair at a provided timestamp with the BLS // key. bytes ip_bls_sig = 13; + bool all_subnets = 14; } // Metadata about a peer's P2P client used to determine compatibility @@ -154,6 +155,7 @@ message ClaimedIpPort { // filter. message GetPeerList { BloomFilter known_peers = 1; + bool all_subnets = 2; } // PeerList contains network-level metadata for a set of validators. diff --git a/proto/pb/p2p/p2p.pb.go b/proto/pb/p2p/p2p.pb.go index bd55ad703546..e2d97c73a82f 100644 --- a/proto/pb/p2p/p2p.pb.go +++ b/proto/pb/p2p/p2p.pb.go @@ -678,7 +678,8 @@ type Handshake struct { KnownPeers *BloomFilter `protobuf:"bytes,12,opt,name=known_peers,json=knownPeers,proto3" json:"known_peers,omitempty"` // Signature of the peer IP port pair at a provided timestamp with the BLS // key. - IpBlsSig []byte `protobuf:"bytes,13,opt,name=ip_bls_sig,json=ipBlsSig,proto3" json:"ip_bls_sig,omitempty"` + IpBlsSig []byte `protobuf:"bytes,13,opt,name=ip_bls_sig,json=ipBlsSig,proto3" json:"ip_bls_sig,omitempty"` + AllSubnets bool `protobuf:"varint,14,opt,name=all_subnets,json=allSubnets,proto3" json:"all_subnets,omitempty"` } func (x *Handshake) Reset() { @@ -797,6 +798,13 @@ func (x *Handshake) GetIpBlsSig() []byte { return nil } +func (x *Handshake) GetAllSubnets() bool { + if x != nil { + return x.AllSubnets + } + return false +} + // Metadata about a peer's P2P client used to determine compatibility type Client struct { state protoimpl.MessageState @@ -1033,6 +1041,7 @@ type GetPeerList struct { unknownFields protoimpl.UnknownFields KnownPeers *BloomFilter `protobuf:"bytes,1,opt,name=known_peers,json=knownPeers,proto3" json:"known_peers,omitempty"` + AllSubnets bool `protobuf:"varint,2,opt,name=all_subnets,json=allSubnets,proto3" json:"all_subnets,omitempty"` } func (x *GetPeerList) Reset() { @@ -1074,6 +1083,13 @@ func (x *GetPeerList) GetKnownPeers() *BloomFilter { return nil } +func (x *GetPeerList) GetAllSubnets() bool { + if x != nil { + return x.AllSubnets + } + return false +} + // PeerList contains network-level metadata for a set of validators. // // PeerList must be sent in response to an inbound Handshake message from a @@ -2640,7 +2656,7 @@ var file_p2p_p2p_proto_rawDesc = []byte{ 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x6e, - 0x67, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x22, 0xb3, 0x03, + 0x67, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x22, 0xd4, 0x03, 0x0a, 0x09, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x6d, 0x79, @@ -2667,202 +2683,206 @@ var file_p2p_p2p_proto_rawDesc = []byte{ 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x0a, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x1c, 0x0a, 0x0a, 0x69, 0x70, 0x5f, 0x62, 0x6c, 0x73, 0x5f, 0x73, 0x69, 0x67, 0x18, 0x0d, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x08, 0x69, 0x70, 0x42, 0x6c, 0x73, 0x53, 0x69, 0x67, 0x4a, 0x04, 0x08, - 0x05, 0x10, 0x06, 0x22, 0x5e, 0x0a, 0x06, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x12, 0x14, 0x0a, - 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x70, 0x61, - 0x74, 0x63, 0x68, 0x22, 0x39, 0x0a, 0x0b, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, - 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x61, - 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x73, 0x61, 0x6c, 0x74, 0x22, 0xbd, - 0x01, 0x0a, 0x0d, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, - 0x12, 0x29, 0x0a, 0x10, 0x78, 0x35, 0x30, 0x39, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x78, 0x35, 0x30, 0x39, - 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x69, - 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x69, 0x70, - 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x69, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1c, 0x0a, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1c, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x78, 0x5f, - 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x78, 0x49, 0x64, 0x22, 0x40, - 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x31, 0x0a, - 0x0b, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x52, 0x0a, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x50, 0x65, 0x65, 0x72, 0x73, - 0x22, 0x48, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x10, - 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x5f, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x43, 0x6c, 0x61, - 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x52, 0x0e, 0x63, 0x6c, 0x61, 0x69, - 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x22, 0x6f, 0x0a, 0x17, 0x47, 0x65, - 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, + 0x01, 0x28, 0x0c, 0x52, 0x08, 0x69, 0x70, 0x42, 0x6c, 0x73, 0x53, 0x69, 0x67, 0x12, 0x1f, 0x0a, + 0x0b, 0x61, 0x6c, 0x6c, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x0e, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x4a, 0x04, + 0x08, 0x05, 0x10, 0x06, 0x22, 0x5e, 0x0a, 0x06, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x69, 0x6e, 0x6f, + 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x12, 0x14, + 0x0a, 0x05, 0x70, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x70, + 0x61, 0x74, 0x63, 0x68, 0x22, 0x39, 0x0a, 0x0b, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x73, + 0x61, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x73, 0x61, 0x6c, 0x74, 0x22, + 0xbd, 0x01, 0x0a, 0x0d, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, + 0x74, 0x12, 0x29, 0x0a, 0x10, 0x78, 0x35, 0x30, 0x39, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, + 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x78, 0x35, 0x30, + 0x39, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x17, 0x0a, 0x07, + 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x69, + 0x70, 0x41, 0x64, 0x64, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x69, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1c, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1c, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x78, + 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x78, 0x49, 0x64, 0x22, + 0x61, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x31, + 0x0a, 0x0b, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x0a, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x50, 0x65, 0x65, 0x72, + 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x6c, 0x6c, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x53, 0x75, 0x62, 0x6e, 0x65, + 0x74, 0x73, 0x22, 0x48, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x3c, + 0x0a, 0x10, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x5f, 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, + 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x43, + 0x6c, 0x61, 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x52, 0x0e, 0x63, 0x6c, + 0x61, 0x69, 0x6d, 0x65, 0x64, 0x49, 0x70, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x22, 0x6f, 0x0a, 0x17, + 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, + 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, + 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x6a, 0x0a, + 0x14, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, + 0x18, 0x0a, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x22, 0x89, 0x01, 0x0a, 0x17, 0x47, 0x65, + 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, + 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, + 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x6a, 0x0a, 0x14, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x46, 0x72, 0x6f, 0x6e, 0x74, - 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, - 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x18, 0x0a, - 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, - 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x22, 0x89, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x41, - 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, - 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, - 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, - 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x69, - 0x67, 0x68, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x04, 0x52, 0x07, 0x68, 0x65, 0x69, 0x67, - 0x68, 0x74, 0x73, 0x22, 0x71, 0x0a, 0x14, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, - 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x75, 0x6d, 0x6d, - 0x61, 0x72, 0x79, 0x49, 0x64, 0x73, 0x22, 0x71, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, - 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, - 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, - 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, - 0x69, 0x6e, 0x65, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x6f, 0x0a, 0x10, 0x41, 0x63, 0x63, - 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, + 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, + 0x65, 0x69, 0x67, 0x68, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x04, 0x52, 0x07, 0x68, 0x65, + 0x69, 0x67, 0x68, 0x74, 0x73, 0x22, 0x71, 0x0a, 0x14, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, + 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0x8e, 0x01, 0x0a, 0x0b, 0x47, - 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, - 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, - 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x49, 0x64, 0x73, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0x69, 0x0a, 0x08, 0x41, - 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, - 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, - 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x49, 0x64, 0x73, 0x22, 0xb9, 0x01, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x41, 0x6e, - 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, - 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, - 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, - 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, 0x67, 0x69, - 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x22, 0x65, 0x0a, 0x09, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x75, 0x6d, 0x6d, 0x61, + 0x72, 0x79, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x75, + 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x49, 0x64, 0x73, 0x22, 0x71, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x41, + 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, - 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x22, 0x84, 0x01, 0x0a, 0x03, 0x47, 0x65, - 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, + 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, + 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x6f, 0x0a, 0x10, 0x41, + 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x69, 0x65, 0x72, 0x12, + 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0x8e, 0x01, 0x0a, + 0x0b, 0x47, 0x65, 0x74, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, + 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, + 0x6e, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x73, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0x69, 0x0a, + 0x08, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x73, 0x22, 0xb9, 0x01, 0x0a, 0x0c, 0x47, 0x65, 0x74, + 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, + 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x49, 0x64, 0x12, 0x30, 0x0a, 0x0b, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x70, 0x32, 0x70, 0x2e, 0x45, 0x6e, + 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x22, 0x65, 0x0a, 0x09, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x6f, 0x72, + 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, - 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, - 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, - 0x22, 0x5d, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, + 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x22, 0x84, 0x01, 0x0a, 0x03, + 0x47, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, + 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, + 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x4a, 0x04, 0x08, 0x05, + 0x10, 0x06, 0x22, 0x5d, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x22, 0xb0, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x73, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, + 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, + 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, + 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, + 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x4a, 0x04, + 0x08, 0x05, 0x10, 0x06, 0x22, 0xb5, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x6c, 0x6c, 0x51, 0x75, 0x65, + 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, + 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, + 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, + 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, + 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0xba, 0x01, 0x0a, + 0x05, 0x43, 0x68, 0x69, 0x74, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, + 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, + 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x5f, 0x69, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, + 0x64, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, + 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, + 0x65, 0x64, 0x49, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, + 0x64, 0x5f, 0x69, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x13, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x49, + 0x64, 0x41, 0x74, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x7f, 0x0a, 0x0a, 0x41, 0x70, 0x70, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, - 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x22, - 0xb0, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x73, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x19, 0x0a, + 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1b, 0x0a, + 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x0b, 0x41, 0x70, + 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, + 0x22, 0x88, 0x01, 0x0a, 0x08, 0x41, 0x70, 0x70, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, - 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, - 0x69, 0x6e, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, - 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x4a, 0x04, 0x08, 0x05, - 0x10, 0x06, 0x22, 0xb5, 0x01, 0x0a, 0x09, 0x50, 0x75, 0x6c, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, - 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, - 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x65, - 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x48, 0x65, - 0x69, 0x67, 0x68, 0x74, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0xba, 0x01, 0x0a, 0x05, 0x43, - 0x68, 0x69, 0x74, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, - 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x21, - 0x0a, 0x0c, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x49, - 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x64, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, - 0x49, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x5f, - 0x69, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x13, 0x70, 0x72, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0x49, 0x64, 0x41, - 0x74, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x7f, 0x0a, 0x0a, 0x41, 0x70, 0x70, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, - 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, - 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x08, 0x64, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x61, - 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, - 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x0b, 0x41, 0x70, 0x70, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, - 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x88, - 0x01, 0x0a, 0x08, 0x41, 0x70, 0x70, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, - 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x11, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x43, 0x6f, 0x64, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x43, 0x0a, 0x09, 0x41, 0x70, 0x70, - 0x47, 0x6f, 0x73, 0x73, 0x69, 0x70, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, - 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, 0x2a, 0x5d, - 0x0a, 0x0a, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x17, - 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, - 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x4e, 0x47, - 0x49, 0x4e, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x56, 0x41, 0x4c, 0x41, 0x4e, 0x43, - 0x48, 0x45, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4e, 0x4f, 0x57, 0x4d, 0x41, 0x4e, 0x10, 0x02, 0x42, 0x2e, 0x5a, - 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x76, 0x61, 0x2d, - 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x61, 0x76, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x67, 0x6f, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x2f, 0x70, 0x32, 0x70, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x11, 0x52, 0x09, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x43, 0x0a, 0x09, 0x41, + 0x70, 0x70, 0x47, 0x6f, 0x73, 0x73, 0x69, 0x70, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, + 0x6e, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x70, 0x70, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x61, 0x70, 0x70, 0x42, 0x79, 0x74, 0x65, 0x73, + 0x2a, 0x5d, 0x0a, 0x0a, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, + 0x0a, 0x17, 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x45, + 0x4e, 0x47, 0x49, 0x4e, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x56, 0x41, 0x4c, 0x41, + 0x4e, 0x43, 0x48, 0x45, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x4e, 0x47, 0x49, 0x4e, 0x45, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4e, 0x4f, 0x57, 0x4d, 0x41, 0x4e, 0x10, 0x02, 0x42, + 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x76, + 0x61, 0x2d, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x61, 0x76, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x68, 0x65, + 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x2f, 0x70, 0x32, 0x70, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( From 9a3ee34d3c5474f24118a418eaf5f3bb9c65a0d6 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 29 Apr 2024 18:12:25 -0400 Subject: [PATCH 12/62] use network bool --- network/ip_tracker.go | 12 +++++++----- network/network.go | 2 +- network/peer/network.go | 2 +- network/peer/peer.go | 12 ++++++++++-- network/peer/test_network.go | 2 +- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 11a2b3fcfab3..d47462da12e3 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -462,12 +462,14 @@ func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) { i.numGossipableIPs.Dec() } -// GetGossipableIPs returns the latest IPs of connected validators. The returned -// IPs will not contain [exceptNodeID] or any IPs contained in [exceptIPs]. If -// the number of eligible IPs to return low, it's possible that every IP will be -// iterated over while handling this call. +// GetGossipableIPs returns the latest IPs of connected validators. If +// [trackedSubnets] is non-nil, only IPs of validators in the provided subnets +// or the primary network are returned. The returned IPs will not contain +// [exceptNodeID] or any IPs contained in [exceptIPs]. If the number of eligible +// IPs to return low, it's possible that every IP will be iterated over while +// handling this call. func (i *ipTracker) GetGossipableIPs( - trackedSubnets set.Set[ids.ID], + trackedSubnets *set.Set[ids.ID], exceptNodeID ids.NodeID, exceptIPs *bloom.ReadFilter, salt []byte, diff --git a/network/network.go b/network/network.go index 0fb1d5fab36d..588667c5a7b8 100644 --- a/network/network.go +++ b/network/network.go @@ -514,7 +514,7 @@ func (n *network) KnownPeers() ([]byte, []byte) { } func (n *network) Peers( - trackedSubnets set.Set[ids.ID], + trackedSubnets *set.Set[ids.ID], except ids.NodeID, knownPeers *bloom.ReadFilter, salt []byte, diff --git a/network/peer/network.go b/network/peer/network.go index eb1f5fb7ec68..49500fea57bf 100644 --- a/network/peer/network.go +++ b/network/peer/network.go @@ -35,7 +35,7 @@ type Network interface { // Peers returns peers that are not known. Peers( - trackedSubnets set.Set[ids.ID], + trackedSubnets *set.Set[ids.ID], // If nil then all peers are returned peerID ids.NodeID, knownPeers *bloom.ReadFilter, peerSalt []byte, diff --git a/network/peer/peer.go b/network/peer/peer.go index 1150255009a8..167bce3cd4fb 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -1135,7 +1135,11 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { p.gotHandshake.Set(true) - peerIPs := p.Network.Peers(p.trackedSubnets, p.id, knownPeers, salt) + var trackedSubnets *set.Set[ids.ID] + if !msg.AllSubnets { + trackedSubnets = &p.trackedSubnets + } + peerIPs := p.Network.Peers(trackedSubnets, p.id, knownPeers, salt) // We bypass throttling here to ensure that the handshake message is // acknowledged correctly. @@ -1192,7 +1196,11 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { return } - peerIPs := p.Network.Peers(p.trackedSubnets, p.id, filter, salt) + var trackedSubnets *set.Set[ids.ID] + if !msg.AllSubnets { + trackedSubnets = &p.trackedSubnets + } + peerIPs := p.Network.Peers(trackedSubnets, p.id, filter, salt) if len(peerIPs) == 0 { p.Log.Debug("skipping sending of empty peer list", zap.Stringer("nodeID", p.id), diff --git a/network/peer/test_network.go b/network/peer/test_network.go index 796c4cfc5440..bd85ad049aec 100644 --- a/network/peer/test_network.go +++ b/network/peer/test_network.go @@ -30,6 +30,6 @@ func (testNetwork) KnownPeers() ([]byte, []byte) { return bloom.EmptyFilter.Marshal(), nil } -func (testNetwork) Peers(set.Set[ids.ID], ids.NodeID, *bloom.ReadFilter, []byte) []*ips.ClaimedIPPort { +func (testNetwork) Peers(*set.Set[ids.ID], ids.NodeID, *bloom.ReadFilter, []byte) []*ips.ClaimedIPPort { return nil } From 2ad3f0ee4f98289d40a3b91bf588cf5370a3ee19 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 29 Apr 2024 18:23:35 -0400 Subject: [PATCH 13/62] cleanup tests --- network/peer/peer_test.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/network/peer/peer_test.go b/network/peer/peer_test.go index cb09e1409f15..e30ffe95911a 100644 --- a/network/peer/peer_test.go +++ b/network/peer/peer_test.go @@ -40,7 +40,6 @@ type testPeer struct { type rawTestPeer struct { config *Config cert *staking.Certificate - nodeID ids.NodeID inboundMsgChan <-chan message.InboundMessage } @@ -109,7 +108,7 @@ func newRawTestPeer(t *testing.T, config Config) *rawTestPeer { require.NoError(err) cert, err := staking.ParseCertificate(tlsCert.Leaf.Raw) require.NoError(err) - nodeID := ids.NodeIDFromCert(cert) + config.MyNodeID = ids.NodeIDFromCert(cert) ip := ips.NewDynamicIPPort(net.IPv6loopback, 1) tls := tlsCert.PrivateKey.(crypto.Signer) @@ -126,7 +125,6 @@ func newRawTestPeer(t *testing.T, config Config) *rawTestPeer { return &rawTestPeer{ config: &config, cert: cert, - nodeID: nodeID, inboundMsgChan: inboundMsgChan, } } @@ -137,10 +135,10 @@ func startTestPeer(self *rawTestPeer, peer *rawTestPeer, conn net.Conn) *testPee self.config, conn, peer.cert, - peer.nodeID, + peer.config.MyNodeID, NewThrottledMessageQueue( self.config.Metrics, - peer.nodeID, + peer.config.MyNodeID, logging.NoLog{}, throttling.NewNoOutboundThrottler(), ), @@ -410,7 +408,7 @@ func TestInvalidBLSKeyDisconnects(t *testing.T) { require.NoError(rawPeer0.config.Validators.AddStaker( constants.PrimaryNetworkID, - rawPeer1.nodeID, + rawPeer1.config.MyNodeID, bls.PublicFromSecretKey(rawPeer1.config.IPSigner.blsSigner), ids.GenerateTestID(), 1, @@ -420,7 +418,7 @@ func TestInvalidBLSKeyDisconnects(t *testing.T) { require.NoError(err) require.NoError(rawPeer1.config.Validators.AddStaker( constants.PrimaryNetworkID, - rawPeer0.nodeID, + rawPeer0.config.MyNodeID, bls.PublicFromSecretKey(bogusBLSKey), // This is the wrong BLS key for this peer ids.GenerateTestID(), 1, From 75122586a9b57e617d0ce6f0a442662dbebc11f6 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 29 Apr 2024 18:28:05 -0400 Subject: [PATCH 14/62] fix merge --- network/peer/peer.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/network/peer/peer.go b/network/peer/peer.go index 167bce3cd4fb..38d4c0dd4635 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -960,17 +960,6 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { } } - if numTrackedSubnets := len(msg.TrackedSubnets); numTrackedSubnets > maxNumTrackedSubnets { - p.Log.Debug("message with invalid field", - zap.Stringer("nodeID", p.id), - zap.Stringer("messageOp", message.HandshakeOp), - zap.String("field", "TrackedSubnets"), - zap.Int("numTrackedSubnets", numTrackedSubnets), - ) - p.StartClose() - return - } - // handle subnet IDs if numTrackedSubnets := len(msg.TrackedSubnets); numTrackedSubnets > maxNumTrackedSubnets { p.Log.Debug("message with invalid field", From d6bdaa8fd654df45b5921fad4ed473509401201b Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 30 Apr 2024 10:55:55 -0400 Subject: [PATCH 15/62] cleanup --- network/ip_tracker.go | 51 +++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index d47462da12e3..c58c16950990 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -92,10 +92,16 @@ type trackedNode struct { manuallyTracked bool // subnets contains all the subnets that this node is a validator of. subnets set.Set[ids.ID] + // subnets contains the subset of [subnets] that the local node also tracks. + trackedSubnets set.Set[ids.ID] // ip is the most recently known IP of this node. ip *ips.ClaimedIPPort } +func (n *trackedNode) wantsConnection() bool { + return n.manuallyTracked || n.trackedSubnets.Len() > 0 +} + type connectedNode struct { // trackedSubnets contains all the subnets that this node is syncing. trackedSubnets set.Set[ids.ID] @@ -215,18 +221,19 @@ func (i *ipTracker) ManuallyGossip(subnetID ids.ID, nodeID ids.NodeID) { // WantsConnection returns true if any of the following conditions are met: // 1. The node has been manually tracked. -// 2. The node has been manually gossiped. -// 3. The node is currently a validator. +// 2. The node has been manually gossiped on a tracked subnet. +// 3. The node is currently a validator on a tracked subnet. func (i *ipTracker) WantsConnection(nodeID ids.NodeID) bool { i.lock.RLock() defer i.lock.RUnlock() - return i.trackedIDs.Contains(nodeID) + node, ok := i.tracked[nodeID] + return ok && node.wantsConnection() } // ShouldVerifyIP is used as an optimization to avoid unnecessary IP // verification. It returns true if all of the following conditions are met: -// 1. The provided IP is from a node whose connection is desired. +// 1. The provided IP is from a node whose connection is desired on any subnet. // 2. This IP is newer than the most recent IP we know of for the node. func (i *ipTracker) ShouldVerifyIP(ip *ips.ClaimedIPPort) bool { i.lock.RLock() @@ -244,29 +251,34 @@ func (i *ipTracker) ShouldVerifyIP(ip *ips.ClaimedIPPort) bool { // AddIP attempts to update the node's IP to the provided IP. This function // assumes the provided IP has been verified. Returns true if all of the // following conditions are met: -// 1. The provided IP is from a node whose connection is desired. +// 1. The provided IP is from a node whose connection is desired on a tracked +// subnet. // 2. This IP is newer than the most recent IP we know of for the node. // -// If the previous IP was marked as gossipable, calling this function will -// remove the IP from the gossipable set. +// If the previous IP for this node was marked as gossipable, calling this +// function will remove the previous IP from the gossipable set. func (i *ipTracker) AddIP(ip *ips.ClaimedIPPort) bool { i.lock.Lock() defer i.lock.Unlock() - return i.addIP(ip) > sameTimestamp + timestampComparison, wantsConnection := i.addIP(ip) + return wantsConnection && timestampComparison > sameTimestamp } -// GetIP returns the most recent IP of the provided nodeID. If a connection to -// this nodeID is not desired, this function will return false. +// GetIP returns the most recent IP of the provided nodeID. Returns true if all +// of the following conditions are met: +// 1. There is currently an IP for the provided nodeID. +// 1. The provided IP is from a node whose connection is desired on a tracked +// subnet. func (i *ipTracker) GetIP(nodeID ids.NodeID) (*ips.ClaimedIPPort, bool) { i.lock.RLock() defer i.lock.RUnlock() node, ok := i.tracked[nodeID] - if !ok { + if !ok || node.ip == nil { return nil, false } - return node.ip, node.ip != nil + return node.ip, node.wantsConnection() } // Connected is called when a connection is established. The peer should have @@ -279,15 +291,16 @@ func (i *ipTracker) Connected(ip *ips.ClaimedIPPort, trackedSubnets set.Set[ids. trackedSubnets: trackedSubnets, ip: ip, } - if i.addIP(ip) >= sameTimestamp && i.gossipableIDs.Contains(ip.NodeID) { + timestampComparison, _ := i.addIP(ip) + if timestampComparison >= sameTimestamp && i.gossipableIDs.Contains(ip.NodeID) { i.addGossipableIP(ip) } } -func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) int { +func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) (int, bool) { node, ok := i.tracked[ip.NodeID] if !ok { - return untrackedTimestamp + return untrackedTimestamp, false } if node.ip == nil { @@ -296,19 +309,19 @@ func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) int { i.updateMostRecentTrackedIP(ip) // Because we didn't previously have an IP, we know we aren't currently // connected to them. - return newTimestamp + return newTimestamp, node.wantsConnection() } if node.ip.Timestamp > ip.Timestamp { - return olderTimestamp // This IP is old than the previously known IP. + return olderTimestamp, false // This IP is older than the previously known IP. } if node.ip.Timestamp == ip.Timestamp { - return sameTimestamp // This IP is equal to the previously known IP. + return sameTimestamp, false // This IP is equal to the previously known IP. } i.updateMostRecentTrackedIP(ip) i.removeGossipableIP(ip.NodeID) - return newerTimestamp + return newerTimestamp, node.wantsConnection() } // Disconnected is called when a connection to the peer is closed. From 25b3c26c27487a0d05ccd1f1fb8dbd125fb4f614 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 30 Apr 2024 17:03:21 -0400 Subject: [PATCH 16/62] not obviously broken --- network/ip_tracker.go | 302 ++++++++++++++++++++++++++---------------- 1 file changed, 187 insertions(+), 115 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index c58c16950990..13f7b158afc5 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -70,9 +70,6 @@ func newIPTracker( bloomAdditions: make(map[ids.NodeID]int), connected: make(map[ids.NodeID]*connectedNode), subnet: make(map[ids.ID]*gossipableSubnet), - - mostRecentTrackedIPs: make(map[ids.NodeID]*ips.ClaimedIPPort), - gossipableIndices: make(map[ids.NodeID]int), } err = utils.Err( registerer.Register(tracker.numTrackedIPs), @@ -99,6 +96,9 @@ type trackedNode struct { } func (n *trackedNode) wantsConnection() bool { + if n == nil { + return false + } return n.manuallyTracked || n.trackedSubnets.Len() > 0 } @@ -125,6 +125,59 @@ type gossipableSubnet struct { gossipableIDs set.Set[ids.NodeID] } +func (s *gossipableSubnet) addGossipableIP(ip *ips.ClaimedIPPort) { + s.gossipableIndices[ip.NodeID] = len(s.gossipableIPs) + s.gossipableIPs = append(s.gossipableIPs, ip) +} + +func (s *gossipableSubnet) removeGossipableIP(nodeID ids.NodeID) { + indexToRemove, wasGossipable := s.gossipableIndices[nodeID] + if !wasGossipable { + return + } + + newNumGossipable := len(s.gossipableIPs) - 1 + if newNumGossipable != indexToRemove { + replacementIP := s.gossipableIPs[newNumGossipable] + s.gossipableIndices[replacementIP.NodeID] = indexToRemove + s.gossipableIPs[indexToRemove] = replacementIP + } + + delete(s.gossipableIndices, nodeID) + s.gossipableIPs[newNumGossipable] = nil + s.gossipableIPs = s.gossipableIPs[:newNumGossipable] +} + +func (s *gossipableSubnet) getGossipableIPs( + exceptNodeID ids.NodeID, + exceptIPs *bloom.ReadFilter, + salt []byte, + maxNumIPs int, + ips []*ips.ClaimedIPPort, + nodeIDs set.Set[ids.NodeID], +) []*ips.ClaimedIPPort { + uniform := sampler.NewUniform() + uniform.Initialize(uint64(len(s.gossipableIPs))) + + for len(ips) < maxNumIPs { + index, err := uniform.Next() + if err != nil { + return ips + } + + ip := s.gossipableIPs[index] + if ip.NodeID == exceptNodeID || nodeIDs.Contains(ip.NodeID) { + continue + } + + if !bloom.Contains(exceptIPs, ip.GossipID[:], salt) { + ips = append(ips, ip) + nodeIDs.Add(ip.NodeID) + } + } + return ips +} + type ipTracker struct { trackedSubnets set.Set[ids.ID] log logging.Logger @@ -149,39 +202,6 @@ type ipTracker struct { // tracked and untracked nodes. connected map[ids.NodeID]*connectedNode subnet map[ids.ID]*gossipableSubnet - - // TODO: Remove everything below this line - - // manuallyTracked contains the nodeIDs of all nodes whose connection was - // manually requested. - manuallyTracked set.Set[ids.NodeID] - - // TODO: Should include what subnets are causing the IPs to be tracked - // mostRecentTrackedIPs tracks the most recent IP of each node whose - // connection is desired. - // - // An IP is tracked if any of the following conditions are met: - // - The node was manually tracked - // - The node was manually requested to be gossiped - // - The node is a validator - mostRecentTrackedIPs map[ids.NodeID]*ips.ClaimedIPPort - // trackedIDs contains the nodeIDs of all nodes whose connection is desired. - trackedIDs set.Set[ids.NodeID] - - // TODO: Should be by subnet - // manuallyGossipable contains the nodeIDs of all nodes whose IP was - // manually configured to be gossiped. - manuallyGossipable set.Set[ids.NodeID] - - // TODO: Should be by subnet - // An IP is marked as gossipable if all of the following conditions are met: - // - The node is a validator or was manually requested to be gossiped - // - The node is connected - // - The IP the node connected with is its latest IP - gossipableIndices map[ids.NodeID]int - // gossipableIPs is guaranteed to be a subset of [mostRecentTrackedIPs]. - gossipableIPs []*ips.ClaimedIPPort - gossipableIDs set.Set[ids.NodeID] } // ManuallyTrack marks the provided nodeID as being desirable to connect to. @@ -204,10 +224,6 @@ func (i *ipTracker) ManuallyTrack(nodeID ids.NodeID) { // In order to avoid persistent network gossip, it's important for nodes in the // network to agree upon manually gossiped nodeIDs. func (i *ipTracker) ManuallyGossip(subnetID ids.ID, nodeID ids.NodeID) { - if subnetID != constants.PrimaryNetworkID { - return - } - i.lock.Lock() defer i.lock.Unlock() @@ -215,8 +231,7 @@ func (i *ipTracker) ManuallyGossip(subnetID ids.ID, nodeID ids.NodeID) { i.addTrackableID(nodeID, nil) } - i.addGossipableID(nodeID) - i.manuallyGossipable.Add(nodeID) + i.addGossipableID(nodeID, subnetID, true) } // WantsConnection returns true if any of the following conditions are met: @@ -261,8 +276,8 @@ func (i *ipTracker) AddIP(ip *ips.ClaimedIPPort) bool { i.lock.Lock() defer i.lock.Unlock() - timestampComparison, wantsConnection := i.addIP(ip) - return wantsConnection && timestampComparison > sameTimestamp + timestampComparison, trackedNode := i.addIP(ip) + return timestampComparison > sameTimestamp && trackedNode.wantsConnection() } // GetIP returns the most recent IP of the provided nodeID. Returns true if all @@ -291,16 +306,21 @@ func (i *ipTracker) Connected(ip *ips.ClaimedIPPort, trackedSubnets set.Set[ids. trackedSubnets: trackedSubnets, ip: ip, } - timestampComparison, _ := i.addIP(ip) - if timestampComparison >= sameTimestamp && i.gossipableIDs.Contains(ip.NodeID) { - i.addGossipableIP(ip) + if timestampComparison, _ := i.addIP(ip); timestampComparison < sameTimestamp { + return + } + + for subnetID := range trackedSubnets { + if subnet, ok := i.subnet[subnetID]; ok && subnet.gossipableIDs.Contains(ip.NodeID) { + subnet.addGossipableIP(ip) + } } } -func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) (int, bool) { +func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) (int, *trackedNode) { node, ok := i.tracked[ip.NodeID] if !ok { - return untrackedTimestamp, false + return untrackedTimestamp, nil } if node.ip == nil { @@ -309,19 +329,19 @@ func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) (int, bool) { i.updateMostRecentTrackedIP(ip) // Because we didn't previously have an IP, we know we aren't currently // connected to them. - return newTimestamp, node.wantsConnection() + return newTimestamp, node } if node.ip.Timestamp > ip.Timestamp { - return olderTimestamp, false // This IP is older than the previously known IP. + return olderTimestamp, node // This IP is older than the previously known IP. } if node.ip.Timestamp == ip.Timestamp { - return sameTimestamp, false // This IP is equal to the previously known IP. + return sameTimestamp, node // This IP is equal to the previously known IP. } i.updateMostRecentTrackedIP(ip) i.removeGossipableIP(ip.NodeID) - return newerTimestamp, node.wantsConnection() + return newerTimestamp, node } // Disconnected is called when a connection to the peer is closed. @@ -329,41 +349,55 @@ func (i *ipTracker) Disconnected(nodeID ids.NodeID) { i.lock.Lock() defer i.lock.Unlock() - delete(i.connected, nodeID) i.removeGossipableIP(nodeID) + delete(i.connected, nodeID) } -func (i *ipTracker) OnValidatorAdded(subnetID ids.ID, nodeID ids.NodeID, _ *bls.PublicKey, _ ids.ID, _ uint64) { - if subnetID != constants.PrimaryNetworkID { +func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) { + connectedNode, ok := i.connected[nodeID] + if !ok { return } + if primaryNetwork, ok := i.subnet[constants.PrimaryNetworkID]; ok { + primaryNetwork.removeGossipableIP(nodeID) + } + for subnetID := range connectedNode.trackedSubnets { + if subnet, ok := i.subnet[subnetID]; ok { + subnet.removeGossipableIP(nodeID) + } + } +} + +func (i *ipTracker) OnValidatorAdded(subnetID ids.ID, nodeID ids.NodeID, _ *bls.PublicKey, _ ids.ID, _ uint64) { i.lock.Lock() defer i.lock.Unlock() i.addTrackableID(nodeID, &subnetID) - i.addGossipableID(nodeID) + i.addGossipableID(nodeID, subnetID, false) } func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { nodeTracker, previouslyTracked := i.tracked[nodeID] if !previouslyTracked { nodeTracker = &trackedNode{} + i.tracked[nodeID] = nodeTracker } nodeTracker.manuallyTracked = nodeTracker.manuallyTracked || subnetID == nil if subnetID == nil { nodeTracker.manuallyTracked = true - i.manuallyTracked.Add(nodeID) } else { nodeTracker.subnets.Add(*subnetID) + if i.trackedSubnets.Contains(*subnetID) { + nodeTracker.trackedSubnets.Add(*subnetID) + } } if previouslyTracked { return } - i.trackedIDs.Add(nodeID) node, connected := i.connected[nodeID] if !connected { return @@ -374,54 +408,70 @@ func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { i.updateMostRecentTrackedIP(node.ip) } -func (i *ipTracker) addGossipableID(nodeID ids.NodeID) { - if i.gossipableIDs.Contains(nodeID) { +func (i *ipTracker) addGossipableID(nodeID ids.NodeID, subnetID ids.ID, manuallyGossiped bool) { + subnet, ok := i.subnet[subnetID] + if !ok { + subnet = &gossipableSubnet{ + gossipableIndices: make(map[ids.NodeID]int), + } + i.subnet[subnetID] = subnet + } + + if manuallyGossiped { + subnet.manuallyGossipable.Add(nodeID) + } + if subnet.gossipableIDs.Contains(nodeID) { return } - i.gossipableIDs.Add(nodeID) + subnet.gossipableIDs.Add(nodeID) node, connected := i.connected[nodeID] if !connected { return } - if updatedIP, ok := i.mostRecentTrackedIPs[nodeID]; !ok || node.ip.Timestamp != updatedIP.Timestamp { + if trackedNode, ok := i.tracked[nodeID]; !ok || trackedNode.ip == nil || trackedNode.ip.Timestamp != node.ip.Timestamp { return } - i.addGossipableIP(node.ip) + subnet.addGossipableIP(node.ip) } func (*ipTracker) OnValidatorWeightChanged(ids.ID, ids.NodeID, uint64, uint64) {} func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uint64) { - if subnetID != constants.PrimaryNetworkID { + i.lock.Lock() + defer i.lock.Unlock() + + subnet, ok := i.subnet[subnetID] + if !ok { return } - i.lock.Lock() - defer i.lock.Unlock() + if subnet.manuallyGossipable.Contains(nodeID) { + return + } + + subnet.gossipableIDs.Remove(nodeID) + subnet.removeGossipableIP(nodeID) - if i.manuallyGossipable.Contains(nodeID) { + trackedNode, ok := i.tracked[nodeID] + if !ok { return } - i.gossipableIDs.Remove(nodeID) - i.removeGossipableIP(nodeID) + trackedNode.subnets.Remove(subnetID) + trackedNode.trackedSubnets.Remove(subnetID) - if i.manuallyTracked.Contains(nodeID) { + if trackedNode.manuallyTracked || trackedNode.subnets.Len() > 0 { return } - i.trackedIDs.Remove(nodeID) - delete(i.mostRecentTrackedIPs, nodeID) - i.numTrackedIPs.Set(float64(len(i.mostRecentTrackedIPs))) + delete(i.tracked, nodeID) } func (i *ipTracker) updateMostRecentTrackedIP(ip *ips.ClaimedIPPort) { i.tracked[ip.NodeID].ip = ip - i.mostRecentTrackedIPs[ip.NodeID] = ip - i.numTrackedIPs.Set(float64(len(i.mostRecentTrackedIPs))) oldCount := i.bloomAdditions[ip.NodeID] if oldCount >= maxIPEntriesPerNode { @@ -450,31 +500,6 @@ func (i *ipTracker) updateMostRecentTrackedIP(ip *ips.ClaimedIPPort) { i.bloomMetrics.Count.Inc() } -func (i *ipTracker) addGossipableIP(ip *ips.ClaimedIPPort) { - i.gossipableIndices[ip.NodeID] = len(i.gossipableIPs) - i.gossipableIPs = append(i.gossipableIPs, ip) - i.numGossipableIPs.Inc() -} - -func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) { - indexToRemove, wasGossipable := i.gossipableIndices[nodeID] - if !wasGossipable { - return - } - - newNumGossipable := len(i.gossipableIPs) - 1 - if newNumGossipable != indexToRemove { - replacementIP := i.gossipableIPs[newNumGossipable] - i.gossipableIndices[replacementIP.NodeID] = indexToRemove - i.gossipableIPs[indexToRemove] = replacementIP - } - - delete(i.gossipableIndices, nodeID) - i.gossipableIPs[newNumGossipable] = nil - i.gossipableIPs = i.gossipableIPs[:newNumGossipable] - i.numGossipableIPs.Dec() -} - // GetGossipableIPs returns the latest IPs of connected validators. If // [trackedSubnets] is non-nil, only IPs of validators in the provided subnets // or the primary network are returned. The returned IPs will not contain @@ -488,30 +513,73 @@ func (i *ipTracker) GetGossipableIPs( salt []byte, maxNumIPs int, ) []*ips.ClaimedIPPort { - _ = trackedSubnets + if trackedSubnets == nil { + return i.getAllGossipableIPs(exceptNodeID, exceptIPs, salt, maxNumIPs) + } + return i.getGossipableIPs(*trackedSubnets, exceptNodeID, exceptIPs, salt, maxNumIPs) +} +func (i *ipTracker) getAllGossipableIPs( + exceptNodeID ids.NodeID, + exceptIPs *bloom.ReadFilter, + salt []byte, + maxNumIPs int, +) []*ips.ClaimedIPPort { var ( - uniform = sampler.NewUniform() ips = make([]*ips.ClaimedIPPort, 0, maxNumIPs) + nodeIDs = set.NewSet[ids.NodeID](maxNumIPs) ) i.lock.RLock() defer i.lock.RUnlock() - uniform.Initialize(uint64(len(i.gossipableIPs))) - for len(ips) < maxNumIPs { - index, err := uniform.Next() - if err != nil { - return ips + for _, subnet := range i.subnet { + ips = subnet.getGossipableIPs( + exceptNodeID, + exceptIPs, + salt, + maxNumIPs, + ips, + nodeIDs, + ) + if len(ips) >= maxNumIPs { + break } + } + return ips +} - ip := i.gossipableIPs[index] - if ip.NodeID == exceptNodeID { +func (i *ipTracker) getGossipableIPs( + trackedSubnets set.Set[ids.ID], + exceptNodeID ids.NodeID, + exceptIPs *bloom.ReadFilter, + salt []byte, + maxNumIPs int, +) []*ips.ClaimedIPPort { + var ( + ips = make([]*ips.ClaimedIPPort, 0, maxNumIPs) + nodeIDs = set.NewSet[ids.NodeID](maxNumIPs) + ) + + i.lock.RLock() + defer i.lock.RUnlock() + + for subnetID := range trackedSubnets { + subnet, ok := i.subnet[subnetID] + if !ok { continue } - if !bloom.Contains(exceptIPs, ip.GossipID[:], salt) { - ips = append(ips, ip) + ips = subnet.getGossipableIPs( + exceptNodeID, + exceptIPs, + salt, + maxNumIPs, + ips, + nodeIDs, + ) + if len(ips) >= maxNumIPs { + break } } return ips @@ -546,7 +614,7 @@ func (i *ipTracker) resetBloom() error { return err } - count := max(maxIPEntriesPerNode*i.trackedIDs.Len(), minCountEstimate) + count := max(maxIPEntriesPerNode*len(i.tracked), minCountEstimate) numHashes, numEntries := bloom.OptimalParameters( count, targetFalsePositiveProbability, @@ -561,8 +629,12 @@ func (i *ipTracker) resetBloom() error { i.bloomSalt = newSalt i.maxBloomCount = bloom.EstimateCount(numHashes, numEntries, maxFalsePositiveProbability) - for nodeID, ip := range i.mostRecentTrackedIPs { - bloom.Add(newFilter, ip.GossipID[:], newSalt) + for nodeID, trackedNode := range i.tracked { + if trackedNode.ip == nil { + continue + } + + bloom.Add(newFilter, trackedNode.ip.GossipID[:], newSalt) i.bloomAdditions[nodeID] = 1 } i.bloomMetrics.Reset(newFilter, i.maxBloomCount) From 8a15984dd2a3f0fb9a679e531cf3bc5dee2ed5c0 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 13 May 2024 13:25:39 -0400 Subject: [PATCH 17/62] nit --- network/peer/peer_test.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/network/peer/peer_test.go b/network/peer/peer_test.go index cb09e1409f15..83eb640b28d5 100644 --- a/network/peer/peer_test.go +++ b/network/peer/peer_test.go @@ -231,6 +231,7 @@ func TestPingUptimes(t *testing.T) { require.NoError(t, err) return pingMsg }(), + shouldClose: false, assertFn: func(require *require.Assertions, peer *testPeer) { uptime, ok := peer.ObservedUptime(constants.PrimaryNetworkID) require.True(ok) @@ -256,6 +257,7 @@ func TestPingUptimes(t *testing.T) { require.NoError(t, err) return pingMsg }(), + shouldClose: false, assertFn: func(require *require.Assertions, peer *testPeer) { uptime, ok := peer.ObservedUptime(constants.PrimaryNetworkID) require.True(ok) @@ -288,31 +290,30 @@ func TestPingUptimes(t *testing.T) { return pingMsg }(), shouldClose: true, + assertFn: nil, }, } - // Note: we reuse peers across tests because newRawTestPeer takes awhile to - // run. + // The raw peers are generated outside of the test cases to avoid generating + // many TLS keys. rawPeer0 := newRawTestPeer(t, sharedConfig) rawPeer1 := newRawTestPeer(t, sharedConfig) - peer0, peer1 := startTestPeers(rawPeer0, rawPeer1) - awaitReady(t, peer0, peer1) - defer func() { - peer1.StartClose() - peer0.StartClose() - require.NoError(t, peer0.AwaitClosed(context.Background())) - require.NoError(t, peer1.AwaitClosed(context.Background())) - }() - for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { require := require.New(t) + peer0, peer1 := startTestPeers(rawPeer0, rawPeer1) + awaitReady(t, peer0, peer1) + defer func() { + peer1.StartClose() + peer0.StartClose() + require.NoError(peer0.AwaitClosed(context.Background())) + require.NoError(peer1.AwaitClosed(context.Background())) + }() + require.True(peer0.Send(context.Background(), tc.msg)) - // Note: shouldClose can only be `true` for the last test because - // we reuse peers across tests. if tc.shouldClose { require.NoError(peer1.AwaitClosed(context.Background())) return From 4c9fc816f0addd8c645bc10c3fc2577f32cd2571 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 13 May 2024 17:08:12 -0400 Subject: [PATCH 18/62] Standardize peer logging --- network/peer/msg_length.go | 2 - network/peer/peer.go | 316 ++++++++++++++++++++----------------- 2 files changed, 172 insertions(+), 146 deletions(-) diff --git a/network/peer/msg_length.go b/network/peer/msg_length.go index 0072734c5875..00f3396672b3 100644 --- a/network/peer/msg_length.go +++ b/network/peer/msg_length.go @@ -16,7 +16,6 @@ var ( errMaxMessageLengthExceeded = errors.New("maximum message length exceeded") ) -// Assumes the specified [msgLen] will never >= 1<<31. func writeMsgLen(msgLen uint32, maxMsgLen uint32) ([wrappers.IntLen]byte, error) { if msgLen > maxMsgLen { return [wrappers.IntLen]byte{}, fmt.Errorf( @@ -33,7 +32,6 @@ func writeMsgLen(msgLen uint32, maxMsgLen uint32) ([wrappers.IntLen]byte, error) return b, nil } -// Assumes the read [msgLen] will never >= 1<<31. func readMsgLen(b []byte, maxMsgLen uint32) (uint32, error) { if len(b) != wrappers.IntLen { return 0, fmt.Errorf( diff --git a/network/peer/peer.go b/network/peer/peer.go index 08b635600c74..4feda2ab55bb 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -31,9 +31,18 @@ import ( "github.com/ava-labs/avalanchego/version" ) -// maxBloomSaltLen restricts the allowed size of the bloom salt to prevent -// excessively expensive bloom filter contains checks. -const maxBloomSaltLen = 32 +const ( + // maxBloomSaltLen restricts the allowed size of the bloom salt to prevent + // excessively expensive bloom filter contains checks. + maxBloomSaltLen = 32 + + disconnectingLogMessage = "disconnecting from peer" + failedToCreateMessageLogMessage = "failed to create message" + failedToSendMessageLogMessage = "failed to send message" + failedToSetDeadlineLogMessage = "failed to set connection deadline" + failedToGetUptimeLogMessage = "failed to get peer uptime percentage" + malformedMessageLogMessage = "malformed message" +) var ( errClosed = errors.New("closed") @@ -384,8 +393,9 @@ func (p *peer) readMessages() { for { // Time out and close connection if we can't read the message length if err := p.conn.SetReadDeadline(p.nextTimeout()); err != nil { - p.Log.Verbo("error setting the connection read timeout", + p.Log.Verbo(failedToSetDeadlineLogMessage, zap.Stringer("nodeID", p.id), + zap.String("direction", "read"), zap.Error(err), ) return @@ -393,7 +403,7 @@ func (p *peer) readMessages() { // Read the message length if _, err := io.ReadFull(reader, msgLenBytes); err != nil { - p.Log.Verbo("error reading message", + p.Log.Verbo("error reading message length", zap.Stringer("nodeID", p.id), zap.Error(err), ) @@ -403,7 +413,7 @@ func (p *peer) readMessages() { // Parse the message length msgLen, err := readMsgLen(msgLenBytes, constants.DefaultMaxMessageSize) if err != nil { - p.Log.Verbo("error reading message length", + p.Log.Verbo("error parsing message length", zap.Stringer("nodeID", p.id), zap.Error(err), ) @@ -437,8 +447,9 @@ func (p *peer) readMessages() { // Time out and close connection if we can't read message if err := p.conn.SetReadDeadline(p.nextTimeout()); err != nil { - p.Log.Verbo("error setting the connection read timeout", + p.Log.Verbo(failedToSetDeadlineLogMessage, zap.Stringer("nodeID", p.id), + zap.String("direction", "read"), zap.Error(err), ) onFinishedHandling() @@ -543,9 +554,9 @@ func (p *peer) writeMessages() { knownPeersSalt, ) if err != nil { - p.Log.Error("failed to create message", - zap.Stringer("messageOp", message.HandshakeOp), + p.Log.Error(failedToCreateMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.HandshakeOp), zap.Error(err), ) return @@ -588,8 +599,9 @@ func (p *peer) writeMessage(writer io.Writer, msg message.OutboundMessage) { ) if err := p.conn.SetWriteDeadline(p.nextTimeout()); err != nil { - p.Log.Verbo("error setting write deadline", + p.Log.Verbo(failedToSetDeadlineLogMessage, zap.Stringer("nodeID", p.id), + zap.String("direction", "write"), zap.Error(err), ) return @@ -635,21 +647,23 @@ func (p *peer) sendNetworkMessages() { knownPeersFilter, knownPeersSalt := p.Config.Network.KnownPeers() msg, err := p.Config.MessageCreator.GetPeerList(knownPeersFilter, knownPeersSalt) if err != nil { - p.Log.Error("failed to create get peer list message", + p.Log.Error(failedToCreateMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.GetPeerListOp), zap.Error(err), ) continue } if !p.Send(p.onClosingCtx, msg) { - p.Log.Debug("failed to send get peer list", + p.Log.Debug(failedToSendMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.GetPeerListOp), ) } case <-sendPingsTicker.C: if !p.Network.AllowConnection(p.id) { - p.Log.Debug("disconnecting from peer", + p.Log.Debug(disconnectingLogMessage, zap.String("reason", "connection is no longer desired"), zap.Stringer("nodeID", p.id), ) @@ -666,15 +680,20 @@ func (p *peer) sendNetworkMessages() { primaryUptime, subnetUptimes := p.getUptimes() pingMessage, err := p.MessageCreator.Ping(primaryUptime, subnetUptimes) if err != nil { - p.Log.Error("failed to create message", - zap.Stringer("messageOp", message.PingOp), + p.Log.Error(failedToCreateMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.PingOp), zap.Error(err), ) return } - p.Send(p.onClosingCtx, pingMessage) + if !p.Send(p.onClosingCtx, pingMessage) { + p.Log.Debug(failedToSendMessageLogMessage, + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.PingOp), + ) + } case <-p.onClosingCtx.Done(): return } @@ -692,7 +711,7 @@ func (p *peer) sendNetworkMessages() { // callback to avoid signature verification on the P-chain accept path. func (p *peer) shouldDisconnect() bool { if err := p.VersionCompatibility.Compatible(p.version); err != nil { - p.Log.Debug("disconnecting from peer", + p.Log.Debug(disconnectingLogMessage, zap.String("reason", "version not compatible"), zap.Stringer("nodeID", p.id), zap.Stringer("peerVersion", p.version), @@ -709,7 +728,7 @@ func (p *peer) shouldDisconnect() bool { } if p.ip.BLSSignature == nil { - p.Log.Debug("disconnecting from peer", + p.Log.Debug(disconnectingLogMessage, zap.String("reason", "missing BLS signature"), zap.Stringer("nodeID", p.id), ) @@ -722,7 +741,7 @@ func (p *peer) shouldDisconnect() bool { p.ip.UnsignedIP.bytes(), ) if !validSignature { - p.Log.Debug("disconnecting from peer", + p.Log.Debug(disconnectingLogMessage, zap.String("reason", "invalid BLS signature"), zap.Stringer("nodeID", p.id), ) @@ -759,8 +778,7 @@ func (p *peer) handle(msg message.InboundMessage) { return } if !p.finishedHandshake.Get() { - p.Log.Debug( - "dropping message", + p.Log.Debug("dropping message", zap.String("reason", "handshake isn't finished"), zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", msg.Op()), @@ -774,18 +792,72 @@ func (p *peer) handle(msg message.InboundMessage) { } func (p *peer) handlePing(msg *p2p.Ping) { - p.observeUptimes(msg.Uptime, msg.SubnetUptimes) + if msg.Uptime > 100 { + p.Log.Debug(malformedMessageLogMessage, + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.PingOp), + zap.Stringer("subnetID", constants.PrimaryNetworkID), + zap.Uint32("uptime", msg.Uptime), + ) + p.StartClose() + return + } + p.observeUptime(constants.PrimaryNetworkID, msg.Uptime) + + for _, subnetUptime := range msg.SubnetUptimes { + subnetID, err := ids.ToID(subnetUptime.SubnetId) + if err != nil { + p.Log.Debug(malformedMessageLogMessage, + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.PingOp), + zap.String("field", "subnetID"), + zap.Error(err), + ) + p.StartClose() + return + } + + if !p.MySubnets.Contains(subnetID) { + p.Log.Debug(malformedMessageLogMessage, + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.PingOp), + zap.Stringer("subnetID", subnetID), + zap.String("reason", "not tracking subnet"), + ) + p.StartClose() + return + } + + uptime := subnetUptime.Uptime + if uptime > 100 { + p.Log.Debug(malformedMessageLogMessage, + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.PingOp), + zap.Stringer("subnetID", subnetID), + zap.Uint32("uptime", uptime), + ) + p.StartClose() + return + } + p.observeUptime(subnetID, uptime) + } pongMessage, err := p.MessageCreator.Pong() if err != nil { - p.Log.Error("failed to create message", - zap.Stringer("messageOp", message.PongOp), + p.Log.Error(failedToCreateMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.PongOp), zap.Error(err), ) return } - p.Send(p.onClosingCtx, pongMessage) + + if !p.Send(p.onClosingCtx, pongMessage) { + p.Log.Debug(failedToSendMessageLogMessage, + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.PongOp), + ) + } } func (p *peer) getUptimes() (uint32, []*p2p.SubnetUptime) { @@ -794,7 +866,7 @@ func (p *peer) getUptimes() (uint32, []*p2p.SubnetUptime) { constants.PrimaryNetworkID, ) if err != nil { - p.Log.Debug("failed to get peer primary uptime percentage", + p.Log.Debug(failedToGetUptimeLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("subnetID", constants.PrimaryNetworkID), zap.Error(err), @@ -806,7 +878,7 @@ func (p *peer) getUptimes() (uint32, []*p2p.SubnetUptime) { for subnetID := range p.trackedSubnets { subnetUptime, err := p.UptimeCalculator.CalculateUptimePercent(p.id, subnetID) if err != nil { - p.Log.Debug("failed to get peer uptime percentage", + p.Log.Debug(failedToGetUptimeLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("subnetID", subnetID), zap.Error(err), @@ -827,52 +899,6 @@ func (p *peer) getUptimes() (uint32, []*p2p.SubnetUptime) { func (*peer) handlePong(*p2p.Pong) {} -func (p *peer) observeUptimes(primaryUptime uint32, subnetUptimes []*p2p.SubnetUptime) { - if primaryUptime > 100 { - p.Log.Debug("dropping message with invalid uptime", - zap.Stringer("nodeID", p.id), - zap.Stringer("subnetID", constants.PrimaryNetworkID), - zap.Uint32("uptime", primaryUptime), - ) - p.StartClose() - return - } - p.observeUptime(constants.PrimaryNetworkID, primaryUptime) - - for _, subnetUptime := range subnetUptimes { - subnetID, err := ids.ToID(subnetUptime.SubnetId) - if err != nil { - p.Log.Debug("dropping message with invalid subnetID", - zap.Stringer("nodeID", p.id), - zap.Error(err), - ) - p.StartClose() - return - } - - if !p.MySubnets.Contains(subnetID) { - p.Log.Debug("dropping message with unexpected subnetID", - zap.Stringer("nodeID", p.id), - zap.Stringer("subnetID", subnetID), - ) - p.StartClose() - return - } - - uptime := subnetUptime.Uptime - if uptime > 100 { - p.Log.Debug("dropping message with invalid uptime", - zap.Stringer("nodeID", p.id), - zap.Stringer("subnetID", subnetID), - zap.Uint32("uptime", uptime), - ) - p.StartClose() - return - } - p.observeUptime(subnetID, uptime) - } -} - // Record that the given peer perceives our uptime for the given [subnetID] // to be [uptime]. // Assumes [uptime] is in the range [0, 100] and [subnetID] is a valid ID of a @@ -885,16 +911,20 @@ func (p *peer) observeUptime(subnetID ids.ID, uptime uint32) { func (p *peer) handleHandshake(msg *p2p.Handshake) { if p.gotHandshake.Get() { - // TODO: this should never happen, should we close the connection here? - p.Log.Verbo("dropping duplicated handshake message", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.HandshakeOp), + zap.String("reason", "already received handshake"), ) + p.StartClose() return } if msg.NetworkId != p.NetworkID { - p.Log.Debug("networkID mismatch", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.HandshakeOp), + zap.String("field", "networkID"), zap.Uint32("peerNetworkID", msg.NetworkId), zap.Uint32("ourNetworkID", p.NetworkID), ) @@ -902,27 +932,25 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { return } - myTime := p.Clock.Time() - myTimeUnix := uint64(myTime.Unix()) - clockDifference := math.Abs(float64(msg.MyTime) - float64(myTimeUnix)) + localTime := p.Clock.Time() + localUnixTime := uint64(localTime.Unix()) + clockDifference := math.Abs(float64(msg.MyTime) - float64(localUnixTime)) p.Metrics.ClockSkewCount.Inc() p.Metrics.ClockSkewSum.Add(clockDifference) if clockDifference > p.MaxClockDifference.Seconds() { + log := p.Log.Debug if _, ok := p.Beacons.GetValidator(constants.PrimaryNetworkID, p.id); ok { - p.Log.Warn("beacon reports out of sync time", - zap.Stringer("nodeID", p.id), - zap.Uint64("peerTime", msg.MyTime), - zap.Uint64("myTime", myTimeUnix), - ) - } else { - p.Log.Debug("peer reports out of sync time", - zap.Stringer("nodeID", p.id), - zap.Uint64("peerTime", msg.MyTime), - zap.Uint64("myTime", myTimeUnix), - ) + log = p.Log.Warn } + log(malformedMessageLogMessage, + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.HandshakeOp), + zap.String("field", "myTime"), + zap.Uint64("peerTime", msg.MyTime), + zap.Uint64("localTime", localUnixTime), + ) p.StartClose() return } @@ -935,25 +963,24 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { } if p.VersionCompatibility.Version().Before(p.version) { + log := p.Log.Debug if _, ok := p.Beacons.GetValidator(constants.PrimaryNetworkID, p.id); ok { - p.Log.Info("beacon attempting to connect with newer version. You may want to update your client", - zap.Stringer("nodeID", p.id), - zap.Stringer("beaconVersion", p.version), - ) - } else { - p.Log.Debug("peer attempting to connect with newer version. You may want to update your client", - zap.Stringer("nodeID", p.id), - zap.Stringer("peerVersion", p.version), - ) + log = p.Log.Info } + log("peer attempting to connect with newer version. You may want to update your client", + zap.Stringer("nodeID", p.id), + zap.Stringer("peerVersion", p.version), + ) } // handle subnet IDs for _, subnetIDBytes := range msg.TrackedSubnets { subnetID, err := ids.ToID(subnetIDBytes) if err != nil { - p.Log.Debug("failed to parse peer's tracked subnets", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.HandshakeOp), + zap.String("field", "trackedSubnets"), zap.Error(err), ) p.StartClose() @@ -977,10 +1004,10 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { } if p.supportedACPs.Overlaps(p.objectedACPs) { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), - zap.String("field", "ACPs"), + zap.String("field", "acps"), zap.Reflect("supportedACPs", p.supportedACPs), zap.Reflect("objectedACPs", p.objectedACPs), ) @@ -996,10 +1023,10 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { var err error knownPeers, err = bloom.Parse(msg.KnownPeers.Filter) if err != nil { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), - zap.String("field", "KnownPeers.Filter"), + zap.String("field", "knownPeers.filter"), zap.Error(err), ) p.StartClose() @@ -1008,10 +1035,10 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { salt = msg.KnownPeers.Salt if saltLen := len(salt); saltLen > maxBloomSaltLen { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), - zap.String("field", "KnownPeers.Salt"), + zap.String("field", "knownPeers.salt"), zap.Int("saltLen", saltLen), ) p.StartClose() @@ -1021,20 +1048,20 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { // "net.IP" type in Golang is 16-byte if ipLen := len(msg.IpAddr); ipLen != net.IPv6len { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), - zap.String("field", "IP"), + zap.String("field", "ip"), zap.Int("ipLen", ipLen), ) p.StartClose() return } if msg.IpPort == 0 { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), - zap.String("field", "Port"), + zap.String("field", "port"), zap.Uint32("port", msg.IpPort), ) p.StartClose() @@ -1051,25 +1078,20 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { }, TLSSignature: msg.IpNodeIdSig, } - maxTimestamp := myTime.Add(p.MaxClockDifference) + maxTimestamp := localTime.Add(p.MaxClockDifference) if err := p.ip.Verify(p.cert, maxTimestamp); err != nil { + log := p.Log.Debug if _, ok := p.Beacons.GetValidator(constants.PrimaryNetworkID, p.id); ok { - p.Log.Warn("beacon has invalid signature or is out of sync", - zap.Stringer("nodeID", p.id), - zap.String("signatureType", "tls"), - zap.Uint64("peerTime", msg.MyTime), - zap.Uint64("myTime", myTimeUnix), - zap.Error(err), - ) - } else { - p.Log.Debug("peer has invalid signature or is out of sync", - zap.Stringer("nodeID", p.id), - zap.String("signatureType", "tls"), - zap.Uint64("peerTime", msg.MyTime), - zap.Uint64("myTime", myTimeUnix), - zap.Error(err), - ) + log = p.Log.Warn } + log(malformedMessageLogMessage, + zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.HandshakeOp), + zap.String("field", "tlsSignature"), + zap.Uint64("peerTime", msg.MyTime), + zap.Uint64("localTime", localUnixTime), + zap.Error(err), + ) p.StartClose() return @@ -1079,9 +1101,10 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { if len(msg.IpBlsSig) > 0 { signature, err := bls.SignatureFromBytes(msg.IpBlsSig) if err != nil { - p.Log.Debug("peer has malformed signature", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), - zap.String("signatureType", "bls"), + zap.Stringer("messageOp", message.HandshakeOp), + zap.String("field", "blsSignature"), zap.Error(err), ) p.StartClose() @@ -1108,7 +1131,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { // acknowledged correctly. peerListMsg, err := p.Config.MessageCreator.PeerList(peerIPs, true /*=bypassThrottling*/) if err != nil { - p.Log.Error("failed to create peer list handshake message", + p.Log.Error(failedToCreateMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), zap.Error(err), @@ -1119,8 +1142,9 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { if !p.Send(p.onClosingCtx, peerListMsg) { // Because throttling was marked to be bypassed with this message, // sending should only fail if the peer has started closing. - p.Log.Debug("failed to send peer list for handshake", + p.Log.Debug(failedToSendMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.PeerListOp), zap.Error(p.onClosingCtx.Err()), ) } @@ -1128,8 +1152,10 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { if !p.finishedHandshake.Get() { - p.Log.Verbo("dropping get peer list message", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.GetPeerListOp), + zap.String("reason", "not finished handshake"), ) return } @@ -1137,10 +1163,10 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { knownPeersMsg := msg.GetKnownPeers() filter, err := bloom.Parse(knownPeersMsg.GetFilter()) if err != nil { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.GetPeerListOp), - zap.String("field", "KnownPeers.Filter"), + zap.String("field", "knownPeers.filter"), zap.Error(err), ) p.StartClose() @@ -1149,10 +1175,10 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { salt := knownPeersMsg.GetSalt() if saltLen := len(salt); saltLen > maxBloomSaltLen { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.GetPeerListOp), - zap.String("field", "KnownPeers.Salt"), + zap.String("field", "knownPeers.salt"), zap.Int("saltLen", saltLen), ) p.StartClose() @@ -1171,16 +1197,18 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { // sending pattern. peerListMsg, err := p.Config.MessageCreator.PeerList(peerIPs, false /*=bypassThrottling*/) if err != nil { - p.Log.Error("failed to create peer list message", + p.Log.Error(failedToCreateMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.PeerListOp), zap.Error(err), ) return } if !p.Send(p.onClosingCtx, peerListMsg) { - p.Log.Debug("failed to send peer list", + p.Log.Debug(failedToSendMessageLogMessage, zap.Stringer("nodeID", p.id), + zap.Stringer("messageOp", message.PeerListOp), ) } } @@ -1200,10 +1228,10 @@ func (p *peer) handlePeerList(msg *p2p.PeerList) { for i, claimedIPPort := range msg.ClaimedIpPorts { tlsCert, err := staking.ParseCertificate(claimedIPPort.X509Certificate) if err != nil { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), - zap.String("field", "Cert"), + zap.String("field", "cert"), zap.Error(err), ) p.StartClose() @@ -1212,20 +1240,20 @@ func (p *peer) handlePeerList(msg *p2p.PeerList) { // "net.IP" type in Golang is 16-byte if ipLen := len(claimedIPPort.IpAddr); ipLen != net.IPv6len { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), - zap.String("field", "IP"), + zap.String("field", "ip"), zap.Int("ipLen", ipLen), ) p.StartClose() return } if claimedIPPort.IpPort == 0 { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), - zap.String("field", "Port"), + zap.String("field", "port"), zap.Uint32("port", claimedIPPort.IpPort), ) // TODO: After v1.11.x is activated, close the peer here. @@ -1244,7 +1272,7 @@ func (p *peer) handlePeerList(msg *p2p.PeerList) { } if err := p.Network.Track(discoveredIPs); err != nil { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLogMessage, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), zap.String("field", "claimedIP"), From 5a25bd177d7037027dd392eb7cfbc017865ecf19 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 13 May 2024 17:10:46 -0400 Subject: [PATCH 19/62] nit --- network/peer/peer.go | 96 ++++++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/network/peer/peer.go b/network/peer/peer.go index 4feda2ab55bb..0a47978fb8e4 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -36,12 +36,12 @@ const ( // excessively expensive bloom filter contains checks. maxBloomSaltLen = 32 - disconnectingLogMessage = "disconnecting from peer" - failedToCreateMessageLogMessage = "failed to create message" - failedToSendMessageLogMessage = "failed to send message" - failedToSetDeadlineLogMessage = "failed to set connection deadline" - failedToGetUptimeLogMessage = "failed to get peer uptime percentage" - malformedMessageLogMessage = "malformed message" + disconnectingLog = "disconnecting from peer" + failedToCreateMessageLog = "failed to create message" + failedToSendMessageLog = "failed to send message" + failedToSetDeadlineLog = "failed to set connection deadline" + failedToGetUptimeLog = "failed to get peer uptime percentage" + malformedMessageLog = "malformed message" ) var ( @@ -393,7 +393,7 @@ func (p *peer) readMessages() { for { // Time out and close connection if we can't read the message length if err := p.conn.SetReadDeadline(p.nextTimeout()); err != nil { - p.Log.Verbo(failedToSetDeadlineLogMessage, + p.Log.Verbo(failedToSetDeadlineLog, zap.Stringer("nodeID", p.id), zap.String("direction", "read"), zap.Error(err), @@ -447,7 +447,7 @@ func (p *peer) readMessages() { // Time out and close connection if we can't read message if err := p.conn.SetReadDeadline(p.nextTimeout()); err != nil { - p.Log.Verbo(failedToSetDeadlineLogMessage, + p.Log.Verbo(failedToSetDeadlineLog, zap.Stringer("nodeID", p.id), zap.String("direction", "read"), zap.Error(err), @@ -554,7 +554,7 @@ func (p *peer) writeMessages() { knownPeersSalt, ) if err != nil { - p.Log.Error(failedToCreateMessageLogMessage, + p.Log.Error(failedToCreateMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.Error(err), @@ -599,7 +599,7 @@ func (p *peer) writeMessage(writer io.Writer, msg message.OutboundMessage) { ) if err := p.conn.SetWriteDeadline(p.nextTimeout()); err != nil { - p.Log.Verbo(failedToSetDeadlineLogMessage, + p.Log.Verbo(failedToSetDeadlineLog, zap.Stringer("nodeID", p.id), zap.String("direction", "write"), zap.Error(err), @@ -647,7 +647,7 @@ func (p *peer) sendNetworkMessages() { knownPeersFilter, knownPeersSalt := p.Config.Network.KnownPeers() msg, err := p.Config.MessageCreator.GetPeerList(knownPeersFilter, knownPeersSalt) if err != nil { - p.Log.Error(failedToCreateMessageLogMessage, + p.Log.Error(failedToCreateMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.GetPeerListOp), zap.Error(err), @@ -656,14 +656,14 @@ func (p *peer) sendNetworkMessages() { } if !p.Send(p.onClosingCtx, msg) { - p.Log.Debug(failedToSendMessageLogMessage, + p.Log.Debug(failedToSendMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.GetPeerListOp), ) } case <-sendPingsTicker.C: if !p.Network.AllowConnection(p.id) { - p.Log.Debug(disconnectingLogMessage, + p.Log.Debug(disconnectingLog, zap.String("reason", "connection is no longer desired"), zap.Stringer("nodeID", p.id), ) @@ -680,7 +680,7 @@ func (p *peer) sendNetworkMessages() { primaryUptime, subnetUptimes := p.getUptimes() pingMessage, err := p.MessageCreator.Ping(primaryUptime, subnetUptimes) if err != nil { - p.Log.Error(failedToCreateMessageLogMessage, + p.Log.Error(failedToCreateMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PingOp), zap.Error(err), @@ -689,7 +689,7 @@ func (p *peer) sendNetworkMessages() { } if !p.Send(p.onClosingCtx, pingMessage) { - p.Log.Debug(failedToSendMessageLogMessage, + p.Log.Debug(failedToSendMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PingOp), ) @@ -711,7 +711,7 @@ func (p *peer) sendNetworkMessages() { // callback to avoid signature verification on the P-chain accept path. func (p *peer) shouldDisconnect() bool { if err := p.VersionCompatibility.Compatible(p.version); err != nil { - p.Log.Debug(disconnectingLogMessage, + p.Log.Debug(disconnectingLog, zap.String("reason", "version not compatible"), zap.Stringer("nodeID", p.id), zap.Stringer("peerVersion", p.version), @@ -728,7 +728,7 @@ func (p *peer) shouldDisconnect() bool { } if p.ip.BLSSignature == nil { - p.Log.Debug(disconnectingLogMessage, + p.Log.Debug(disconnectingLog, zap.String("reason", "missing BLS signature"), zap.Stringer("nodeID", p.id), ) @@ -741,7 +741,7 @@ func (p *peer) shouldDisconnect() bool { p.ip.UnsignedIP.bytes(), ) if !validSignature { - p.Log.Debug(disconnectingLogMessage, + p.Log.Debug(disconnectingLog, zap.String("reason", "invalid BLS signature"), zap.Stringer("nodeID", p.id), ) @@ -793,7 +793,7 @@ func (p *peer) handle(msg message.InboundMessage) { func (p *peer) handlePing(msg *p2p.Ping) { if msg.Uptime > 100 { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PingOp), zap.Stringer("subnetID", constants.PrimaryNetworkID), @@ -807,7 +807,7 @@ func (p *peer) handlePing(msg *p2p.Ping) { for _, subnetUptime := range msg.SubnetUptimes { subnetID, err := ids.ToID(subnetUptime.SubnetId) if err != nil { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PingOp), zap.String("field", "subnetID"), @@ -818,7 +818,7 @@ func (p *peer) handlePing(msg *p2p.Ping) { } if !p.MySubnets.Contains(subnetID) { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PingOp), zap.Stringer("subnetID", subnetID), @@ -830,7 +830,7 @@ func (p *peer) handlePing(msg *p2p.Ping) { uptime := subnetUptime.Uptime if uptime > 100 { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PingOp), zap.Stringer("subnetID", subnetID), @@ -844,7 +844,7 @@ func (p *peer) handlePing(msg *p2p.Ping) { pongMessage, err := p.MessageCreator.Pong() if err != nil { - p.Log.Error(failedToCreateMessageLogMessage, + p.Log.Error(failedToCreateMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PongOp), zap.Error(err), @@ -853,7 +853,7 @@ func (p *peer) handlePing(msg *p2p.Ping) { } if !p.Send(p.onClosingCtx, pongMessage) { - p.Log.Debug(failedToSendMessageLogMessage, + p.Log.Debug(failedToSendMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PongOp), ) @@ -866,7 +866,7 @@ func (p *peer) getUptimes() (uint32, []*p2p.SubnetUptime) { constants.PrimaryNetworkID, ) if err != nil { - p.Log.Debug(failedToGetUptimeLogMessage, + p.Log.Debug(failedToGetUptimeLog, zap.Stringer("nodeID", p.id), zap.Stringer("subnetID", constants.PrimaryNetworkID), zap.Error(err), @@ -878,7 +878,7 @@ func (p *peer) getUptimes() (uint32, []*p2p.SubnetUptime) { for subnetID := range p.trackedSubnets { subnetUptime, err := p.UptimeCalculator.CalculateUptimePercent(p.id, subnetID) if err != nil { - p.Log.Debug(failedToGetUptimeLogMessage, + p.Log.Debug(failedToGetUptimeLog, zap.Stringer("nodeID", p.id), zap.Stringer("subnetID", subnetID), zap.Error(err), @@ -911,7 +911,7 @@ func (p *peer) observeUptime(subnetID ids.ID, uptime uint32) { func (p *peer) handleHandshake(msg *p2p.Handshake) { if p.gotHandshake.Get() { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.String("reason", "already received handshake"), @@ -921,7 +921,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { } if msg.NetworkId != p.NetworkID { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.String("field", "networkID"), @@ -944,7 +944,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { if _, ok := p.Beacons.GetValidator(constants.PrimaryNetworkID, p.id); ok { log = p.Log.Warn } - log(malformedMessageLogMessage, + log(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.String("field", "myTime"), @@ -977,7 +977,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { for _, subnetIDBytes := range msg.TrackedSubnets { subnetID, err := ids.ToID(subnetIDBytes) if err != nil { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.String("field", "trackedSubnets"), @@ -1004,7 +1004,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { } if p.supportedACPs.Overlaps(p.objectedACPs) { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.String("field", "acps"), @@ -1023,7 +1023,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { var err error knownPeers, err = bloom.Parse(msg.KnownPeers.Filter) if err != nil { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.String("field", "knownPeers.filter"), @@ -1035,7 +1035,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { salt = msg.KnownPeers.Salt if saltLen := len(salt); saltLen > maxBloomSaltLen { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.String("field", "knownPeers.salt"), @@ -1048,7 +1048,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { // "net.IP" type in Golang is 16-byte if ipLen := len(msg.IpAddr); ipLen != net.IPv6len { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.String("field", "ip"), @@ -1058,7 +1058,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { return } if msg.IpPort == 0 { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.String("field", "port"), @@ -1084,7 +1084,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { if _, ok := p.Beacons.GetValidator(constants.PrimaryNetworkID, p.id); ok { log = p.Log.Warn } - log(malformedMessageLogMessage, + log(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.String("field", "tlsSignature"), @@ -1101,7 +1101,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { if len(msg.IpBlsSig) > 0 { signature, err := bls.SignatureFromBytes(msg.IpBlsSig) if err != nil { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), zap.String("field", "blsSignature"), @@ -1131,7 +1131,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { // acknowledged correctly. peerListMsg, err := p.Config.MessageCreator.PeerList(peerIPs, true /*=bypassThrottling*/) if err != nil { - p.Log.Error(failedToCreateMessageLogMessage, + p.Log.Error(failedToCreateMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), zap.Error(err), @@ -1142,7 +1142,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { if !p.Send(p.onClosingCtx, peerListMsg) { // Because throttling was marked to be bypassed with this message, // sending should only fail if the peer has started closing. - p.Log.Debug(failedToSendMessageLogMessage, + p.Log.Debug(failedToSendMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), zap.Error(p.onClosingCtx.Err()), @@ -1152,7 +1152,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { if !p.finishedHandshake.Get() { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.GetPeerListOp), zap.String("reason", "not finished handshake"), @@ -1163,7 +1163,7 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { knownPeersMsg := msg.GetKnownPeers() filter, err := bloom.Parse(knownPeersMsg.GetFilter()) if err != nil { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.GetPeerListOp), zap.String("field", "knownPeers.filter"), @@ -1175,7 +1175,7 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { salt := knownPeersMsg.GetSalt() if saltLen := len(salt); saltLen > maxBloomSaltLen { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.GetPeerListOp), zap.String("field", "knownPeers.salt"), @@ -1197,7 +1197,7 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { // sending pattern. peerListMsg, err := p.Config.MessageCreator.PeerList(peerIPs, false /*=bypassThrottling*/) if err != nil { - p.Log.Error(failedToCreateMessageLogMessage, + p.Log.Error(failedToCreateMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), zap.Error(err), @@ -1206,7 +1206,7 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { } if !p.Send(p.onClosingCtx, peerListMsg) { - p.Log.Debug(failedToSendMessageLogMessage, + p.Log.Debug(failedToSendMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), ) @@ -1228,7 +1228,7 @@ func (p *peer) handlePeerList(msg *p2p.PeerList) { for i, claimedIPPort := range msg.ClaimedIpPorts { tlsCert, err := staking.ParseCertificate(claimedIPPort.X509Certificate) if err != nil { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), zap.String("field", "cert"), @@ -1240,7 +1240,7 @@ func (p *peer) handlePeerList(msg *p2p.PeerList) { // "net.IP" type in Golang is 16-byte if ipLen := len(claimedIPPort.IpAddr); ipLen != net.IPv6len { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), zap.String("field", "ip"), @@ -1250,7 +1250,7 @@ func (p *peer) handlePeerList(msg *p2p.PeerList) { return } if claimedIPPort.IpPort == 0 { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), zap.String("field", "port"), @@ -1272,7 +1272,7 @@ func (p *peer) handlePeerList(msg *p2p.PeerList) { } if err := p.Network.Track(discoveredIPs); err != nil { - p.Log.Debug(malformedMessageLogMessage, + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), zap.String("field", "claimedIP"), From 66cae7aef590bdf0e4c195c303d244aff9421da3 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 13 May 2024 17:14:55 -0400 Subject: [PATCH 20/62] nit --- network/peer/peer.go | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/network/peer/peer.go b/network/peer/peer.go index 0a47978fb8e4..90d1157f3228 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -38,7 +38,6 @@ const ( disconnectingLog = "disconnecting from peer" failedToCreateMessageLog = "failed to create message" - failedToSendMessageLog = "failed to send message" failedToSetDeadlineLog = "failed to set connection deadline" failedToGetUptimeLog = "failed to get peer uptime percentage" malformedMessageLog = "malformed message" @@ -655,12 +654,7 @@ func (p *peer) sendNetworkMessages() { continue } - if !p.Send(p.onClosingCtx, msg) { - p.Log.Debug(failedToSendMessageLog, - zap.Stringer("nodeID", p.id), - zap.Stringer("messageOp", message.GetPeerListOp), - ) - } + p.Send(p.onClosingCtx, msg) case <-sendPingsTicker.C: if !p.Network.AllowConnection(p.id) { p.Log.Debug(disconnectingLog, @@ -688,12 +682,7 @@ func (p *peer) sendNetworkMessages() { return } - if !p.Send(p.onClosingCtx, pingMessage) { - p.Log.Debug(failedToSendMessageLog, - zap.Stringer("nodeID", p.id), - zap.Stringer("messageOp", message.PingOp), - ) - } + p.Send(p.onClosingCtx, pingMessage) case <-p.onClosingCtx.Done(): return } @@ -852,12 +841,7 @@ func (p *peer) handlePing(msg *p2p.Ping) { return } - if !p.Send(p.onClosingCtx, pongMessage) { - p.Log.Debug(failedToSendMessageLog, - zap.Stringer("nodeID", p.id), - zap.Stringer("messageOp", message.PongOp), - ) - } + p.Send(p.onClosingCtx, pongMessage) } func (p *peer) getUptimes() (uint32, []*p2p.SubnetUptime) { @@ -1142,7 +1126,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { if !p.Send(p.onClosingCtx, peerListMsg) { // Because throttling was marked to be bypassed with this message, // sending should only fail if the peer has started closing. - p.Log.Debug(failedToSendMessageLog, + p.Log.Debug("failed to send reliable message", zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.PeerListOp), zap.Error(p.onClosingCtx.Err()), @@ -1205,12 +1189,7 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { return } - if !p.Send(p.onClosingCtx, peerListMsg) { - p.Log.Debug(failedToSendMessageLog, - zap.Stringer("nodeID", p.id), - zap.Stringer("messageOp", message.PeerListOp), - ) - } + p.Send(p.onClosingCtx, peerListMsg) } func (p *peer) handlePeerList(msg *p2p.PeerList) { From ec069b7ee29dfb7f40707e48715a15c601cca54a Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 13 May 2024 17:16:11 -0400 Subject: [PATCH 21/62] nit --- network/peer/peer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/peer/peer.go b/network/peer/peer.go index 90d1157f3228..3d1ab3e6a6af 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -768,9 +768,9 @@ func (p *peer) handle(msg message.InboundMessage) { } if !p.finishedHandshake.Get() { p.Log.Debug("dropping message", - zap.String("reason", "handshake isn't finished"), zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", msg.Op()), + zap.String("reason", "handshake isn't finished"), ) msg.OnFinishedHandling() return From f44b492f1dc054dedd56262ab434efbcde2368e7 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 13 May 2024 17:19:47 -0400 Subject: [PATCH 22/62] nit --- network/peer/peer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/network/peer/peer.go b/network/peer/peer.go index 3d1ab3e6a6af..ea721087c387 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -1120,6 +1120,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { zap.Stringer("messageOp", message.PeerListOp), zap.Error(err), ) + p.StartClose() return } @@ -1131,6 +1132,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { zap.Stringer("messageOp", message.PeerListOp), zap.Error(p.onClosingCtx.Err()), ) + p.StartClose() } } From d6e99b5cd49fcf08f23c652d7aeb1da23dc37341 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 13 May 2024 17:29:48 -0400 Subject: [PATCH 23/62] nit --- network/peer/peer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/network/peer/peer.go b/network/peer/peer.go index ea721087c387..2b0f5788895b 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -651,7 +651,7 @@ func (p *peer) sendNetworkMessages() { zap.Stringer("messageOp", message.GetPeerListOp), zap.Error(err), ) - continue + return } p.Send(p.onClosingCtx, msg) @@ -838,6 +838,7 @@ func (p *peer) handlePing(msg *p2p.Ping) { zap.Stringer("messageOp", message.PongOp), zap.Error(err), ) + p.StartClose() return } From 40bb983350a130df3a0cedf539e3e5af61a12142 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 13 May 2024 17:41:17 -0400 Subject: [PATCH 24/62] merged --- network/peer/peer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/peer/peer.go b/network/peer/peer.go index 9866be545cad..0e65b0177086 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -966,10 +966,10 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { // handle subnet IDs if numTrackedSubnets := len(msg.TrackedSubnets); numTrackedSubnets > maxNumTrackedSubnets { - p.Log.Debug("message with invalid field", + p.Log.Debug(malformedMessageLog, zap.Stringer("nodeID", p.id), zap.Stringer("messageOp", message.HandshakeOp), - zap.String("field", "TrackedSubnets"), + zap.String("field", "trackedSubnets"), zap.Int("numTrackedSubnets", numTrackedSubnets), ) p.StartClose() From 85edf060387e1f89ade47953c31e36ccc7b59b51 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 13 May 2024 19:31:27 -0400 Subject: [PATCH 25/62] require both sides to think they are validators --- network/peer/peer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/peer/peer.go b/network/peer/peer.go index 9619354574f5..f34206aec07b 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -1121,7 +1121,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { p.gotHandshake.Set(true) var trackedSubnets *set.Set[ids.ID] - if !msg.AllSubnets { + if !msg.AllSubnets || p.Validators.GetWeight(constants.PrimaryNetworkID, p.MyNodeID) == 0 { trackedSubnets = &p.trackedSubnets } peerIPs := p.Network.Peers(trackedSubnets, p.id, knownPeers, salt) @@ -1187,7 +1187,7 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { } var trackedSubnets *set.Set[ids.ID] - if !msg.AllSubnets { + if !msg.AllSubnets || p.Validators.GetWeight(constants.PrimaryNetworkID, p.MyNodeID) == 0 { trackedSubnets = &p.trackedSubnets } peerIPs := p.Network.Peers(trackedSubnets, p.id, filter, salt) From 1a08def0e6ee2da2662774306580c1b769f7e025 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 14 May 2024 15:14:23 -0400 Subject: [PATCH 26/62] nit --- network/peer/peer_test.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/network/peer/peer_test.go b/network/peer/peer_test.go index c10fedfda25e..b0f723a90fe6 100644 --- a/network/peer/peer_test.go +++ b/network/peer/peer_test.go @@ -334,6 +334,18 @@ func TestTrackedSubnets(t *testing.T) { rawPeer0 := newRawTestPeer(t, sharedConfig) rawPeer1 := newRawTestPeer(t, sharedConfig) + makeSubnetIDs := func(numSubnets int) set.Set[ids.ID] { + if numSubnets == 0 { + return nil + } + + subnetIDs := set.NewSet[ids.ID](numSubnets) + for i := 0; i < numSubnets; i++ { + subnetIDs.Add(ids.GenerateTestID()) + } + return subnetIDs + } + tests := []struct { name string trackedSubnets set.Set[ids.ID] @@ -341,34 +353,22 @@ func TestTrackedSubnets(t *testing.T) { }{ { name: "primary network only", - trackedSubnets: nil, + trackedSubnets: makeSubnetIDs(0), shouldDisconnect: false, }, { name: "single subnet", - trackedSubnets: set.Of(ids.GenerateTestID()), + trackedSubnets: makeSubnetIDs(1), shouldDisconnect: false, }, { - name: "max subnets", - trackedSubnets: func() set.Set[ids.ID] { - trackedSubnets := set.NewSet[ids.ID](maxNumTrackedSubnets) - for i := 0; i < maxNumTrackedSubnets; i++ { - trackedSubnets.Add(ids.GenerateTestID()) - } - return trackedSubnets - }(), + name: "max subnets", + trackedSubnets: makeSubnetIDs(maxNumTrackedSubnets), shouldDisconnect: false, }, { - name: "too many subnets", - trackedSubnets: func() set.Set[ids.ID] { - trackedSubnets := set.NewSet[ids.ID](maxNumTrackedSubnets + 1) - for i := 0; i < maxNumTrackedSubnets+1; i++ { - trackedSubnets.Add(ids.GenerateTestID()) - } - return trackedSubnets - }(), + name: "too many subnets", + trackedSubnets: makeSubnetIDs(maxNumTrackedSubnets + 1), shouldDisconnect: true, }, } From e740fe95d733822ae8bb161cbfb0974956d484d4 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 14 May 2024 15:55:42 -0400 Subject: [PATCH 27/62] Delete empty subnets --- network/ip_tracker.go | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 13f7b158afc5..3c3923e18ed5 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -102,6 +102,13 @@ func (n *trackedNode) wantsConnection() bool { return n.manuallyTracked || n.trackedSubnets.Len() > 0 } +func (n *trackedNode) canDelete() bool { + if n == nil { + return true + } + return !n.manuallyTracked && n.subnets.Len() == 0 +} + type connectedNode struct { // trackedSubnets contains all the subnets that this node is syncing. trackedSubnets set.Set[ids.ID] @@ -166,18 +173,25 @@ func (s *gossipableSubnet) getGossipableIPs( } ip := s.gossipableIPs[index] - if ip.NodeID == exceptNodeID || nodeIDs.Contains(ip.NodeID) { + if ip.NodeID == exceptNodeID || + nodeIDs.Contains(ip.NodeID) || + bloom.Contains(exceptIPs, ip.GossipID[:], salt) { continue } - if !bloom.Contains(exceptIPs, ip.GossipID[:], salt) { - ips = append(ips, ip) - nodeIDs.Add(ip.NodeID) - } + ips = append(ips, ip) + nodeIDs.Add(ip.NodeID) } return ips } +func (s *gossipableSubnet) canDelete() bool { + if s == nil { + return true + } + return s.manuallyGossipable.Len() == 0 && s.gossipableIDs.Len() == 0 +} + type ipTracker struct { trackedSubnets set.Set[ids.ID] log logging.Logger @@ -384,7 +398,6 @@ func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { i.tracked[nodeID] = nodeTracker } - nodeTracker.manuallyTracked = nodeTracker.manuallyTracked || subnetID == nil if subnetID == nil { nodeTracker.manuallyTracked = true } else { @@ -455,6 +468,10 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin subnet.gossipableIDs.Remove(nodeID) subnet.removeGossipableIP(nodeID) + if subnet.canDelete() { + delete(i.subnet, subnetID) + } + trackedNode, ok := i.tracked[nodeID] if !ok { return @@ -463,11 +480,9 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin trackedNode.subnets.Remove(subnetID) trackedNode.trackedSubnets.Remove(subnetID) - if trackedNode.manuallyTracked || trackedNode.subnets.Len() > 0 { - return + if trackedNode.canDelete() { + delete(i.tracked, nodeID) } - - delete(i.tracked, nodeID) } func (i *ipTracker) updateMostRecentTrackedIP(ip *ips.ClaimedIPPort) { From c6e4e4d2ec58b7c49347b6b58d437ff224d6e692 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 14 May 2024 17:48:28 -0400 Subject: [PATCH 28/62] cleanup --- network/ip_tracker.go | 127 ++++++++++++----------------------- network/network.go | 64 +++++++++++++++--- network/peer/network.go | 3 +- network/peer/peer.go | 12 +--- network/peer/test_network.go | 8 ++- 5 files changed, 108 insertions(+), 106 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 3c3923e18ed5..b5f1214f671a 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -515,91 +515,6 @@ func (i *ipTracker) updateMostRecentTrackedIP(ip *ips.ClaimedIPPort) { i.bloomMetrics.Count.Inc() } -// GetGossipableIPs returns the latest IPs of connected validators. If -// [trackedSubnets] is non-nil, only IPs of validators in the provided subnets -// or the primary network are returned. The returned IPs will not contain -// [exceptNodeID] or any IPs contained in [exceptIPs]. If the number of eligible -// IPs to return low, it's possible that every IP will be iterated over while -// handling this call. -func (i *ipTracker) GetGossipableIPs( - trackedSubnets *set.Set[ids.ID], - exceptNodeID ids.NodeID, - exceptIPs *bloom.ReadFilter, - salt []byte, - maxNumIPs int, -) []*ips.ClaimedIPPort { - if trackedSubnets == nil { - return i.getAllGossipableIPs(exceptNodeID, exceptIPs, salt, maxNumIPs) - } - return i.getGossipableIPs(*trackedSubnets, exceptNodeID, exceptIPs, salt, maxNumIPs) -} - -func (i *ipTracker) getAllGossipableIPs( - exceptNodeID ids.NodeID, - exceptIPs *bloom.ReadFilter, - salt []byte, - maxNumIPs int, -) []*ips.ClaimedIPPort { - var ( - ips = make([]*ips.ClaimedIPPort, 0, maxNumIPs) - nodeIDs = set.NewSet[ids.NodeID](maxNumIPs) - ) - - i.lock.RLock() - defer i.lock.RUnlock() - - for _, subnet := range i.subnet { - ips = subnet.getGossipableIPs( - exceptNodeID, - exceptIPs, - salt, - maxNumIPs, - ips, - nodeIDs, - ) - if len(ips) >= maxNumIPs { - break - } - } - return ips -} - -func (i *ipTracker) getGossipableIPs( - trackedSubnets set.Set[ids.ID], - exceptNodeID ids.NodeID, - exceptIPs *bloom.ReadFilter, - salt []byte, - maxNumIPs int, -) []*ips.ClaimedIPPort { - var ( - ips = make([]*ips.ClaimedIPPort, 0, maxNumIPs) - nodeIDs = set.NewSet[ids.NodeID](maxNumIPs) - ) - - i.lock.RLock() - defer i.lock.RUnlock() - - for subnetID := range trackedSubnets { - subnet, ok := i.subnet[subnetID] - if !ok { - continue - } - - ips = subnet.getGossipableIPs( - exceptNodeID, - exceptIPs, - salt, - maxNumIPs, - ips, - nodeIDs, - ) - if len(ips) >= maxNumIPs { - break - } - } - return ips -} - // ResetBloom prunes the current bloom filter. This must be called periodically // to ensure that validators that change their IPs are updated correctly and // that validators that left the validator set are removed. @@ -655,3 +570,45 @@ func (i *ipTracker) resetBloom() error { i.bloomMetrics.Reset(newFilter, i.maxBloomCount) return nil } + +func getGossipableIPs[T any]( + i *ipTracker, + iter map[ids.ID]T, // The values in this map aren't actually used. + allowed func(ids.ID) bool, + exceptNodeID ids.NodeID, + exceptIPs *bloom.ReadFilter, + salt []byte, + maxNumIPs int, +) []*ips.ClaimedIPPort { + var ( + ips = make([]*ips.ClaimedIPPort, 0, maxNumIPs) + nodeIDs = set.NewSet[ids.NodeID](maxNumIPs) + ) + + i.lock.RLock() + defer i.lock.RUnlock() + + for subnetID := range iter { + if subnetID != constants.PrimaryNetworkID && !allowed(subnetID) { + continue + } + + subnet, ok := i.subnet[subnetID] + if !ok { + continue + } + + ips = subnet.getGossipableIPs( + exceptNodeID, + exceptIPs, + salt, + maxNumIPs, + ips, + nodeIDs, + ) + if len(ips) >= maxNumIPs { + break + } + } + return ips +} diff --git a/network/network.go b/network/network.go index 58d34a96ef28..564d48336708 100644 --- a/network/network.go +++ b/network/network.go @@ -513,19 +513,65 @@ func (n *network) KnownPeers() ([]byte, []byte) { return n.ipTracker.Bloom() } +// There are 3 types of responses: +// +// - Respond with all subnet IPs +// - The peer requests all peers +// - We believe the peer to be a primary network validator +// - We believe ourself to be a primary network validator +// +// - Respond with subnet IPs tracked by the peer +// - Either the peer does not request all peers or we don't consider them to +// be a primary network validator +// - We believe ourself to be a primary network validator +// +// - Respond with subnet IPs tracked by both ourselves and the peer +// - We do not consider ourself to be a primary network validator +// +// The reason we allow the peer to request all peers is so that we can avoid +// sending unnecessary data in the case that we consider them a primary network +// validator but they do not consider themselves one. func (n *network) Peers( - trackedSubnets *set.Set[ids.ID], - except ids.NodeID, + peerID ids.NodeID, + trackedSubnets set.Set[ids.ID], + requestAllPeers bool, knownPeers *bloom.ReadFilter, salt []byte, ) []*ips.ClaimedIPPort { - return n.ipTracker.GetGossipableIPs( - trackedSubnets, - except, - knownPeers, - salt, - int(n.config.PeerListNumValidatorIPs), - ) + areTheyAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, peerID) != 0 + areWeAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, n.config.MyNodeID) != 0 + switch { + case requestAllPeers && areTheyAValidator && areWeAValidator: + return getGossipableIPs( + n.ipTracker, + n.ipTracker.subnet, + func(ids.ID) bool { return true }, + peerID, + knownPeers, + salt, + int(n.config.PeerListNumValidatorIPs), + ) + case areWeAValidator: + return getGossipableIPs( + n.ipTracker, + trackedSubnets, + func(ids.ID) bool { return true }, + peerID, + knownPeers, + salt, + int(n.config.PeerListNumValidatorIPs), + ) + default: + return getGossipableIPs( + n.ipTracker, + trackedSubnets, + n.ipTracker.trackedSubnets.Contains, + peerID, + knownPeers, + salt, + int(n.config.PeerListNumValidatorIPs), + ) + } } // Dispatch starts accepting connections from other nodes attempting to connect diff --git a/network/peer/network.go b/network/peer/network.go index 49500fea57bf..19161321f1c3 100644 --- a/network/peer/network.go +++ b/network/peer/network.go @@ -35,8 +35,9 @@ type Network interface { // Peers returns peers that are not known. Peers( - trackedSubnets *set.Set[ids.ID], // If nil then all peers are returned peerID ids.NodeID, + trackedSubnets set.Set[ids.ID], + requestAllPeers bool, knownPeers *bloom.ReadFilter, peerSalt []byte, ) []*ips.ClaimedIPPort diff --git a/network/peer/peer.go b/network/peer/peer.go index f34206aec07b..a3f534441152 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -1120,11 +1120,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { p.gotHandshake.Set(true) - var trackedSubnets *set.Set[ids.ID] - if !msg.AllSubnets || p.Validators.GetWeight(constants.PrimaryNetworkID, p.MyNodeID) == 0 { - trackedSubnets = &p.trackedSubnets - } - peerIPs := p.Network.Peers(trackedSubnets, p.id, knownPeers, salt) + peerIPs := p.Network.Peers(p.id, p.trackedSubnets, msg.AllSubnets, knownPeers, salt) // We bypass throttling here to ensure that the handshake message is // acknowledged correctly. @@ -1186,11 +1182,7 @@ func (p *peer) handleGetPeerList(msg *p2p.GetPeerList) { return } - var trackedSubnets *set.Set[ids.ID] - if !msg.AllSubnets || p.Validators.GetWeight(constants.PrimaryNetworkID, p.MyNodeID) == 0 { - trackedSubnets = &p.trackedSubnets - } - peerIPs := p.Network.Peers(trackedSubnets, p.id, filter, salt) + peerIPs := p.Network.Peers(p.id, p.trackedSubnets, msg.AllSubnets, filter, salt) if len(peerIPs) == 0 { p.Log.Debug("skipping sending of empty peer list", zap.Stringer("nodeID", p.id), diff --git a/network/peer/test_network.go b/network/peer/test_network.go index bd85ad049aec..b38c7788081e 100644 --- a/network/peer/test_network.go +++ b/network/peer/test_network.go @@ -30,6 +30,12 @@ func (testNetwork) KnownPeers() ([]byte, []byte) { return bloom.EmptyFilter.Marshal(), nil } -func (testNetwork) Peers(*set.Set[ids.ID], ids.NodeID, *bloom.ReadFilter, []byte) []*ips.ClaimedIPPort { +func (testNetwork) Peers( + ids.NodeID, + set.Set[ids.ID], + bool, + *bloom.ReadFilter, + []byte, +) []*ips.ClaimedIPPort { return nil } From 4fe32d70f0bab22dc8a69ca181fea9ccff64d883 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 14 May 2024 17:57:10 -0400 Subject: [PATCH 29/62] add primary network to tracked subnets --- network/metrics.go | 7 ++++++- network/network.go | 6 ++---- network/peer/config.go | 15 ++++++++------- network/peer/peer.go | 7 +++---- network/peer/peer_test.go | 24 ++++++++++++------------ 5 files changed, 31 insertions(+), 28 deletions(-) diff --git a/network/metrics.go b/network/metrics.go index ac1df146ccd7..c6b47a1360ab 100644 --- a/network/metrics.go +++ b/network/metrics.go @@ -16,6 +16,7 @@ import ( ) type metrics struct { + // trackedSubnets does not include the primary network ID trackedSubnets set.Set[ids.ID] numTracked prometheus.Gauge @@ -42,7 +43,11 @@ type metrics struct { peerConnectedStartTimesSum float64 } -func newMetrics(namespace string, registerer prometheus.Registerer, trackedSubnets set.Set[ids.ID]) (*metrics, error) { +func newMetrics( + namespace string, + registerer prometheus.Registerer, + trackedSubnets set.Set[ids.ID], +) (*metrics, error) { m := &metrics{ trackedSubnets: trackedSubnets, numPeers: prometheus.NewGauge(prometheus.GaugeOpts{ diff --git a/network/network.go b/network/network.go index a8173b13cb82..9963612c0161 100644 --- a/network/network.go +++ b/network/network.go @@ -698,8 +698,7 @@ func (n *network) getPeers( continue } - trackedSubnets := peer.TrackedSubnets() - if subnetID != constants.PrimaryNetworkID && !trackedSubnets.Contains(subnetID) { + if trackedSubnets := peer.TrackedSubnets(); !trackedSubnets.Contains(subnetID) { continue } @@ -735,8 +734,7 @@ func (n *network) samplePeers( numValidatorsToSample+config.NonValidators+config.Peers, func(p peer.Peer) bool { // Only return peers that are tracking [subnetID] - trackedSubnets := p.TrackedSubnets() - if subnetID != constants.PrimaryNetworkID && !trackedSubnets.Contains(subnetID) { + if trackedSubnets := p.TrackedSubnets(); !trackedSubnets.Contains(subnetID) { return false } diff --git a/network/peer/config.go b/network/peer/config.go index 3eb8319216d7..8aa12820cc41 100644 --- a/network/peer/config.go +++ b/network/peer/config.go @@ -33,13 +33,14 @@ type Config struct { Network Network Router router.InboundHandler VersionCompatibility version.Compatibility - MySubnets set.Set[ids.ID] - Beacons validators.Manager - Validators validators.Manager - NetworkID uint32 - PingFrequency time.Duration - PongTimeout time.Duration - MaxClockDifference time.Duration + // MySubnets does not include the primary network ID + MySubnets set.Set[ids.ID] + Beacons validators.Manager + Validators validators.Manager + NetworkID uint32 + PingFrequency time.Duration + PongTimeout time.Duration + MaxClockDifference time.Duration SupportedACPs []uint32 ObjectedACPs []uint32 diff --git a/network/peer/peer.go b/network/peer/peer.go index 9085c39dd30c..a87bca708544 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -143,7 +143,7 @@ type peer struct { // the Handshake message. version *version.Application // trackedSubnets are the subnetIDs the peer sent us in the Handshake - // message. + // message. The primary network ID is always included. trackedSubnets set.Set[ids.ID] // options of ACPs provided in the Handshake message. supportedACPs set.Set[uint32] @@ -968,6 +968,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { return } + p.trackedSubnets.Add(constants.PrimaryNetworkID) for _, subnetIDBytes := range msg.TrackedSubnets { subnetID, err := ids.ToID(subnetIDBytes) if err != nil { @@ -980,9 +981,7 @@ func (p *peer) handleHandshake(msg *p2p.Handshake) { p.StartClose() return } - if subnetID != constants.PrimaryNetworkID { - p.trackedSubnets.Add(subnetID) - } + p.trackedSubnets.Add(subnetID) } for _, acp := range msg.SupportedAcps { diff --git a/network/peer/peer_test.go b/network/peer/peer_test.go index b0f723a90fe6..ffd5915aa2ce 100644 --- a/network/peer/peer_test.go +++ b/network/peer/peer_test.go @@ -334,21 +334,17 @@ func TestTrackedSubnets(t *testing.T) { rawPeer0 := newRawTestPeer(t, sharedConfig) rawPeer1 := newRawTestPeer(t, sharedConfig) - makeSubnetIDs := func(numSubnets int) set.Set[ids.ID] { - if numSubnets == 0 { - return nil - } - - subnetIDs := set.NewSet[ids.ID](numSubnets) - for i := 0; i < numSubnets; i++ { - subnetIDs.Add(ids.GenerateTestID()) + makeSubnetIDs := func(numSubnets int) []ids.ID { + subnetIDs := make([]ids.ID, numSubnets) + for i := range subnetIDs { + subnetIDs[i] = ids.GenerateTestID() } return subnetIDs } tests := []struct { name string - trackedSubnets set.Set[ids.ID] + trackedSubnets []ids.ID shouldDisconnect bool }{ { @@ -377,7 +373,7 @@ func TestTrackedSubnets(t *testing.T) { t.Run(test.name, func(t *testing.T) { require := require.New(t) - rawPeer0.config.MySubnets = test.trackedSubnets + rawPeer0.config.MySubnets = set.Of(test.trackedSubnets...) peer0, peer1 := startTestPeers(rawPeer0, rawPeer1) if test.shouldDisconnect { require.NoError(peer0.AwaitClosed(context.Background())) @@ -393,8 +389,12 @@ func TestTrackedSubnets(t *testing.T) { }() awaitReady(t, peer0, peer1) - require.Empty(peer0.TrackedSubnets()) - require.Equal(test.trackedSubnets, peer1.TrackedSubnets()) + + require.Equal(set.Of(constants.PrimaryNetworkID), peer0.TrackedSubnets()) + + expectedTrackedSubnets := set.Of(test.trackedSubnets...) + expectedTrackedSubnets.Add(constants.PrimaryNetworkID) + require.Equal(expectedTrackedSubnets, peer1.TrackedSubnets()) }) } } From 24cff0850f7d73b9e5bda0f3d6384f456d2a55a8 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 14 May 2024 18:18:47 -0400 Subject: [PATCH 30/62] nit --- network/ip_tracker.go | 2 +- network/network.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index b5f1214f671a..6b58fbf26882 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -589,7 +589,7 @@ func getGossipableIPs[T any]( defer i.lock.RUnlock() for subnetID := range iter { - if subnetID != constants.PrimaryNetworkID && !allowed(subnetID) { + if !allowed(subnetID) { continue } diff --git a/network/network.go b/network/network.go index 70a8a84548da..206dbc9f7d71 100644 --- a/network/network.go +++ b/network/network.go @@ -565,7 +565,9 @@ func (n *network) Peers( return getGossipableIPs( n.ipTracker, trackedSubnets, - n.ipTracker.trackedSubnets.Contains, + func(subnetID ids.ID) bool { + return subnetID == constants.PrimaryNetworkID || n.ipTracker.trackedSubnets.Contains(subnetID) + }, peerID, knownPeers, salt, From 4fe584be88a083dc2ce2bdc3df7129f1732eb037 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 14 May 2024 18:30:06 -0400 Subject: [PATCH 31/62] nit --- network/network.go | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/network/network.go b/network/network.go index 206dbc9f7d71..c6d7681b2710 100644 --- a/network/network.go +++ b/network/network.go @@ -515,6 +515,9 @@ func (n *network) KnownPeers() ([]byte, []byte) { // There are 3 types of responses: // +// - Respond with subnet IPs tracked by both ourselves and the peer +// - We do not consider ourself to be a primary network validator +// // - Respond with all subnet IPs // - The peer requests all peers // - We believe the peer to be a primary network validator @@ -525,9 +528,6 @@ func (n *network) KnownPeers() ([]byte, []byte) { // be a primary network validator // - We believe ourself to be a primary network validator // -// - Respond with subnet IPs tracked by both ourselves and the peer -// - We do not consider ourself to be a primary network validator -// // The reason we allow the peer to request all peers is so that we can avoid // sending unnecessary data in the case that we consider them a primary network // validator but they do not consider themselves one. @@ -538,42 +538,43 @@ func (n *network) Peers( knownPeers *bloom.ReadFilter, salt []byte, ) []*ips.ClaimedIPPort { - areTheyAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, peerID) != 0 areWeAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, n.config.MyNodeID) != 0 - switch { - case requestAllPeers && areTheyAValidator && areWeAValidator: - return getGossipableIPs( - n.ipTracker, - n.ipTracker.subnet, - func(ids.ID) bool { return true }, - peerID, - knownPeers, - salt, - int(n.config.PeerListNumValidatorIPs), - ) - case areWeAValidator: + if !areWeAValidator { return getGossipableIPs( n.ipTracker, trackedSubnets, - func(ids.ID) bool { return true }, + func(subnetID ids.ID) bool { + return subnetID == constants.PrimaryNetworkID || n.ipTracker.trackedSubnets.Contains(subnetID) + }, peerID, knownPeers, salt, int(n.config.PeerListNumValidatorIPs), ) - default: + } + + areTheyAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, peerID) != 0 + if requestAllPeers && areTheyAValidator { return getGossipableIPs( n.ipTracker, - trackedSubnets, - func(subnetID ids.ID) bool { - return subnetID == constants.PrimaryNetworkID || n.ipTracker.trackedSubnets.Contains(subnetID) - }, + n.ipTracker.subnet, + func(ids.ID) bool { return true }, peerID, knownPeers, salt, int(n.config.PeerListNumValidatorIPs), ) } + + return getGossipableIPs( + n.ipTracker, + trackedSubnets, + func(ids.ID) bool { return true }, + peerID, + knownPeers, + salt, + int(n.config.PeerListNumValidatorIPs), + ) } // Dispatch starts accepting connections from other nodes attempting to connect From f5d57e837bfc31a82ff811f3f06e8de7467e382f Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 14 May 2024 18:43:33 -0400 Subject: [PATCH 32/62] metrics --- network/ip_tracker.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 6b58fbf26882..3420625c321d 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -58,12 +58,17 @@ func newIPTracker( numTrackedIPs: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Name: "tracked_ips", - Help: "Number of IPs this node is willing to dial", + Help: "number of IPs this node is willing to dial", }), numGossipableIPs: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Name: "gossipable_ips", - Help: "Number of IPs this node is willing to gossip", + Help: "number of IPs this node has observed as able to be gossiped", + }), + numSubnets: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "subnets", + Help: "number of subnets this node is monitoring", }), bloomMetrics: bloomMetrics, tracked: make(map[ids.NodeID]*trackedNode), @@ -74,6 +79,7 @@ func newIPTracker( err = utils.Err( registerer.Register(tracker.numTrackedIPs), registerer.Register(tracker.numGossipableIPs), + registerer.Register(tracker.numSubnets), ) if err != nil { return nil, err @@ -118,6 +124,8 @@ type connectedNode struct { } type gossipableSubnet struct { + numGossipableIPs prometheus.Gauge + // manuallyGossipable contains the nodeIDs of all nodes whose IP was // manually configured to be gossiped for this subnet. manuallyGossipable set.Set[ids.NodeID] @@ -133,6 +141,7 @@ type gossipableSubnet struct { } func (s *gossipableSubnet) addGossipableIP(ip *ips.ClaimedIPPort) { + s.numGossipableIPs.Inc() s.gossipableIndices[ip.NodeID] = len(s.gossipableIPs) s.gossipableIPs = append(s.gossipableIPs, ip) } @@ -150,6 +159,7 @@ func (s *gossipableSubnet) removeGossipableIP(nodeID ids.NodeID) { s.gossipableIPs[indexToRemove] = replacementIP } + s.numGossipableIPs.Dec() delete(s.gossipableIndices, nodeID) s.gossipableIPs[newNumGossipable] = nil s.gossipableIPs = s.gossipableIPs[:newNumGossipable] @@ -197,6 +207,7 @@ type ipTracker struct { log logging.Logger numTrackedIPs prometheus.Gauge numGossipableIPs prometheus.Gauge + numSubnets prometheus.Gauge bloomMetrics *bloom.Metrics lock sync.RWMutex @@ -340,7 +351,7 @@ func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) (int, *trackedNode) { if node.ip == nil { // This is the first IP we've heard from the validator, so it is the // most recent. - i.updateMostRecentTrackedIP(ip) + i.updateMostRecentTrackedIP(node, ip) // Because we didn't previously have an IP, we know we aren't currently // connected to them. return newTimestamp, node @@ -353,7 +364,7 @@ func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) (int, *trackedNode) { return sameTimestamp, node // This IP is equal to the previously known IP. } - i.updateMostRecentTrackedIP(ip) + i.updateMostRecentTrackedIP(node, ip) i.removeGossipableIP(ip.NodeID) return newerTimestamp, node } @@ -394,6 +405,7 @@ func (i *ipTracker) OnValidatorAdded(subnetID ids.ID, nodeID ids.NodeID, _ *bls. func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { nodeTracker, previouslyTracked := i.tracked[nodeID] if !previouslyTracked { + i.numTrackedIPs.Inc() nodeTracker = &trackedNode{} i.tracked[nodeID] = nodeTracker } @@ -418,13 +430,15 @@ func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { // Because we previously weren't tracking this nodeID, the IP from the // connection is guaranteed to be the most up-to-date IP that we know. - i.updateMostRecentTrackedIP(node.ip) + i.updateMostRecentTrackedIP(nodeTracker, node.ip) } func (i *ipTracker) addGossipableID(nodeID ids.NodeID, subnetID ids.ID, manuallyGossiped bool) { subnet, ok := i.subnet[subnetID] if !ok { + i.numSubnets.Inc() subnet = &gossipableSubnet{ + numGossipableIPs: i.numGossipableIPs, gossipableIndices: make(map[ids.NodeID]int), } i.subnet[subnetID] = subnet @@ -469,6 +483,7 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin subnet.removeGossipableIP(nodeID) if subnet.canDelete() { + i.numSubnets.Dec() delete(i.subnet, subnetID) } @@ -481,12 +496,13 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin trackedNode.trackedSubnets.Remove(subnetID) if trackedNode.canDelete() { + i.numTrackedIPs.Dec() delete(i.tracked, nodeID) } } -func (i *ipTracker) updateMostRecentTrackedIP(ip *ips.ClaimedIPPort) { - i.tracked[ip.NodeID].ip = ip +func (i *ipTracker) updateMostRecentTrackedIP(node *trackedNode, ip *ips.ClaimedIPPort) { + node.ip = ip oldCount := i.bloomAdditions[ip.NodeID] if oldCount >= maxIPEntriesPerNode { From 9f2f84d24559a26a7b8d1e489a7e9aed9f5b8e19 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 14 May 2024 18:46:54 -0400 Subject: [PATCH 33/62] metrics --- network/ip_tracker.go | 36 ++++++++++++++++++------------------ network/ip_tracker_test.go | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 3420625c321d..82a45f30df0d 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -55,19 +55,19 @@ func newIPTracker( tracker := &ipTracker{ trackedSubnets: trackedSubnets, log: log, - numTrackedIPs: prometheus.NewGauge(prometheus.GaugeOpts{ + numTrackedPeers: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, - Name: "tracked_ips", - Help: "number of IPs this node is willing to dial", + Name: "tracked_peers", + Help: "number of peers this node is monitoring", }), numGossipableIPs: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Name: "gossipable_ips", - Help: "number of IPs this node has observed as able to be gossiped", + Help: "number of IPs this node considers able to be gossiped", }), - numSubnets: prometheus.NewGauge(prometheus.GaugeOpts{ + numTrackedSubnets: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, - Name: "subnets", + Name: "tracked_subnets", Help: "number of subnets this node is monitoring", }), bloomMetrics: bloomMetrics, @@ -77,9 +77,9 @@ func newIPTracker( subnet: make(map[ids.ID]*gossipableSubnet), } err = utils.Err( - registerer.Register(tracker.numTrackedIPs), + registerer.Register(tracker.numTrackedPeers), registerer.Register(tracker.numGossipableIPs), - registerer.Register(tracker.numSubnets), + registerer.Register(tracker.numTrackedSubnets), ) if err != nil { return nil, err @@ -203,12 +203,12 @@ func (s *gossipableSubnet) canDelete() bool { } type ipTracker struct { - trackedSubnets set.Set[ids.ID] - log logging.Logger - numTrackedIPs prometheus.Gauge - numGossipableIPs prometheus.Gauge - numSubnets prometheus.Gauge - bloomMetrics *bloom.Metrics + trackedSubnets set.Set[ids.ID] + log logging.Logger + numTrackedPeers prometheus.Gauge + numGossipableIPs prometheus.Gauge + numTrackedSubnets prometheus.Gauge + bloomMetrics *bloom.Metrics lock sync.RWMutex tracked map[ids.NodeID]*trackedNode @@ -405,7 +405,7 @@ func (i *ipTracker) OnValidatorAdded(subnetID ids.ID, nodeID ids.NodeID, _ *bls. func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { nodeTracker, previouslyTracked := i.tracked[nodeID] if !previouslyTracked { - i.numTrackedIPs.Inc() + i.numTrackedPeers.Inc() nodeTracker = &trackedNode{} i.tracked[nodeID] = nodeTracker } @@ -436,7 +436,7 @@ func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { func (i *ipTracker) addGossipableID(nodeID ids.NodeID, subnetID ids.ID, manuallyGossiped bool) { subnet, ok := i.subnet[subnetID] if !ok { - i.numSubnets.Inc() + i.numTrackedSubnets.Inc() subnet = &gossipableSubnet{ numGossipableIPs: i.numGossipableIPs, gossipableIndices: make(map[ids.NodeID]int), @@ -483,7 +483,7 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin subnet.removeGossipableIP(nodeID) if subnet.canDelete() { - i.numSubnets.Dec() + i.numTrackedSubnets.Dec() delete(i.subnet, subnetID) } @@ -496,7 +496,7 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin trackedNode.trackedSubnets.Remove(subnetID) if trackedNode.canDelete() { - i.numTrackedIPs.Dec() + i.numTrackedPeers.Dec() delete(i.tracked, nodeID) } } diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index 4c8f621667a2..fb49a6c6e6b4 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -48,7 +48,7 @@ func requireEqual(t *testing.T, expected, actual *ipTracker) { func requireMetricsConsistent(t *testing.T, tracker *ipTracker) { require := require.New(t) - require.Equal(float64(len(tracker.mostRecentTrackedIPs)), testutil.ToFloat64(tracker.numTrackedIPs)) + require.Equal(float64(len(tracker.tracked)), testutil.ToFloat64(tracker.numTrackedPeers)) require.Equal(float64(len(tracker.gossipableIPs)), testutil.ToFloat64(tracker.numGossipableIPs)) require.Equal(float64(tracker.bloom.Count()), testutil.ToFloat64(tracker.bloomMetrics.Count)) require.Equal(float64(tracker.maxBloomCount), testutil.ToFloat64(tracker.bloomMetrics.MaxCount)) From 5384f3763094990c003561940da6157a473505cb Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Wed, 15 May 2024 13:12:02 -0400 Subject: [PATCH 34/62] Add comments --- network/ip_tracker.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 82a45f30df0d..8e8f88c81a38 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -93,9 +93,11 @@ func newIPTracker( type trackedNode struct { // manuallyTracked tracks if this node's connection was manually requested. manuallyTracked bool - // subnets contains all the subnets that this node is a validator of. + // subnets contains all the subnets that this node is a validator of, + // including potentially the primary network. subnets set.Set[ids.ID] - // subnets contains the subset of [subnets] that the local node also tracks. + // subnets contains the subset of [subnets] that the local node also tracks, + // including potentially the primary network. trackedSubnets set.Set[ids.ID] // ip is the most recently known IP of this node. ip *ips.ClaimedIPPort @@ -116,7 +118,8 @@ func (n *trackedNode) canDelete() bool { } type connectedNode struct { - // trackedSubnets contains all the subnets that this node is syncing. + // trackedSubnets contains all the subnets that this node is syncing, + // including the primary network. trackedSubnets set.Set[ids.ID] // ip this node claimed when connecting to use. The IP is not necessarily // the same IP as in mostRecentTrackedIPs. @@ -203,6 +206,7 @@ func (s *gossipableSubnet) canDelete() bool { } type ipTracker struct { + // trackedSubnets does not include the primary network. trackedSubnets set.Set[ids.ID] log logging.Logger numTrackedPeers prometheus.Gauge From 2b0d0d561444d748bfa20aab17163810f2868962 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Wed, 15 May 2024 23:52:02 -0400 Subject: [PATCH 35/62] Fix tests + fix bugs --- network/ip_tracker.go | 41 +- network/ip_tracker_test.go | 1226 ++++++++++++++++++++++++++++-------- network/network.go | 7 +- 3 files changed, 985 insertions(+), 289 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 8e8f88c81a38..c06be78485c1 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -104,16 +104,10 @@ type trackedNode struct { } func (n *trackedNode) wantsConnection() bool { - if n == nil { - return false - } return n.manuallyTracked || n.trackedSubnets.Len() > 0 } func (n *trackedNode) canDelete() bool { - if n == nil { - return true - } return !n.manuallyTracked && n.subnets.Len() == 0 } @@ -133,6 +127,10 @@ type gossipableSubnet struct { // manually configured to be gossiped for this subnet. manuallyGossipable set.Set[ids.NodeID] + // gossipableIDs contains the nodeIDs of all nodes whose IP could be + // gossiped. + gossipableIDs set.Set[ids.NodeID] + // An IP is marked as gossipable if all of the following conditions are met: // - The node is a validator or was manually requested to be gossiped // - The node is connected @@ -140,7 +138,6 @@ type gossipableSubnet struct { // - The IP the node connected with is its latest IP gossipableIndices map[ids.NodeID]int gossipableIPs []*ips.ClaimedIPPort - gossipableIDs set.Set[ids.NodeID] } func (s *gossipableSubnet) addGossipableIP(ip *ips.ClaimedIPPort) { @@ -199,9 +196,6 @@ func (s *gossipableSubnet) getGossipableIPs( } func (s *gossipableSubnet) canDelete() bool { - if s == nil { - return true - } return s.manuallyGossipable.Len() == 0 && s.gossipableIDs.Len() == 0 } @@ -260,6 +254,7 @@ func (i *ipTracker) ManuallyGossip(subnetID ids.ID, nodeID ids.NodeID) { i.addTrackableID(nodeID, nil) } + i.addTrackableID(nodeID, &subnetID) i.addGossipableID(nodeID, subnetID, true) } @@ -277,9 +272,12 @@ func (i *ipTracker) WantsConnection(nodeID ids.NodeID) bool { // ShouldVerifyIP is used as an optimization to avoid unnecessary IP // verification. It returns true if all of the following conditions are met: -// 1. The provided IP is from a node whose connection is desired on any subnet. +// 1. The provided IP is from a node whose connection is desired. // 2. This IP is newer than the most recent IP we know of for the node. -func (i *ipTracker) ShouldVerifyIP(ip *ips.ClaimedIPPort) bool { +func (i *ipTracker) ShouldVerifyIP( + ip *ips.ClaimedIPPort, + trackAllSubnets bool, +) bool { i.lock.RLock() defer i.lock.RUnlock() @@ -288,6 +286,10 @@ func (i *ipTracker) ShouldVerifyIP(ip *ips.ClaimedIPPort) bool { return false } + if !trackAllSubnets && !node.wantsConnection() { + return false + } + return node.ip == nil || // This would be the first IP node.ip.Timestamp < ip.Timestamp // This would be a newer IP } @@ -388,9 +390,6 @@ func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) { return } - if primaryNetwork, ok := i.subnet[constants.PrimaryNetworkID]; ok { - primaryNetwork.removeGossipableIP(nodeID) - } for subnetID := range connectedNode.trackedSubnets { if subnet, ok := i.subnet[subnetID]; ok { subnet.removeGossipableIP(nodeID) @@ -418,7 +417,7 @@ func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { nodeTracker.manuallyTracked = true } else { nodeTracker.subnets.Add(*subnetID) - if i.trackedSubnets.Contains(*subnetID) { + if *subnetID == constants.PrimaryNetworkID || i.trackedSubnets.Contains(*subnetID) { nodeTracker.trackedSubnets.Add(*subnetID) } } @@ -457,7 +456,7 @@ func (i *ipTracker) addGossipableID(nodeID ids.NodeID, subnetID ids.ID, manually subnet.gossipableIDs.Add(nodeID) node, connected := i.connected[nodeID] - if !connected { + if !connected || !node.trackedSubnets.Contains(subnetID) { return } @@ -476,6 +475,10 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin subnet, ok := i.subnet[subnetID] if !ok { + i.log.Error("removing validator from untracked subnet", + zap.Stringer("subnetID", subnetID), + zap.Stringer("nodeID", nodeID), + ) return } @@ -493,6 +496,10 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin trackedNode, ok := i.tracked[nodeID] if !ok { + i.log.Error("removing untracked validator", + zap.Stringer("subnetID", subnetID), + zap.Stringer("nodeID", nodeID), + ) return } diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index fb49a6c6e6b4..4a99dd3dfc8b 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -15,10 +15,16 @@ import ( "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/ips" "github.com/ava-labs/avalanchego/utils/logging" + "github.com/ava-labs/avalanchego/utils/set" ) -func newTestIPTracker(t *testing.T) *ipTracker { - tracker, err := newIPTracker(nil, logging.NoLog{}, "", prometheus.NewRegistry()) +func newTestIPTracker(t *testing.T, trackedSubnets ...ids.ID) *ipTracker { + tracker, err := newIPTracker( + set.Of(trackedSubnets...), + logging.NoLog{}, + "", + prometheus.NewRegistry(), + ) require.NoError(t, err) return tracker } @@ -34,27 +40,29 @@ func newerTestIP(ip *ips.ClaimedIPPort) *ips.ClaimedIPPort { func requireEqual(t *testing.T, expected, actual *ipTracker) { require := require.New(t) - require.Equal(expected.manuallyTracked, actual.manuallyTracked) - require.Equal(expected.manuallyGossipable, actual.manuallyGossipable) - require.Equal(expected.mostRecentTrackedIPs, actual.mostRecentTrackedIPs) - require.Equal(expected.trackedIDs, actual.trackedIDs) + require.Equal(expected.tracked, actual.tracked) require.Equal(expected.bloomAdditions, actual.bloomAdditions) require.Equal(expected.maxBloomCount, actual.maxBloomCount) require.Equal(expected.connected, actual.connected) - require.Equal(expected.gossipableIndices, actual.gossipableIndices) - require.Equal(expected.gossipableIPs, actual.gossipableIPs) - require.Equal(expected.gossipableIDs, actual.gossipableIDs) + require.Equal(expected.connected, actual.connected) + require.Equal(expected.subnet, actual.subnet) } func requireMetricsConsistent(t *testing.T, tracker *ipTracker) { require := require.New(t) require.Equal(float64(len(tracker.tracked)), testutil.ToFloat64(tracker.numTrackedPeers)) - require.Equal(float64(len(tracker.gossipableIPs)), testutil.ToFloat64(tracker.numGossipableIPs)) + var numGossipableIPs int + for _, subnet := range tracker.subnet { + numGossipableIPs += len(subnet.gossipableIndices) + } + require.Equal(float64(numGossipableIPs), testutil.ToFloat64(tracker.numGossipableIPs)) + require.Equal(float64(len(tracker.subnet)), testutil.ToFloat64(tracker.numTrackedSubnets)) require.Equal(float64(tracker.bloom.Count()), testutil.ToFloat64(tracker.bloomMetrics.Count)) require.Equal(float64(tracker.maxBloomCount), testutil.ToFloat64(tracker.bloomMetrics.MaxCount)) } func TestIPTracker_ManuallyTrack(t *testing.T) { + subnetID := ids.GenerateTestID() tests := []struct { name string initialState *ipTracker @@ -67,8 +75,11 @@ func TestIPTracker_ManuallyTrack(t *testing.T) { nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.trackedIDs.Add(ip.NodeID) + + tracker.numTrackedPeers.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + manuallyTracked: true, + } return tracker }(), }, @@ -76,22 +87,25 @@ func TestIPTracker_ManuallyTrack(t *testing.T) { name: "connected non-validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.mostRecentTrackedIPs[ip.NodeID] = ip - tracker.trackedIDs.Add(ip.NodeID) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + + tracker.numTrackedPeers.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + manuallyTracked: true, + ip: ip, + } tracker.bloomAdditions[ip.NodeID] = 1 return tracker }(), }, { - name: "non-connected validator", + name: "non-connected tracked validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) @@ -101,24 +115,60 @@ func TestIPTracker_ManuallyTrack(t *testing.T) { expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.manuallyTracked.Add(ip.NodeID) + + tracker.tracked[ip.NodeID].manuallyTracked = true return tracker }(), }, { - name: "connected validator", + name: "non-connected untracked validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + + tracker.tracked[ip.NodeID].manuallyTracked = true + return tracker + }(), + }, + { + name: "connected tracked validator", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.manuallyTracked.Add(ip.NodeID) + + tracker.tracked[ip.NodeID].manuallyTracked = true + return tracker + }(), + }, + { + name: "connected untracked validator", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + return tracker + }(), + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + + tracker.tracked[ip.NodeID].manuallyTracked = true return tracker }(), }, @@ -133,136 +183,358 @@ func TestIPTracker_ManuallyTrack(t *testing.T) { } func TestIPTracker_ManuallyGossip(t *testing.T) { + subnetID := ids.GenerateTestID() tests := []struct { name string initialState *ipTracker + subnetID ids.ID nodeID ids.NodeID expectedState *ipTracker }{ { - name: "non-connected non-validator", + name: "non-connected tracked non-validator", initialState: newTestIPTracker(t), + subnetID: constants.PrimaryNetworkID, nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.manuallyGossipable.Add(ip.NodeID) - tracker.trackedIDs.Add(ip.NodeID) - tracker.gossipableIDs.Add(ip.NodeID) + + tracker.numTrackedPeers.Inc() + tracker.numTrackedSubnets.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + manuallyTracked: true, + subnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + } + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + manuallyGossipable: set.Of(ip.NodeID), + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), + } return tracker }(), }, { - name: "connected non-validator", + name: "non-connected untracked non-validator", + initialState: newTestIPTracker(t), + subnetID: subnetID, + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + + tracker.numTrackedPeers.Inc() + tracker.numTrackedSubnets.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + subnets: set.Of(subnetID), + } + tracker.subnet[subnetID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + manuallyGossipable: set.Of(ip.NodeID), + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), + } + return tracker + }(), + }, + { + name: "connected tracked non-validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.manuallyGossipable.Add(ip.NodeID) - tracker.mostRecentTrackedIPs[ip.NodeID] = ip - tracker.trackedIDs.Add(ip.NodeID) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + + tracker.numTrackedPeers.Inc() + tracker.numGossipableIPs.Inc() + tracker.numTrackedSubnets.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + manuallyTracked: true, + subnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } tracker.bloomAdditions[ip.NodeID] = 1 - tracker.gossipableIndices[ip.NodeID] = 0 - tracker.gossipableIPs = []*ips.ClaimedIPPort{ - ip, + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + manuallyGossipable: set.Of(ip.NodeID), + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: map[ids.NodeID]int{ + ip.NodeID: 0, + }, + gossipableIPs: []*ips.ClaimedIPPort{ + ip, + }, + } + return tracker + }(), + }, + { + name: "connected untracked non-validator", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + return tracker + }(), + subnetID: subnetID, + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + + tracker.numTrackedPeers.Inc() + tracker.numTrackedSubnets.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + subnets: set.Of(subnetID), + ip: ip, + } + tracker.bloomAdditions[ip.NodeID] = 1 + tracker.subnet[subnetID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + manuallyGossipable: set.Of(ip.NodeID), + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), } - tracker.gossipableIDs.Add(ip.NodeID) return tracker }(), }, { - name: "non-connected validator", + name: "non-connected tracked validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.manuallyGossipable.Add(ip.NodeID) + + tracker.tracked[ip.NodeID].manuallyTracked = true + tracker.subnet[constants.PrimaryNetworkID].manuallyGossipable = set.Of(ip.NodeID) return tracker }(), }, { - name: "connected validator", + name: "non-connected untracked validator", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + return tracker + }(), + subnetID: subnetID, + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + + tracker.subnet[subnetID].manuallyGossipable = set.Of(ip.NodeID) + return tracker + }(), + }, + { + name: "connected tracked validator", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.manuallyTracked.Add(ip.NodeID) - tracker.manuallyGossipable.Add(ip.NodeID) + + tracker.tracked[ip.NodeID].manuallyTracked = true + tracker.subnet[constants.PrimaryNetworkID].manuallyGossipable = set.Of(ip.NodeID) + return tracker + }(), + }, + { + name: "connected untracked validator", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + return tracker + }(), + subnetID: subnetID, + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + + tracker.subnet[subnetID].manuallyGossipable = set.Of(ip.NodeID) return tracker }(), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.ManuallyGossip(constants.PrimaryNetworkID, test.nodeID) + test.initialState.ManuallyGossip(test.subnetID, test.nodeID) requireEqual(t, test.expectedState, test.initialState) requireMetricsConsistent(t, test.initialState) }) } } +func TestIPTracker_ShouldVerifyIP(t *testing.T) { + newerIP := newerTestIP(ip) + tests := []struct { + name string + tracker *ipTracker + ip *ips.ClaimedIPPort + expectedTrackAllSubnets bool + expectedTrackRequestedSubnets bool + }{ + { + name: "node not tracked", + tracker: newTestIPTracker(t), + ip: ip, + expectedTrackAllSubnets: false, + expectedTrackRequestedSubnets: false, + }, + { + name: "undesired connection", + tracker: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(ids.GenerateTestID(), ip.NodeID, nil, ids.Empty, 0) + return tracker + }(), + ip: ip, + expectedTrackAllSubnets: true, + expectedTrackRequestedSubnets: false, + }, + { + name: "desired connection first IP", + tracker: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + return tracker + }(), + ip: ip, + expectedTrackAllSubnets: true, + expectedTrackRequestedSubnets: true, + }, + { + name: "desired connection older IP", + tracker: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + require.True(t, tracker.AddIP(newerIP)) + return tracker + }(), + ip: ip, + expectedTrackAllSubnets: false, + expectedTrackRequestedSubnets: false, + }, + { + name: "desired connection same IP", + tracker: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + require.True(t, tracker.AddIP(ip)) + return tracker + }(), + ip: ip, + expectedTrackAllSubnets: false, + expectedTrackRequestedSubnets: false, + }, + { + name: "desired connection newer IP", + tracker: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + require.True(t, tracker.AddIP(ip)) + return tracker + }(), + ip: newerIP, + expectedTrackAllSubnets: true, + expectedTrackRequestedSubnets: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require := require.New(t) + + require.Equal(test.expectedTrackAllSubnets, test.tracker.ShouldVerifyIP(test.ip, true)) + require.Equal(test.expectedTrackRequestedSubnets, test.tracker.ShouldVerifyIP(test.ip, false)) + }) + } +} + func TestIPTracker_AddIP(t *testing.T) { + subnetID := ids.GenerateTestID() newerIP := newerTestIP(ip) tests := []struct { - name string - initialState *ipTracker - ip *ips.ClaimedIPPort - expectedUpdated bool - expectedState *ipTracker + name string + initialState *ipTracker + ip *ips.ClaimedIPPort + expectedUpdatedAndDesired bool + expectedState *ipTracker }{ { - name: "non-validator", - initialState: newTestIPTracker(t), - ip: ip, - expectedUpdated: false, - expectedState: newTestIPTracker(t), + name: "non-validator", + initialState: newTestIPTracker(t), + ip: ip, + expectedUpdatedAndDesired: false, + expectedState: newTestIPTracker(t), }, { - name: "first known IP", + name: "first known IP of tracked node", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), - ip: ip, - expectedUpdated: true, + ip: ip, + expectedUpdatedAndDesired: true, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.mostRecentTrackedIPs[ip.NodeID] = ip + + tracker.tracked[ip.NodeID].ip = ip tracker.bloomAdditions[ip.NodeID] = 1 return tracker }(), }, { - name: "older IP", + name: "first known IP of untracked node", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + return tracker + }(), + ip: ip, + expectedUpdatedAndDesired: false, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + + tracker.tracked[ip.NodeID].ip = ip + tracker.bloomAdditions[ip.NodeID] = 1 + return tracker + }(), + }, + { + name: "older IP of tracked node", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) return tracker }(), - ip: ip, - expectedUpdated: false, + ip: ip, + expectedUpdatedAndDesired: false, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) @@ -271,15 +543,32 @@ func TestIPTracker_AddIP(t *testing.T) { }(), }, { - name: "same IP", + name: "older IP of untracked node", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(newerIP)) + return tracker + }(), + ip: ip, + expectedUpdatedAndDesired: false, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(newerIP)) + return tracker + }(), + }, + { + name: "same IP of tracked node", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker }(), - ip: ip, - expectedUpdated: false, + ip: ip, + expectedUpdatedAndDesired: false, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) @@ -288,42 +577,108 @@ func TestIPTracker_AddIP(t *testing.T) { }(), }, { - name: "disconnected newer IP", + name: "same IP of untracked node", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) + return tracker + }(), + ip: ip, + expectedUpdatedAndDesired: false, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) + return tracker + }(), + }, + { + name: "disconnected newer IP of tracked node", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker }(), - ip: newerIP, - expectedUpdated: true, + ip: newerIP, + expectedUpdatedAndDesired: true, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) - tracker.mostRecentTrackedIPs[newerIP.NodeID] = newerIP + + tracker.tracked[newerIP.NodeID].ip = newerIP + tracker.bloomAdditions[newerIP.NodeID] = 2 + return tracker + }(), + }, + { + name: "disconnected newer IP of untracked node", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) + return tracker + }(), + ip: newerIP, + expectedUpdatedAndDesired: false, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) + + tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 return tracker }(), }, { - name: "connected newer IP", + name: "connected newer IP of tracked node", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker }(), - ip: newerIP, - expectedUpdated: true, + ip: newerIP, + expectedUpdatedAndDesired: true, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) - tracker.mostRecentTrackedIPs[newerIP.NodeID] = newerIP + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + + tracker.numGossipableIPs.Dec() + tracker.tracked[newerIP.NodeID].ip = newerIP + tracker.bloomAdditions[newerIP.NodeID] = 2 + + subnet := tracker.subnet[constants.PrimaryNetworkID] + delete(subnet.gossipableIndices, newerIP.NodeID) + subnet.gossipableIPs = subnet.gossipableIPs[:0] + return tracker + }(), + }, + { + name: "connected newer IP of untracked node", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + return tracker + }(), + ip: newerIP, + expectedUpdatedAndDesired: false, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + + tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 - delete(tracker.gossipableIndices, newerIP.NodeID) - tracker.gossipableIPs = tracker.gossipableIPs[:0] + + subnet := tracker.subnet[subnetID] + delete(subnet.gossipableIndices, newerIP.NodeID) + subnet.gossipableIPs = subnet.gossipableIPs[:0] return tracker }(), }, @@ -331,7 +686,7 @@ func TestIPTracker_AddIP(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { updated := test.initialState.AddIP(test.ip) - require.Equal(t, test.expectedUpdated, updated) + require.Equal(t, test.expectedUpdatedAndDesired, updated) requireEqual(t, test.expectedState, test.initialState) requireMetricsConsistent(t, test.initialState) }) @@ -339,119 +694,235 @@ func TestIPTracker_AddIP(t *testing.T) { } func TestIPTracker_Connected(t *testing.T) { + subnetID := ids.GenerateTestID() newerIP := newerTestIP(ip) tests := []struct { - name string - initialState *ipTracker - ip *ips.ClaimedIPPort - expectedState *ipTracker + name string + initialState *ipTracker + ip *ips.ClaimedIPPort + trackedSubnets set.Set[ids.ID] + expectedState *ipTracker }{ { - name: "non-validator", - initialState: newTestIPTracker(t), - ip: ip, + name: "non-validator", + initialState: newTestIPTracker(t), + ip: ip, + trackedSubnets: set.Of(constants.PrimaryNetworkID), expectedState: func() *ipTracker { tracker := newTestIPTracker(t) + tracker.connected[ip.NodeID] = &connectedNode{ - ip: ip, + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, } return tracker }(), }, { - name: "first known IP", + name: "first known IP of node tracking subnet", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker }(), - ip: ip, + ip: ip, + trackedSubnets: set.Of(constants.PrimaryNetworkID), expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.mostRecentTrackedIPs[ip.NodeID] = ip + + tracker.numGossipableIPs.Inc() + tracker.tracked[ip.NodeID].ip = ip tracker.bloomAdditions[ip.NodeID] = 1 tracker.connected[ip.NodeID] = &connectedNode{ - ip: ip, + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, } - tracker.gossipableIndices[ip.NodeID] = 0 - tracker.gossipableIPs = []*ips.ClaimedIPPort{ + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIndices[ip.NodeID] = 0 + subnet.gossipableIPs = []*ips.ClaimedIPPort{ ip, } return tracker }(), }, { - name: "connected with older IP", + name: "first known IP of node not tracking subnet", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + return tracker + }(), + ip: ip, + trackedSubnets: set.Of(constants.PrimaryNetworkID), + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + + tracker.tracked[ip.NodeID].ip = ip + tracker.bloomAdditions[ip.NodeID] = 1 + tracker.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } + return tracker + }(), + }, + { + name: "connected with older IP of node tracking subnet", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) return tracker }(), - ip: ip, + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) + tracker.connected[ip.NodeID] = &connectedNode{ - ip: ip, + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, } return tracker }(), }, { - name: "connected with newer IP", + name: "connected with older IP of node not tracking subnet", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(newerIP)) + return tracker + }(), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(newerIP)) + + tracker.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } + return tracker + }(), + }, + { + name: "connected with newer IP of node tracking subnet", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker }(), - ip: newerIP, + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: newerIP, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) - tracker.mostRecentTrackedIPs[newerIP.NodeID] = newerIP + + tracker.numGossipableIPs.Inc() + tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 tracker.connected[newerIP.NodeID] = &connectedNode{ - ip: newerIP, + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: newerIP, } - tracker.gossipableIndices[newerIP.NodeID] = 0 - tracker.gossipableIPs = []*ips.ClaimedIPPort{ + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIndices[newerIP.NodeID] = 0 + subnet.gossipableIPs = []*ips.ClaimedIPPort{ newerIP, } return tracker }(), }, { - name: "connected with same IP", + name: "connected with newer IP of node not tracking subnet", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) + return tracker + }(), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: newerIP, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) + + tracker.tracked[newerIP.NodeID].ip = newerIP + tracker.bloomAdditions[newerIP.NodeID] = 2 + tracker.connected[newerIP.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: newerIP, + } + return tracker + }(), + }, + { + name: "connected with same IP of node tracking subnet", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker }(), - ip: ip, + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) + + tracker.numGossipableIPs.Inc() tracker.connected[ip.NodeID] = &connectedNode{ - ip: ip, + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, } - tracker.gossipableIndices[ip.NodeID] = 0 - tracker.gossipableIPs = []*ips.ClaimedIPPort{ + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIndices[ip.NodeID] = 0 + subnet.gossipableIPs = []*ips.ClaimedIPPort{ ip, } return tracker }(), }, + { + name: "connected with same IP of node not tracking subnet", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) + return tracker + }(), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + require.False(t, tracker.AddIP(ip)) + + tracker.connected[ip.NodeID] = &connectedNode{ + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } + return tracker + }(), + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.Connected(test.ip, nil) + test.initialState.Connected(test.ip, test.trackedSubnets) requireEqual(t, test.expectedState, test.initialState) requireMetricsConsistent(t, test.initialState) }) @@ -459,6 +930,7 @@ func TestIPTracker_Connected(t *testing.T) { } func TestIPTracker_Disconnected(t *testing.T) { + subnetID := ids.GenerateTestID() tests := []struct { name string initialState *ipTracker @@ -466,75 +938,95 @@ func TestIPTracker_Disconnected(t *testing.T) { expectedState *ipTracker }{ { - name: "not tracked", + name: "not gossipable", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker }(), nodeID: ip.NodeID, expectedState: newTestIPTracker(t), }, { - name: "not gossipable", + name: "latest gossipable", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) - tracker.ManuallyTrack(ip.NodeID) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) - tracker.ManuallyTrack(ip.NodeID) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + + tracker.numGossipableIPs.Dec() delete(tracker.connected, ip.NodeID) + + subnet := tracker.subnet[constants.PrimaryNetworkID] + delete(subnet.gossipableIndices, ip.NodeID) + subnet.gossipableIPs = subnet.gossipableIPs[:0] return tracker }(), }, { - name: "latest gossipable", + name: "non-latest gossipable", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) + tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) + tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) + + tracker.numGossipableIPs.Dec() delete(tracker.connected, ip.NodeID) - delete(tracker.gossipableIndices, ip.NodeID) - tracker.gossipableIPs = tracker.gossipableIPs[:0] + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIndices = map[ids.NodeID]int{ + otherIP.NodeID: 0, + } + subnet.gossipableIPs = []*ips.ClaimedIPPort{ + otherIP, + } return tracker }(), }, { - name: "non-latest gossipable", + name: "remove multiple gossipable IPs", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP, nil) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID, subnetID)) return tracker }(), nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP, nil) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID, subnetID)) + + tracker.numGossipableIPs.Add(-2) delete(tracker.connected, ip.NodeID) - tracker.gossipableIndices = map[ids.NodeID]int{ - otherIP.NodeID: 0, - } - tracker.gossipableIPs = []*ips.ClaimedIPPort{ - otherIP, - } + + primarySubnet := tracker.subnet[constants.PrimaryNetworkID] + delete(primarySubnet.gossipableIndices, ip.NodeID) + primarySubnet.gossipableIPs = primarySubnet.gossipableIPs[:0] + + subnet := tracker.subnet[subnetID] + delete(subnet.gossipableIndices, ip.NodeID) + subnet.gossipableIPs = subnet.gossipableIPs[:0] return tracker }(), }, @@ -550,10 +1042,11 @@ func TestIPTracker_Disconnected(t *testing.T) { func TestIPTracker_OnValidatorAdded(t *testing.T) { newerIP := newerTestIP(ip) - + subnetID := ids.GenerateTestID() tests := []struct { name string initialState *ipTracker + subnetID ids.ID nodeID ids.NodeID expectedState *ipTracker }{ @@ -564,11 +1057,50 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { tracker.ManuallyTrack(ip.NodeID) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.ManuallyTrack(ip.NodeID) + + tracker.tracked[ip.NodeID].subnets.Add(constants.PrimaryNetworkID) + tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), + } + return tracker + }(), + }, + { + name: "manually tracked and connected", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.ManuallyTrack(ip.NodeID) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + return tracker + }(), + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.gossipableIDs.Add(ip.NodeID) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + + tracker.numGossipableIPs.Inc() + tracker.tracked[ip.NodeID].subnets.Add(constants.PrimaryNetworkID) + tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: map[ids.NodeID]int{ + ip.NodeID: 0, + }, + gossipableIPs: []*ips.ClaimedIPPort{ + ip, + }, + } return tracker }(), }, @@ -577,17 +1109,25 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) require.True(t, tracker.AddIP(newerIP)) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) require.True(t, tracker.AddIP(newerIP)) - tracker.gossipableIDs.Add(ip.NodeID) + + tracker.tracked[ip.NodeID].subnets.Add(constants.PrimaryNetworkID) + tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), + } return tracker }(), }, @@ -598,7 +1138,8 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) @@ -608,11 +1149,20 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { { name: "disconnected", initialState: newTestIPTracker(t), + subnetID: constants.PrimaryNetworkID, nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.trackedIDs.Add(ip.NodeID) - tracker.gossipableIDs.Add(ip.NodeID) + + tracker.tracked[ip.NodeID] = &trackedNode{ + subnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + } + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), + } return tracker }(), }, @@ -620,28 +1170,66 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { name: "connected", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) - tracker.mostRecentTrackedIPs[ip.NodeID] = ip - tracker.trackedIDs.Add(ip.NodeID) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + + tracker.numGossipableIPs.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + subnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, + } tracker.bloomAdditions[ip.NodeID] = 1 - tracker.gossipableIndices[ip.NodeID] = 0 - tracker.gossipableIPs = []*ips.ClaimedIPPort{ - ip, + tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: map[ids.NodeID]int{ + ip.NodeID: 0, + }, + gossipableIPs: []*ips.ClaimedIPPort{ + ip, + }, + } + return tracker + }(), + }, + { + name: "connected to other subnet", + initialState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + return tracker + }(), + subnetID: subnetID, + nodeID: ip.NodeID, + expectedState: func() *ipTracker { + tracker := newTestIPTracker(t) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + + tracker.numTrackedSubnets.Inc() + tracker.tracked[ip.NodeID] = &trackedNode{ + subnets: set.Of(subnetID), + ip: ip, + } + tracker.bloomAdditions[ip.NodeID] = 1 + tracker.subnet[subnetID] = &gossipableSubnet{ + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: make(map[ids.NodeID]int), } - tracker.gossipableIDs.Add(ip.NodeID) return tracker }(), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.OnValidatorAdded(constants.PrimaryNetworkID, test.nodeID, nil, ids.Empty, 0) + test.initialState.OnValidatorAdded(test.subnetID, test.nodeID, nil, ids.Empty, 0) requireEqual(t, test.expectedState, test.initialState) requireMetricsConsistent(t, test.initialState) }) @@ -649,107 +1237,128 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { } func TestIPTracker_OnValidatorRemoved(t *testing.T) { + subnetID := ids.GenerateTestID() tests := []struct { name string initialState *ipTracker + subnetID ids.ID nodeID ids.NodeID expectedState *ipTracker }{ { - name: "manually tracked not gossipable", + name: "remove last validator of subnet", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.ManuallyTrack(ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.ManuallyTrack(ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) - tracker.gossipableIDs.Remove(ip.NodeID) + + tracker.numTrackedPeers.Dec() + tracker.numTrackedSubnets.Dec() + delete(tracker.tracked, ip.NodeID) + delete(tracker.subnet, constants.PrimaryNetworkID) return tracker }(), }, { - name: "manually tracked latest gossipable", + name: "manually tracked not gossipable", initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) + require.True(t, tracker.AddIP(ip)) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) - delete(tracker.gossipableIndices, ip.NodeID) - tracker.gossipableIPs = tracker.gossipableIPs[:0] - tracker.gossipableIDs.Remove(ip.NodeID) + require.True(t, tracker.AddIP(ip)) + + tracker.numTrackedSubnets.Dec() + + node := tracker.tracked[ip.NodeID] + node.subnets.Remove(constants.PrimaryNetworkID) + node.trackedSubnets.Remove(constants.PrimaryNetworkID) + + delete(tracker.subnet, constants.PrimaryNetworkID) return tracker }(), }, { - name: "manually gossiped", + name: "manually tracked latest gossipable", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) + tracker.ManuallyTrack(ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) + tracker.ManuallyTrack(ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + + tracker.numGossipableIPs.Dec() + tracker.numTrackedSubnets.Dec() + + node := tracker.tracked[ip.NodeID] + node.subnets.Remove(constants.PrimaryNetworkID) + node.trackedSubnets.Remove(constants.PrimaryNetworkID) + + delete(tracker.subnet, constants.PrimaryNetworkID) return tracker }(), }, { - name: "not gossipable", + name: "manually gossiped", initialState: func() *ipTracker { tracker := newTestIPTracker(t) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) - delete(tracker.mostRecentTrackedIPs, ip.NodeID) - tracker.trackedIDs.Remove(ip.NodeID) - tracker.gossipableIDs.Remove(ip.NodeID) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker }(), }, { - name: "latest gossipable", + name: "manually gossiped on other subnet", initialState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker }(), - nodeID: ip.NodeID, + subnetID: subnetID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) - delete(tracker.mostRecentTrackedIPs, ip.NodeID) - tracker.trackedIDs.Remove(ip.NodeID) - delete(tracker.gossipableIndices, ip.NodeID) - tracker.gossipableIPs = tracker.gossipableIPs[:0] - tracker.gossipableIDs.Remove(ip.NodeID) + tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) + tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) + + tracker.numTrackedSubnets.Dec() + tracker.tracked[ip.NodeID].subnets.Remove(subnetID) + delete(tracker.subnet, subnetID) return tracker }(), }, @@ -758,88 +1367,45 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { initialState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP, nil) + tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) return tracker }(), - nodeID: ip.NodeID, + subnetID: constants.PrimaryNetworkID, + nodeID: ip.NodeID, expectedState: func() *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP, nil) - delete(tracker.mostRecentTrackedIPs, ip.NodeID) - tracker.trackedIDs.Remove(ip.NodeID) - tracker.gossipableIndices = map[ids.NodeID]int{ + tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) + + tracker.numTrackedPeers.Dec() + tracker.numGossipableIPs.Dec() + delete(tracker.tracked, ip.NodeID) + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIDs.Remove(ip.NodeID) + subnet.gossipableIndices = map[ids.NodeID]int{ otherIP.NodeID: 0, } - tracker.gossipableIPs = []*ips.ClaimedIPPort{ + subnet.gossipableIPs = []*ips.ClaimedIPPort{ otherIP, } - tracker.gossipableIDs.Remove(ip.NodeID) return tracker }(), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.OnValidatorRemoved(constants.PrimaryNetworkID, test.nodeID, 0) + test.initialState.OnValidatorRemoved(test.subnetID, test.nodeID, 0) requireEqual(t, test.expectedState, test.initialState) requireMetricsConsistent(t, test.initialState) }) } } -func TestIPTracker_GetGossipableIPs(t *testing.T) { - require := require.New(t) - - tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) - tracker.Connected(otherIP, nil) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - - gossipableIPs := tracker.GetGossipableIPs(nil, ids.EmptyNodeID, bloom.EmptyFilter, nil, 2) - require.ElementsMatch([]*ips.ClaimedIPPort{ip, otherIP}, gossipableIPs) - - gossipableIPs = tracker.GetGossipableIPs(nil, ip.NodeID, bloom.EmptyFilter, nil, 2) - require.Equal([]*ips.ClaimedIPPort{otherIP}, gossipableIPs) - - gossipableIPs = tracker.GetGossipableIPs(nil, ids.EmptyNodeID, bloom.FullFilter, nil, 2) - require.Empty(gossipableIPs) - - filter, err := bloom.New(8, 1024) - require.NoError(err) - bloom.Add(filter, ip.GossipID[:], nil) - - readFilter, err := bloom.Parse(filter.Marshal()) - require.NoError(err) - - gossipableIPs = tracker.GetGossipableIPs(nil, ip.NodeID, readFilter, nil, 2) - require.Equal([]*ips.ClaimedIPPort{otherIP}, gossipableIPs) -} - -func TestIPTracker_BloomFiltersEverything(t *testing.T) { - require := require.New(t) - - tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) - tracker.Connected(otherIP, nil) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - - bloomBytes, salt := tracker.Bloom() - readFilter, err := bloom.Parse(bloomBytes) - require.NoError(err) - - gossipableIPs := tracker.GetGossipableIPs(nil, ids.EmptyNodeID, readFilter, salt, 2) - require.Empty(gossipableIPs) - - require.NoError(tracker.ResetBloom()) -} - func TestIPTracker_BloomGrows(t *testing.T) { tests := []struct { name string @@ -857,6 +1423,12 @@ func TestIPTracker_BloomGrows(t *testing.T) { tracker.ManuallyTrack(ids.GenerateTestNodeID()) }, }, + { + name: "Manually Gossip", + add: func(tracker *ipTracker) { + tracker.ManuallyGossip(ids.GenerateTestID(), ids.GenerateTestNodeID()) + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -880,11 +1452,12 @@ func TestIPTracker_BloomResetsDynamically(t *testing.T) { require := require.New(t) tracker := newTestIPTracker(t) - tracker.Connected(ip, nil) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.OnValidatorRemoved(constants.PrimaryNetworkID, ip.NodeID, 0) + tracker.maxBloomCount = 1 - tracker.Connected(otherIP, nil) + tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) requireMetricsConsistent(t, tracker) @@ -911,18 +1484,133 @@ func TestIPTracker_PreventBloomFilterAddition(t *testing.T) { requireMetricsConsistent(t, tracker) } -func TestIPTracker_ShouldVerifyIP(t *testing.T) { - require := require.New(t) - - newerIP := newerTestIP(ip) +func TestIPTracker_GetGossipableIPs(t *testing.T) { + subnetIDA := ids.GenerateTestID() + subnetIDB := ids.GenerateTestID() + unknownSubnetID := ids.GenerateTestID() tracker := newTestIPTracker(t) - require.False(tracker.ShouldVerifyIP(ip)) - tracker.ManuallyTrack(ip.NodeID) - require.True(tracker.ShouldVerifyIP(ip)) - tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) - require.True(tracker.ShouldVerifyIP(ip)) - require.True(tracker.AddIP(ip)) - require.False(tracker.ShouldVerifyIP(ip)) - require.True(tracker.ShouldVerifyIP(newerIP)) + tracker.Connected(ip, set.Of(constants.PrimaryNetworkID, subnetIDA)) + tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID, subnetIDA, subnetIDB)) + tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(subnetIDA, otherIP.NodeID, nil, ids.Empty, 0) + tracker.OnValidatorAdded(subnetIDB, otherIP.NodeID, nil, ids.Empty, 0) + + myFilterBytes, mySalt := tracker.Bloom() + myFilter, err := bloom.Parse(myFilterBytes) + require.NoError(t, err) + + tests := []struct { + name string + toIterate set.Set[ids.ID] + allowed set.Set[ids.ID] + nodeID ids.NodeID + filter *bloom.ReadFilter + salt []byte + expected []*ips.ClaimedIPPort + }{ + { + name: "fetch both subnets IPs", + toIterate: set.Of(constants.PrimaryNetworkID, subnetIDA), + allowed: set.Of(constants.PrimaryNetworkID, subnetIDA), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{ip, otherIP}, + }, + { + name: "filter nodeID", + toIterate: set.Of(constants.PrimaryNetworkID, subnetIDA), + allowed: set.Of(constants.PrimaryNetworkID, subnetIDA), + nodeID: ip.NodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{otherIP}, + }, + { + name: "filter duplicate nodeIDs", + toIterate: set.Of(subnetIDA, subnetIDB), + allowed: set.Of(subnetIDA, subnetIDB), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{otherIP}, + }, + { + name: "filter known IPs", + toIterate: set.Of(constants.PrimaryNetworkID, subnetIDA), + allowed: set.Of(constants.PrimaryNetworkID, subnetIDA), + nodeID: ids.EmptyNodeID, + filter: func() *bloom.ReadFilter { + filter, err := bloom.New(8, 1024) + require.NoError(t, err) + bloom.Add(filter, ip.GossipID[:], nil) + + readFilter, err := bloom.Parse(filter.Marshal()) + require.NoError(t, err) + return readFilter + }(), + salt: nil, + expected: []*ips.ClaimedIPPort{otherIP}, + }, + { + name: "filter everything", + toIterate: set.Of(constants.PrimaryNetworkID, subnetIDA, subnetIDB), + allowed: set.Of(constants.PrimaryNetworkID, subnetIDA, subnetIDB), + nodeID: ids.EmptyNodeID, + filter: myFilter, + salt: mySalt, + expected: nil, + }, + { + name: "only fetch primary network IPs", + toIterate: set.Of(constants.PrimaryNetworkID), + allowed: set.Of(constants.PrimaryNetworkID), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{ip}, + }, + { + name: "only fetch subnet IPs", + toIterate: set.Of(subnetIDA), + allowed: set.Of(subnetIDA), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{otherIP}, + }, + { + name: "filter subnet", + toIterate: set.Of(constants.PrimaryNetworkID, subnetIDA), + allowed: set.Of(constants.PrimaryNetworkID), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: []*ips.ClaimedIPPort{ip}, + }, + { + name: "skip unknown subnet", + toIterate: set.Of(unknownSubnetID), + allowed: set.Of(unknownSubnetID), + nodeID: ids.EmptyNodeID, + filter: bloom.EmptyFilter, + salt: nil, + expected: nil, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gossipableIPs := getGossipableIPs( + tracker, + test.toIterate, + test.allowed.Contains, + test.nodeID, + test.filter, + test.salt, + 2, + ) + require.ElementsMatch(t, test.expected, gossipableIPs) + }) + } } diff --git a/network/network.go b/network/network.go index c6d7681b2710..f285ace9e17b 100644 --- a/network/network.go +++ b/network/network.go @@ -482,8 +482,9 @@ func (n *network) AllowConnection(nodeID ids.NodeID) bool { } func (n *network) Track(claimedIPPorts []*ips.ClaimedIPPort) error { + iAmAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, n.config.MyNodeID) != 0 for _, ip := range claimedIPPorts { - if err := n.track(ip); err != nil { + if err := n.track(ip, iAmAValidator); err != nil { return err } } @@ -676,7 +677,7 @@ func (n *network) ManuallyTrack(nodeID ids.NodeID, ip ips.IPPort) { } } -func (n *network) track(ip *ips.ClaimedIPPort) error { +func (n *network) track(ip *ips.ClaimedIPPort, trackAllSubnets bool) error { // To avoid signature verification when the IP isn't needed, we // optimistically filter out IPs. This can result in us not tracking an IP // that we otherwise would have. This case can only happen if the node @@ -685,7 +686,7 @@ func (n *network) track(ip *ips.ClaimedIPPort) error { // // Note: Avoiding signature verification when the IP isn't needed is a // **significant** performance optimization. - if !n.ipTracker.ShouldVerifyIP(ip) { + if !n.ipTracker.ShouldVerifyIP(ip, trackAllSubnets) { n.metrics.numUselessPeerListBytes.Add(float64(ip.Size())) return nil } From 5b47b114b02eca9120a47e5cae4cd706d2cbe309 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Wed, 15 May 2024 23:52:33 -0400 Subject: [PATCH 36/62] lint --- network/ip_tracker_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index 4a99dd3dfc8b..88f83b3df3c9 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -18,9 +18,9 @@ import ( "github.com/ava-labs/avalanchego/utils/set" ) -func newTestIPTracker(t *testing.T, trackedSubnets ...ids.ID) *ipTracker { +func newTestIPTracker(t *testing.T) *ipTracker { tracker, err := newIPTracker( - set.Of(trackedSubnets...), + nil, logging.NoLog{}, "", prometheus.NewRegistry(), From 80ca7533575c5bdc42182f32d202cd0de2eb8fa8 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Thu, 16 May 2024 00:01:56 -0400 Subject: [PATCH 37/62] comment --- network/network.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/network/network.go b/network/network.go index f285ace9e17b..462becf3ca30 100644 --- a/network/network.go +++ b/network/network.go @@ -541,6 +541,7 @@ func (n *network) Peers( ) []*ips.ClaimedIPPort { areWeAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, n.config.MyNodeID) != 0 if !areWeAValidator { + // Return IPs for subnets both sides track. return getGossipableIPs( n.ipTracker, trackedSubnets, @@ -556,6 +557,7 @@ func (n *network) Peers( areTheyAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, peerID) != 0 if requestAllPeers && areTheyAValidator { + // Return IPs for all subnets. return getGossipableIPs( n.ipTracker, n.ipTracker.subnet, @@ -567,6 +569,7 @@ func (n *network) Peers( ) } + // Return IPs for all the subnets that the peer is tracking. return getGossipableIPs( n.ipTracker, trackedSubnets, From a783f03e707d83d713f9a21b0fdaed179dbaa17a Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Thu, 16 May 2024 00:05:04 -0400 Subject: [PATCH 38/62] nit --- network/ip_tracker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index c06be78485c1..b51551dc54ab 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -204,7 +204,7 @@ type ipTracker struct { trackedSubnets set.Set[ids.ID] log logging.Logger numTrackedPeers prometheus.Gauge - numGossipableIPs prometheus.Gauge + numGossipableIPs prometheus.Gauge // IPs are not deduplicated across subnets numTrackedSubnets prometheus.Gauge bloomMetrics *bloom.Metrics From 9e1ad1a7ac0622797ebcb0bcee7b5319ad649cf9 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Thu, 16 May 2024 00:07:59 -0400 Subject: [PATCH 39/62] nit --- network/ip_tracker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index b51551dc54ab..163546ec16dc 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -405,6 +405,7 @@ func (i *ipTracker) OnValidatorAdded(subnetID ids.ID, nodeID ids.NodeID, _ *bls. i.addGossipableID(nodeID, subnetID, false) } +// If [subnetID] is nil, the nodeID is being manually tracked. func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { nodeTracker, previouslyTracked := i.tracked[nodeID] if !previouslyTracked { From 20338ab1b319d3fd62a87a4eb28803f4d1979683 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Thu, 16 May 2024 00:23:09 -0400 Subject: [PATCH 40/62] update readme --- network/README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/network/README.md b/network/README.md index b4a6a01d648b..eecdd9bc52f6 100644 --- a/network/README.md +++ b/network/README.md @@ -55,7 +55,7 @@ A peer will then read the full message and attempt to parse it into either a net Upon connection to a new peer, a handshake is performed between the node attempting to establish the outbound connection to the peer and the peer receiving the inbound connection. -When attempting to establish the connection, the first message that the node sends is a `Handshake` message describing the compatibility of the nodes. If the `Handshake` message is successfully received and the peer decides that it wants a connection with this node, it replies with a `PeerList` message that contains metadata about other peers that allows a node to connect to them. See [Peerlist Gossip](#peerlist-gossip). +When attempting to establish the connection, the first message that the node sends is a `Handshake` message describing configuration of the node. If the `Handshake` message is successfully received and the peer decides that it will allow a connection with this node, it replies with a `PeerList` message that contains metadata about other peers that allows a node to connect to them. See [PeerList Gossip](#peerlist-gossip). As an example, nodes that are attempting to connect with an incompatible version of AvalancheGo or a significantly skewed local clock are rejected. @@ -127,10 +127,12 @@ It is expected for Avalanche nodes to allow inbound connections. If a validator Avalanche nodes that have identified the `IP:Port` pair of a node they want to connect to will initiate outbound connections to this `IP:Port` pair. If the connection is not able to complete the [Peer Handshake](#peer-handshake), the connection will be re-attempted with an [Exponential Backoff](https://en.wikipedia.org/wiki/Exponential_backoff). -A node should initiate outbound connections to an `IP:Port` pair that is believed to belong to a node that is not connected and meets at least one of the following conditions: -- The node is in the initial bootstrapper set. -- The node is in the default bootstrapper set. -- The node in the current Primary Network validator set. +A node should initiate outbound connections to an `IP:Port` pair that is believed to belong to another node that is not connected and meets at least one of the following conditions: +- The peer is in the initial bootstrapper set. +- The peer is in the default bootstrapper set. +- The peer is a Primary Network validator. +- The peer is a validator of a tracked Subnet. +- The peer is a validator of a Subnet and the local node is a Primary Network validator. #### IP Authentication @@ -148,7 +150,7 @@ Once connected to an initial set of peers, a node can use these connections to d Peers are discovered by receiving [`PeerList`](#peerlist) messages during the [Peer Handshake](#peer-handshake). These messages quickly provide a node with knowledge of peers in the network. However, they offer no guarantee that the node will connect to and maintain connections with every peer in the network. -To provide an eventual guarantee that all peers learn of one another, nodes periodically send a [`GetPeerList`](#getpeerlist) message to a randomly selected validator with the node's current [Bloom Filter](#bloom-filter) and `Salt`. +To provide an eventual guarantee that all peers learn of one another, nodes periodically send a [`GetPeerList`](#getpeerlist) message to a randomly selected Primary Network validator with the node's current [Bloom Filter](#bloom-filter) and `Salt`. #### Bloom Filter @@ -172,7 +174,7 @@ A `GetPeerList` message contains the Bloom Filter of the currently known peers a - The Bloom Filter sent when requesting the `PeerList` message does not contain the node claiming the `IP:Port` pair. - The node claiming the `IP:Port` pair is currently connected. - The `IP:Port` pair the node shared during the `Handshake` message is the node's most recently known `IP:Port` pair. -- The node claiming the `IP:Port` pair is either in the default bootstrapper set or is a current Primary Network validator. +- The node claiming the `IP:Port` pair is either in the default bootstrapper set, is a current Primary Network validator, is a validator of a tracked Subnet, or is a validator of a Subnet and the peer is a Primary Network validator. #### Example PeerList Gossip From 05465233ffea2a56b4d119f2fd5589ca33387ef3 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Fri, 17 May 2024 19:55:49 -0400 Subject: [PATCH 41/62] 1 less call --- network/network.go | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/network/network.go b/network/network.go index 462becf3ca30..8f6c7139a865 100644 --- a/network/network.go +++ b/network/network.go @@ -540,40 +540,34 @@ func (n *network) Peers( salt []byte, ) []*ips.ClaimedIPPort { areWeAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, n.config.MyNodeID) != 0 - if !areWeAValidator { - // Return IPs for subnets both sides track. - return getGossipableIPs( - n.ipTracker, - trackedSubnets, - func(subnetID ids.ID) bool { - return subnetID == constants.PrimaryNetworkID || n.ipTracker.trackedSubnets.Contains(subnetID) - }, - peerID, - knownPeers, - salt, - int(n.config.PeerListNumValidatorIPs), - ) + + // Only return IPs for subnets that we are tracking. + var allowedSubnets func(ids.ID) bool + if areWeAValidator { + allowedSubnets = func(ids.ID) bool { return true } + } else { + allowedSubnets = func(subnetID ids.ID) bool { + return subnetID == constants.PrimaryNetworkID || n.ipTracker.trackedSubnets.Contains(subnetID) + } } areTheyAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, peerID) != 0 - if requestAllPeers && areTheyAValidator { + if areWeAValidator && requestAllPeers && areTheyAValidator { // Return IPs for all subnets. return getGossipableIPs( n.ipTracker, n.ipTracker.subnet, - func(ids.ID) bool { return true }, + allowedSubnets, peerID, knownPeers, salt, int(n.config.PeerListNumValidatorIPs), ) } - - // Return IPs for all the subnets that the peer is tracking. return getGossipableIPs( n.ipTracker, trackedSubnets, - func(ids.ID) bool { return true }, + allowedSubnets, peerID, knownPeers, salt, From a6e3bb9558315b7bd0164c3c4c58dbe0af2dd45b Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 20 May 2024 16:40:42 -0400 Subject: [PATCH 42/62] Remove testing duplication --- network/ip_tracker_test.go | 833 +++++++++++++------------------------ 1 file changed, 297 insertions(+), 536 deletions(-) diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index 88f83b3df3c9..b6206e4a11f4 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -64,120 +64,93 @@ func requireMetricsConsistent(t *testing.T, tracker *ipTracker) { func TestIPTracker_ManuallyTrack(t *testing.T) { subnetID := ids.GenerateTestID() tests := []struct { - name string - initialState *ipTracker - nodeID ids.NodeID - expectedState *ipTracker + name string + initialState func(t *testing.T) *ipTracker + expectedChange func(*ipTracker) }{ { name: "non-connected non-validator", - initialState: newTestIPTracker(t), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - + initialState: newTestIPTracker, + expectedChange: func(tracker *ipTracker) { tracker.numTrackedPeers.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ manuallyTracked: true, } - return tracker - }(), + }, }, { name: "connected non-validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - + }, + expectedChange: func(tracker *ipTracker) { tracker.numTrackedPeers.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ manuallyTracked: true, ip: ip, } tracker.bloomAdditions[ip.NodeID] = 1 - return tracker - }(), + }, }, { name: "non-connected tracked validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - + }, + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID].manuallyTracked = true - return tracker - }(), + }, }, { name: "non-connected untracked validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - + }, + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID].manuallyTracked = true - return tracker - }(), + }, }, { name: "connected tracked validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - + }, + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID].manuallyTracked = true - return tracker - }(), + }, }, { name: "connected untracked validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - + }, + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID].manuallyTracked = true - return tracker - }(), + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.ManuallyTrack(test.nodeID) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + testState.ManuallyTrack(ip.NodeID) + test.expectedChange(expectedState) + + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } @@ -185,20 +158,16 @@ func TestIPTracker_ManuallyTrack(t *testing.T) { func TestIPTracker_ManuallyGossip(t *testing.T) { subnetID := ids.GenerateTestID() tests := []struct { - name string - initialState *ipTracker - subnetID ids.ID - nodeID ids.NodeID - expectedState *ipTracker + name string + initialState func(t *testing.T) *ipTracker + subnetID ids.ID + expectedChange func(*ipTracker) }{ { name: "non-connected tracked non-validator", - initialState: newTestIPTracker(t), + initialState: newTestIPTracker, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - + expectedChange: func(tracker *ipTracker) { tracker.numTrackedPeers.Inc() tracker.numTrackedSubnets.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ @@ -212,17 +181,13 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { gossipableIDs: set.Of(ip.NodeID), gossipableIndices: make(map[ids.NodeID]int), } - return tracker - }(), + }, }, { name: "non-connected untracked non-validator", - initialState: newTestIPTracker(t), + initialState: newTestIPTracker, subnetID: subnetID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - + expectedChange: func(tracker *ipTracker) { tracker.numTrackedPeers.Inc() tracker.numTrackedSubnets.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ @@ -234,22 +199,17 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { gossipableIDs: set.Of(ip.NodeID), gossipableIndices: make(map[ids.NodeID]int), } - return tracker - }(), + }, }, { name: "connected tracked non-validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - + expectedChange: func(tracker *ipTracker) { tracker.numTrackedPeers.Inc() tracker.numGossipableIPs.Inc() tracker.numTrackedSubnets.Inc() @@ -271,22 +231,17 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { ip, }, } - return tracker - }(), + }, }, { name: "connected untracked non-validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, subnetID: subnetID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - + expectedChange: func(tracker *ipTracker) { tracker.numTrackedPeers.Inc() tracker.numTrackedSubnets.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ @@ -300,89 +255,71 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { gossipableIDs: set.Of(ip.NodeID), gossipableIndices: make(map[ids.NodeID]int), } - return tracker - }(), + }, }, { name: "non-connected tracked validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID].manuallyTracked = true tracker.subnet[constants.PrimaryNetworkID].manuallyGossipable = set.Of(ip.NodeID) - return tracker - }(), + }, }, { name: "non-connected untracked validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, subnetID: subnetID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - + expectedChange: func(tracker *ipTracker) { tracker.subnet[subnetID].manuallyGossipable = set.Of(ip.NodeID) - return tracker - }(), + }, }, { name: "connected tracked validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID].manuallyTracked = true tracker.subnet[constants.PrimaryNetworkID].manuallyGossipable = set.Of(ip.NodeID) - return tracker - }(), + }, }, { name: "connected untracked validator", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, subnetID: subnetID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - + expectedChange: func(tracker *ipTracker) { tracker.subnet[subnetID].manuallyGossipable = set.Of(ip.NodeID) - return tracker - }(), + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.ManuallyGossip(test.subnetID, test.nodeID) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + testState.ManuallyGossip(test.subnetID, ip.NodeID) + test.expectedChange(expectedState) + + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } @@ -391,72 +328,72 @@ func TestIPTracker_ShouldVerifyIP(t *testing.T) { newerIP := newerTestIP(ip) tests := []struct { name string - tracker *ipTracker + tracker func(t *testing.T) *ipTracker ip *ips.ClaimedIPPort expectedTrackAllSubnets bool expectedTrackRequestedSubnets bool }{ { name: "node not tracked", - tracker: newTestIPTracker(t), + tracker: newTestIPTracker, ip: ip, expectedTrackAllSubnets: false, expectedTrackRequestedSubnets: false, }, { name: "undesired connection", - tracker: func() *ipTracker { + tracker: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(ids.GenerateTestID(), ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, ip: ip, expectedTrackAllSubnets: true, expectedTrackRequestedSubnets: false, }, { name: "desired connection first IP", - tracker: func() *ipTracker { + tracker: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, ip: ip, expectedTrackAllSubnets: true, expectedTrackRequestedSubnets: true, }, { name: "desired connection older IP", - tracker: func() *ipTracker { + tracker: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) return tracker - }(), + }, ip: ip, expectedTrackAllSubnets: false, expectedTrackRequestedSubnets: false, }, { name: "desired connection same IP", - tracker: func() *ipTracker { + tracker: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker - }(), + }, ip: ip, expectedTrackAllSubnets: false, expectedTrackRequestedSubnets: false, }, { name: "desired connection newer IP", - tracker: func() *ipTracker { + tracker: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker - }(), + }, ip: newerIP, expectedTrackAllSubnets: true, expectedTrackRequestedSubnets: true, @@ -466,8 +403,9 @@ func TestIPTracker_ShouldVerifyIP(t *testing.T) { t.Run(test.name, func(t *testing.T) { require := require.New(t) - require.Equal(test.expectedTrackAllSubnets, test.tracker.ShouldVerifyIP(test.ip, true)) - require.Equal(test.expectedTrackRequestedSubnets, test.tracker.ShouldVerifyIP(test.ip, false)) + tracker := test.tracker(t) + require.Equal(test.expectedTrackAllSubnets, tracker.ShouldVerifyIP(test.ip, true)) + require.Equal(test.expectedTrackRequestedSubnets, tracker.ShouldVerifyIP(test.ip, false)) }) } } @@ -477,177 +415,135 @@ func TestIPTracker_AddIP(t *testing.T) { newerIP := newerTestIP(ip) tests := []struct { name string - initialState *ipTracker + initialState func(t *testing.T) *ipTracker ip *ips.ClaimedIPPort + expectedChange func(*ipTracker) expectedUpdatedAndDesired bool - expectedState *ipTracker }{ { name: "non-validator", - initialState: newTestIPTracker(t), + initialState: newTestIPTracker, ip: ip, + expectedChange: func(*ipTracker) {}, expectedUpdatedAndDesired: false, - expectedState: newTestIPTracker(t), }, { name: "first known IP of tracked node", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - ip: ip, - expectedUpdatedAndDesired: true, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - + }, + ip: ip, + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID].ip = ip tracker.bloomAdditions[ip.NodeID] = 1 - return tracker - }(), + }, + expectedUpdatedAndDesired: true, }, { name: "first known IP of untracked node", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, ip: ip, expectedUpdatedAndDesired: false, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID].ip = ip tracker.bloomAdditions[ip.NodeID] = 1 - return tracker - }(), + }, }, { name: "older IP of tracked node", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) return tracker - }(), + }, ip: ip, expectedUpdatedAndDesired: false, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(newerIP)) - return tracker - }(), + expectedChange: func(*ipTracker) {}, }, { name: "older IP of untracked node", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) require.False(t, tracker.AddIP(newerIP)) return tracker - }(), + }, ip: ip, expectedUpdatedAndDesired: false, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - require.False(t, tracker.AddIP(newerIP)) - return tracker - }(), + expectedChange: func(*ipTracker) {}, }, { name: "same IP of tracked node", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker - }(), + }, ip: ip, expectedUpdatedAndDesired: false, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) - return tracker - }(), + expectedChange: func(*ipTracker) {}, }, { name: "same IP of untracked node", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) require.False(t, tracker.AddIP(ip)) return tracker - }(), + }, ip: ip, expectedUpdatedAndDesired: false, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - require.False(t, tracker.AddIP(ip)) - return tracker - }(), + expectedChange: func(*ipTracker) {}, }, { name: "disconnected newer IP of tracked node", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker - }(), + }, ip: newerIP, expectedUpdatedAndDesired: true, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) - + expectedChange: func(tracker *ipTracker) { tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 - return tracker - }(), + }, }, { name: "disconnected newer IP of untracked node", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) require.False(t, tracker.AddIP(ip)) return tracker - }(), + }, ip: newerIP, expectedUpdatedAndDesired: false, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - require.False(t, tracker.AddIP(ip)) - + expectedChange: func(tracker *ipTracker) { tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 - return tracker - }(), + }, }, { name: "connected newer IP of tracked node", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, ip: newerIP, expectedUpdatedAndDesired: true, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - + expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Dec() tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 @@ -655,40 +551,39 @@ func TestIPTracker_AddIP(t *testing.T) { subnet := tracker.subnet[constants.PrimaryNetworkID] delete(subnet.gossipableIndices, newerIP.NodeID) subnet.gossipableIPs = subnet.gossipableIPs[:0] - return tracker - }(), + }, }, { name: "connected newer IP of untracked node", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, ip: newerIP, expectedUpdatedAndDesired: false, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - + expectedChange: func(tracker *ipTracker) { tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 subnet := tracker.subnet[subnetID] delete(subnet.gossipableIndices, newerIP.NodeID) subnet.gossipableIPs = subnet.gossipableIPs[:0] - return tracker - }(), + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - updated := test.initialState.AddIP(test.ip) + testState := test.initialState(t) + expectedState := test.initialState(t) + + updated := testState.AddIP(test.ip) + test.expectedChange(expectedState) + require.Equal(t, test.expectedUpdatedAndDesired, updated) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } @@ -698,39 +593,30 @@ func TestIPTracker_Connected(t *testing.T) { newerIP := newerTestIP(ip) tests := []struct { name string - initialState *ipTracker + initialState func(t *testing.T) *ipTracker ip *ips.ClaimedIPPort - trackedSubnets set.Set[ids.ID] - expectedState *ipTracker + expectedChange func(*ipTracker) }{ { - name: "non-validator", - initialState: newTestIPTracker(t), - ip: ip, - trackedSubnets: set.Of(constants.PrimaryNetworkID), - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - + name: "non-validator", + initialState: newTestIPTracker, + ip: ip, + expectedChange: func(tracker *ipTracker) { tracker.connected[ip.NodeID] = &connectedNode{ trackedSubnets: set.Of(constants.PrimaryNetworkID), ip: ip, } - return tracker - }(), + }, }, { name: "first known IP of node tracking subnet", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - ip: ip, - trackedSubnets: set.Of(constants.PrimaryNetworkID), - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - + }, + ip: ip, + expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Inc() tracker.tracked[ip.NodeID].ip = ip tracker.bloomAdditions[ip.NodeID] = 1 @@ -744,90 +630,67 @@ func TestIPTracker_Connected(t *testing.T) { subnet.gossipableIPs = []*ips.ClaimedIPPort{ ip, } - return tracker - }(), + }, }, { name: "first known IP of node not tracking subnet", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), - ip: ip, - trackedSubnets: set.Of(constants.PrimaryNetworkID), - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - + }, + ip: ip, + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID].ip = ip tracker.bloomAdditions[ip.NodeID] = 1 tracker.connected[ip.NodeID] = &connectedNode{ trackedSubnets: set.Of(constants.PrimaryNetworkID), ip: ip, } - return tracker - }(), + }, }, { name: "connected with older IP of node tracking subnet", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(newerIP)) return tracker - }(), - trackedSubnets: set.Of(constants.PrimaryNetworkID), - ip: ip, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(newerIP)) - + }, + ip: ip, + expectedChange: func(tracker *ipTracker) { tracker.connected[ip.NodeID] = &connectedNode{ trackedSubnets: set.Of(constants.PrimaryNetworkID), ip: ip, } - return tracker - }(), + }, }, { name: "connected with older IP of node not tracking subnet", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) require.False(t, tracker.AddIP(newerIP)) return tracker - }(), - trackedSubnets: set.Of(constants.PrimaryNetworkID), - ip: ip, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - require.False(t, tracker.AddIP(newerIP)) - + }, + ip: ip, + expectedChange: func(tracker *ipTracker) { tracker.connected[ip.NodeID] = &connectedNode{ trackedSubnets: set.Of(constants.PrimaryNetworkID), ip: ip, } - return tracker - }(), + }, }, { name: "connected with newer IP of node tracking subnet", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker - }(), - trackedSubnets: set.Of(constants.PrimaryNetworkID), - ip: newerIP, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) - + }, + ip: newerIP, + expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Inc() tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 @@ -841,48 +704,36 @@ func TestIPTracker_Connected(t *testing.T) { subnet.gossipableIPs = []*ips.ClaimedIPPort{ newerIP, } - return tracker - }(), + }, }, { name: "connected with newer IP of node not tracking subnet", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) require.False(t, tracker.AddIP(ip)) return tracker - }(), - trackedSubnets: set.Of(constants.PrimaryNetworkID), - ip: newerIP, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - require.False(t, tracker.AddIP(ip)) - + }, + ip: newerIP, + expectedChange: func(tracker *ipTracker) { tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 tracker.connected[newerIP.NodeID] = &connectedNode{ trackedSubnets: set.Of(constants.PrimaryNetworkID), ip: newerIP, } - return tracker - }(), + }, }, { name: "connected with same IP of node tracking subnet", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker - }(), - trackedSubnets: set.Of(constants.PrimaryNetworkID), - ip: ip, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) - + }, + ip: ip, + expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Inc() tracker.connected[ip.NodeID] = &connectedNode{ trackedSubnets: set.Of(constants.PrimaryNetworkID), @@ -894,37 +745,35 @@ func TestIPTracker_Connected(t *testing.T) { subnet.gossipableIPs = []*ips.ClaimedIPPort{ ip, } - return tracker - }(), + }, }, { name: "connected with same IP of node not tracking subnet", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) require.False(t, tracker.AddIP(ip)) return tracker - }(), - trackedSubnets: set.Of(constants.PrimaryNetworkID), - ip: ip, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - require.False(t, tracker.AddIP(ip)) - + }, + ip: ip, + expectedChange: func(tracker *ipTracker) { tracker.connected[ip.NodeID] = &connectedNode{ trackedSubnets: set.Of(constants.PrimaryNetworkID), ip: ip, } - return tracker - }(), + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.Connected(test.ip, test.trackedSubnets) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + testState.Connected(test.ip, set.Of(constants.PrimaryNetworkID)) + test.expectedChange(expectedState) + + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } @@ -932,62 +781,47 @@ func TestIPTracker_Connected(t *testing.T) { func TestIPTracker_Disconnected(t *testing.T) { subnetID := ids.GenerateTestID() tests := []struct { - name string - initialState *ipTracker - nodeID ids.NodeID - expectedState *ipTracker + name string + initialState func(t *testing.T) *ipTracker + expectedChange func(*ipTracker) }{ { name: "not gossipable", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: newTestIPTracker(t), + }, + expectedChange: func(*ipTracker) {}, }, { name: "latest gossipable", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - + }, + expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Dec() delete(tracker.connected, ip.NodeID) subnet := tracker.subnet[constants.PrimaryNetworkID] delete(subnet.gossipableIndices, ip.NodeID) subnet.gossipableIPs = subnet.gossipableIPs[:0] - return tracker - }(), + }, }, { name: "non-latest gossipable", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) - + }, + expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Dec() delete(tracker.connected, ip.NodeID) @@ -998,25 +832,18 @@ func TestIPTracker_Disconnected(t *testing.T) { subnet.gossipableIPs = []*ips.ClaimedIPPort{ otherIP, } - return tracker - }(), + }, }, { name: "remove multiple gossipable IPs", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID, subnetID)) return tracker - }(), - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID, subnetID)) - + }, + expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Add(-2) delete(tracker.connected, ip.NodeID) @@ -1027,15 +854,19 @@ func TestIPTracker_Disconnected(t *testing.T) { subnet := tracker.subnet[subnetID] delete(subnet.gossipableIndices, ip.NodeID) subnet.gossipableIPs = subnet.gossipableIPs[:0] - return tracker - }(), + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.Disconnected(test.nodeID) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + testState.Disconnected(ip.NodeID) + expectedState.Disconnected(ip.NodeID) + + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } @@ -1044,25 +875,20 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { newerIP := newerTestIP(ip) subnetID := ids.GenerateTestID() tests := []struct { - name string - initialState *ipTracker - subnetID ids.ID - nodeID ids.NodeID - expectedState *ipTracker + name string + initialState func(t *testing.T) *ipTracker + subnetID ids.ID + expectedChange func(*ipTracker) }{ { name: "manually tracked", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) return tracker - }(), + }, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyTrack(ip.NodeID) - + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID].subnets.Add(constants.PrimaryNetworkID) tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ @@ -1070,24 +896,18 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { gossipableIDs: set.Of(ip.NodeID), gossipableIndices: make(map[ids.NodeID]int), } - return tracker - }(), + }, }, { name: "manually tracked and connected", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyTrack(ip.NodeID) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - + expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Inc() tracker.tracked[ip.NodeID].subnets.Add(constants.PrimaryNetworkID) tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) @@ -1101,26 +921,19 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { ip, }, } - return tracker - }(), + }, }, { name: "manually tracked and connected with older IP", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) require.True(t, tracker.AddIP(newerIP)) return tracker - }(), + }, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyTrack(ip.NodeID) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - require.True(t, tracker.AddIP(newerIP)) - + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID].subnets.Add(constants.PrimaryNetworkID) tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ @@ -1128,32 +941,23 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { gossipableIDs: set.Of(ip.NodeID), gossipableIndices: make(map[ids.NodeID]int), } - return tracker - }(), + }, }, { name: "manually gossiped", - initialState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) - return tracker - }(), - subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) return tracker - }(), + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(*ipTracker) {}, }, { name: "disconnected", - initialState: newTestIPTracker(t), + initialState: newTestIPTracker, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - + expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID] = &trackedNode{ subnets: set.Of(constants.PrimaryNetworkID), trackedSubnets: set.Of(constants.PrimaryNetworkID), @@ -1163,22 +967,17 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { gossipableIDs: set.Of(ip.NodeID), gossipableIndices: make(map[ids.NodeID]int), } - return tracker - }(), + }, }, { name: "connected", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - + expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ subnets: set.Of(constants.PrimaryNetworkID), @@ -1196,22 +995,17 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { ip, }, } - return tracker - }(), + }, }, { name: "connected to other subnet", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, subnetID: subnetID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - + expectedChange: func(tracker *ipTracker) { tracker.numTrackedSubnets.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ subnets: set.Of(subnetID), @@ -1223,15 +1017,19 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { gossipableIDs: set.Of(ip.NodeID), gossipableIndices: make(map[ids.NodeID]int), } - return tracker - }(), + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.OnValidatorAdded(test.subnetID, test.nodeID, nil, ids.Empty, 0) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + testState.OnValidatorAdded(test.subnetID, ip.NodeID, nil, ids.Empty, 0) + test.expectedChange(expectedState) + + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } @@ -1239,49 +1037,37 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { func TestIPTracker_OnValidatorRemoved(t *testing.T) { subnetID := ids.GenerateTestID() tests := []struct { - name string - initialState *ipTracker - subnetID ids.ID - nodeID ids.NodeID - expectedState *ipTracker + name string + initialState func(t *testing.T) *ipTracker + subnetID ids.ID + expectedChange func(*ipTracker) }{ { name: "remove last validator of subnet", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) return tracker - }(), + }, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - + expectedChange: func(tracker *ipTracker) { tracker.numTrackedPeers.Dec() tracker.numTrackedSubnets.Dec() delete(tracker.tracked, ip.NodeID) delete(tracker.subnet, constants.PrimaryNetworkID) - return tracker - }(), + }, }, { name: "manually tracked not gossipable", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) require.True(t, tracker.AddIP(ip)) return tracker - }(), + }, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyTrack(ip.NodeID) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - require.True(t, tracker.AddIP(ip)) - + expectedChange: func(tracker *ipTracker) { tracker.numTrackedSubnets.Dec() node := tracker.tracked[ip.NodeID] @@ -1289,26 +1075,19 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { node.trackedSubnets.Remove(constants.PrimaryNetworkID) delete(tracker.subnet, constants.PrimaryNetworkID) - return tracker - }(), + }, }, { name: "manually tracked latest gossipable", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyTrack(ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyTrack(ip.NodeID) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - + expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Dec() tracker.numTrackedSubnets.Dec() @@ -1317,70 +1096,48 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { node.trackedSubnets.Remove(constants.PrimaryNetworkID) delete(tracker.subnet, constants.PrimaryNetworkID) - return tracker - }(), + }, }, { name: "manually gossiped", - initialState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - return tracker - }(), - subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, + subnetID: constants.PrimaryNetworkID, + expectedChange: func(*ipTracker) {}, }, { name: "manually gossiped on other subnet", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, subnetID: subnetID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.ManuallyGossip(constants.PrimaryNetworkID, ip.NodeID) - tracker.OnValidatorAdded(subnetID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - + expectedChange: func(tracker *ipTracker) { tracker.numTrackedSubnets.Dec() tracker.tracked[ip.NodeID].subnets.Remove(subnetID) delete(tracker.subnet, subnetID) - return tracker - }(), + }, }, { name: "non-latest gossipable", - initialState: func() *ipTracker { + initialState: func(t *testing.T) *ipTracker { tracker := newTestIPTracker(t) tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) return tracker - }(), + }, subnetID: constants.PrimaryNetworkID, - nodeID: ip.NodeID, - expectedState: func() *ipTracker { - tracker := newTestIPTracker(t) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, ip.NodeID, nil, ids.Empty, 0) - tracker.Connected(ip, set.Of(constants.PrimaryNetworkID)) - tracker.OnValidatorAdded(constants.PrimaryNetworkID, otherIP.NodeID, nil, ids.Empty, 0) - tracker.Connected(otherIP, set.Of(constants.PrimaryNetworkID)) - + expectedChange: func(tracker *ipTracker) { tracker.numTrackedPeers.Dec() tracker.numGossipableIPs.Dec() delete(tracker.tracked, ip.NodeID) @@ -1393,15 +1150,19 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { subnet.gossipableIPs = []*ips.ClaimedIPPort{ otherIP, } - return tracker - }(), + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - test.initialState.OnValidatorRemoved(test.subnetID, test.nodeID, 0) - requireEqual(t, test.expectedState, test.initialState) - requireMetricsConsistent(t, test.initialState) + testState := test.initialState(t) + expectedState := test.initialState(t) + + testState.OnValidatorRemoved(test.subnetID, ip.NodeID, 0) + test.expectedChange(expectedState) + + requireEqual(t, expectedState, testState) + requireMetricsConsistent(t, testState) }) } } From c754543174cbc23ab28c4a32c6bc397111d99db5 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 20 May 2024 16:50:29 -0400 Subject: [PATCH 43/62] nits --- network/ip_tracker.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 406e953e5088..979b4dcd50f1 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -115,8 +115,8 @@ type connectedNode struct { // trackedSubnets contains all the subnets that this node is syncing, // including the primary network. trackedSubnets set.Set[ids.ID] - // ip this node claimed when connecting to use. The IP is not necessarily - // the same IP as in mostRecentTrackedIPs. + // ip this node claimed when connecting. The IP is not necessarily the same + // IP as in the tracked map. ip *ips.ClaimedIPPort } @@ -128,7 +128,7 @@ type gossipableSubnet struct { manuallyGossipable set.Set[ids.NodeID] // gossipableIDs contains the nodeIDs of all nodes whose IP could be - // gossiped. + // gossiped. This is a superset of manuallyGossipable. gossipableIDs set.Set[ids.NodeID] // An IP is marked as gossipable if all of the following conditions are met: @@ -196,7 +196,7 @@ func (s *gossipableSubnet) getGossipableIPs( } func (s *gossipableSubnet) canDelete() bool { - return s.manuallyGossipable.Len() == 0 && s.gossipableIDs.Len() == 0 + return s.gossipableIDs.Len() == 0 } type ipTracker struct { @@ -224,7 +224,8 @@ type ipTracker struct { // Connected tracks the information of currently connected peers, including // tracked and untracked nodes. connected map[ids.NodeID]*connectedNode - subnet map[ids.ID]*gossipableSubnet + // subnet tracks all the subnets that have at least one gossipable ID. + subnet map[ids.ID]*gossipableSubnet } // ManuallyTrack marks the provided nodeID as being desirable to connect to. @@ -476,7 +477,7 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin subnet, ok := i.subnet[subnetID] if !ok { - i.log.Error("removing validator from untracked subnet", + i.log.Error("attempted removal of validator from untracked subnet", zap.Stringer("subnetID", subnetID), zap.Stringer("nodeID", nodeID), ) @@ -497,7 +498,7 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin trackedNode, ok := i.tracked[nodeID] if !ok { - i.log.Error("removing untracked validator", + i.log.Error("attempted removal of untracked validator", zap.Stringer("subnetID", subnetID), zap.Stringer("nodeID", nodeID), ) From 2bf771f5fbf922d76d3d8ffcf8694722e2e95643 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 20 May 2024 16:52:02 -0400 Subject: [PATCH 44/62] nit --- network/ip_tracker.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 979b4dcd50f1..664f73fe923e 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -152,6 +152,8 @@ func (s *gossipableSubnet) removeGossipableIP(nodeID ids.NodeID) { return } + // If we aren't removing the last IP, we need to swap the last IP with the + // IP we are removing so that the slice is contiguous. newNumGossipable := len(s.gossipableIPs) - 1 if newNumGossipable != indexToRemove { replacementIP := s.gossipableIPs[newNumGossipable] From 934a64bfb90bc87ae541fea816b763ddc7d35158 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 20 May 2024 17:28:37 -0400 Subject: [PATCH 45/62] nit --- network/ip_tracker.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index 664f73fe923e..fee0cd9bd690 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -167,6 +167,9 @@ func (s *gossipableSubnet) removeGossipableIP(nodeID ids.NodeID) { s.gossipableIPs = s.gossipableIPs[:newNumGossipable] } +// [maxNumIPs] applies to the total number of IPs returned, including the IPs +// initially provided in [ips]. +// [ips] and [nodeIDs] are extended and returned with the additional IPs added. func (s *gossipableSubnet) getGossipableIPs( exceptNodeID ids.NodeID, exceptIPs *bloom.ReadFilter, @@ -174,14 +177,14 @@ func (s *gossipableSubnet) getGossipableIPs( maxNumIPs int, ips []*ips.ClaimedIPPort, nodeIDs set.Set[ids.NodeID], -) []*ips.ClaimedIPPort { +) ([]*ips.ClaimedIPPort, set.Set[ids.NodeID]) { uniform := sampler.NewUniform() uniform.Initialize(uint64(len(s.gossipableIPs))) for len(ips) < maxNumIPs { index, hasNext := uniform.Next() if !hasNext { - return ips + return ips, nodeIDs } ip := s.gossipableIPs[index] @@ -194,7 +197,7 @@ func (s *gossipableSubnet) getGossipableIPs( ips = append(ips, ip) nodeIDs.Add(ip.NodeID) } - return ips + return ips, nodeIDs } func (s *gossipableSubnet) canDelete() bool { @@ -629,7 +632,7 @@ func getGossipableIPs[T any]( continue } - ips = subnet.getGossipableIPs( + ips, nodeIDs = subnet.getGossipableIPs( exceptNodeID, exceptIPs, salt, From 8d2a2cc59a810dda6f42562c95550e44530d56ae Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 20 May 2024 18:22:25 -0400 Subject: [PATCH 46/62] nit --- network/ip_tracker_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index b6206e4a11f4..04f17cf02d0a 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -566,10 +566,6 @@ func TestIPTracker_AddIP(t *testing.T) { expectedChange: func(tracker *ipTracker) { tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 - - subnet := tracker.subnet[subnetID] - delete(subnet.gossipableIndices, newerIP.NodeID) - subnet.gossipableIPs = subnet.gossipableIPs[:0] }, }, } From 20e999d64dfab95a013a831a1c84af3fb26ee305 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 21 May 2024 12:36:05 -0400 Subject: [PATCH 47/62] Enable gossiping of newer IPs when connected to an older IP (#3035) --- network/README.md | 1 - network/ip_tracker.go | 54 ++++++++++++++++++++++---------------- network/ip_tracker_test.go | 25 ++++++++++++------ 3 files changed, 48 insertions(+), 32 deletions(-) diff --git a/network/README.md b/network/README.md index eecdd9bc52f6..1b9a4d3235bc 100644 --- a/network/README.md +++ b/network/README.md @@ -173,7 +173,6 @@ A `GetPeerList` message contains the Bloom Filter of the currently known peers a `PeerList` messages are expected to contain `IP:Port` pairs that satisfy all of the following constraints: - The Bloom Filter sent when requesting the `PeerList` message does not contain the node claiming the `IP:Port` pair. - The node claiming the `IP:Port` pair is currently connected. -- The `IP:Port` pair the node shared during the `Handshake` message is the node's most recently known `IP:Port` pair. - The node claiming the `IP:Port` pair is either in the default bootstrapper set, is a current Primary Network validator, is a validator of a tracked Subnet, or is a validator of a Subnet and the peer is a Primary Network validator. #### Example PeerList Gossip diff --git a/network/ip_tracker.go b/network/ip_tracker.go index fee0cd9bd690..58032ff4f202 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -140,7 +140,12 @@ type gossipableSubnet struct { gossipableIPs []*ips.ClaimedIPPort } -func (s *gossipableSubnet) addGossipableIP(ip *ips.ClaimedIPPort) { +func (s *gossipableSubnet) setGossipableIP(ip *ips.ClaimedIPPort) { + if index, ok := s.gossipableIndices[ip.NodeID]; ok { + s.gossipableIPs[index] = ip + return + } + s.numGossipableIPs.Inc() s.gossipableIndices[ip.NodeID] = len(s.gossipableIPs) s.gossipableIPs = append(s.gossipableIPs, ip) @@ -307,14 +312,21 @@ func (i *ipTracker) ShouldVerifyIP( // subnet. // 2. This IP is newer than the most recent IP we know of for the node. // -// If the previous IP for this node was marked as gossipable, calling this -// function will remove the previous IP from the gossipable set. +// If this IP is replacing a gossipable IP, this IP will also be marked as +// gossipable. func (i *ipTracker) AddIP(ip *ips.ClaimedIPPort) bool { i.lock.Lock() defer i.lock.Unlock() timestampComparison, trackedNode := i.addIP(ip) - return timestampComparison > sameTimestamp && trackedNode.wantsConnection() + if timestampComparison <= sameTimestamp { + return false + } + + if connectedNode, ok := i.connected[ip.NodeID]; ok { + i.setGossipableIP(trackedNode.ip, connectedNode.trackedSubnets) + } + return trackedNode.wantsConnection() } // GetIP returns the most recent IP of the provided nodeID. Returns true if all @@ -343,14 +355,10 @@ func (i *ipTracker) Connected(ip *ips.ClaimedIPPort, trackedSubnets set.Set[ids. trackedSubnets: trackedSubnets, ip: ip, } - if timestampComparison, _ := i.addIP(ip); timestampComparison < sameTimestamp { - return - } - for subnetID := range trackedSubnets { - if subnet, ok := i.subnet[subnetID]; ok && subnet.gossipableIDs.Contains(ip.NodeID) { - subnet.addGossipableIP(ip) - } + timestampComparison, trackedNode := i.addIP(ip) + if timestampComparison != untrackedTimestamp { + i.setGossipableIP(trackedNode.ip, trackedSubnets) } } @@ -364,8 +372,6 @@ func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) (int, *trackedNode) { // This is the first IP we've heard from the validator, so it is the // most recent. i.updateMostRecentTrackedIP(node, ip) - // Because we didn't previously have an IP, we know we aren't currently - // connected to them. return newTimestamp, node } @@ -376,25 +382,29 @@ func (i *ipTracker) addIP(ip *ips.ClaimedIPPort) (int, *trackedNode) { return sameTimestamp, node // This IP is equal to the previously known IP. } + // This IP is newer than the previously known IP. i.updateMostRecentTrackedIP(node, ip) - i.removeGossipableIP(ip.NodeID) return newerTimestamp, node } +func (i *ipTracker) setGossipableIP(ip *ips.ClaimedIPPort, trackedSubnets set.Set[ids.ID]) { + for subnetID := range trackedSubnets { + if subnet, ok := i.subnet[subnetID]; ok && subnet.gossipableIDs.Contains(ip.NodeID) { + subnet.setGossipableIP(ip) + } + } +} + // Disconnected is called when a connection to the peer is closed. func (i *ipTracker) Disconnected(nodeID ids.NodeID) { i.lock.Lock() defer i.lock.Unlock() - i.removeGossipableIP(nodeID) - delete(i.connected, nodeID) -} - -func (i *ipTracker) removeGossipableIP(nodeID ids.NodeID) { connectedNode, ok := i.connected[nodeID] if !ok { return } + delete(i.connected, nodeID) for subnetID := range connectedNode.trackedSubnets { if subnet, ok := i.subnet[subnetID]; ok { @@ -467,11 +477,9 @@ func (i *ipTracker) addGossipableID(nodeID ids.NodeID, subnetID ids.ID, manually return } - if trackedNode, ok := i.tracked[nodeID]; !ok || trackedNode.ip == nil || trackedNode.ip.Timestamp != node.ip.Timestamp { - return + if trackedNode, ok := i.tracked[nodeID]; ok { + subnet.setGossipableIP(trackedNode.ip) } - - subnet.addGossipableIP(node.ip) } func (*ipTracker) OnValidatorWeightChanged(ids.ID, ids.NodeID, uint64, uint64) {} diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index 04f17cf02d0a..93e924ab081e 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -544,13 +544,9 @@ func TestIPTracker_AddIP(t *testing.T) { ip: newerIP, expectedUpdatedAndDesired: true, expectedChange: func(tracker *ipTracker) { - tracker.numGossipableIPs.Dec() tracker.tracked[newerIP.NodeID].ip = newerIP tracker.bloomAdditions[newerIP.NodeID] = 2 - - subnet := tracker.subnet[constants.PrimaryNetworkID] - delete(subnet.gossipableIndices, newerIP.NodeID) - subnet.gossipableIPs = subnet.gossipableIPs[:0] + tracker.subnet[constants.PrimaryNetworkID].gossipableIPs[0] = newerIP }, }, { @@ -655,10 +651,17 @@ func TestIPTracker_Connected(t *testing.T) { }, ip: ip, expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Inc() tracker.connected[ip.NodeID] = &connectedNode{ trackedSubnets: set.Of(constants.PrimaryNetworkID), ip: ip, } + + subnet := tracker.subnet[constants.PrimaryNetworkID] + subnet.gossipableIndices[newerIP.NodeID] = 0 + subnet.gossipableIPs = []*ips.ClaimedIPPort{ + newerIP, + } }, }, { @@ -930,12 +933,18 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { }, subnetID: constants.PrimaryNetworkID, expectedChange: func(tracker *ipTracker) { + tracker.numGossipableIPs.Inc() tracker.tracked[ip.NodeID].subnets.Add(constants.PrimaryNetworkID) tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ - numGossipableIPs: tracker.numGossipableIPs, - gossipableIDs: set.Of(ip.NodeID), - gossipableIndices: make(map[ids.NodeID]int), + numGossipableIPs: tracker.numGossipableIPs, + gossipableIDs: set.Of(ip.NodeID), + gossipableIndices: map[ids.NodeID]int{ + ip.NodeID: 0, + }, + gossipableIPs: []*ips.ClaimedIPPort{ + newerIP, + }, } }, }, From f40337a7f19a045835de08f368a81a656124ce36 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 3 Jun 2024 18:37:46 -0400 Subject: [PATCH 48/62] add comment --- proto/p2p/p2p.proto | 2 ++ proto/pb/p2p/p2p.pb.go | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/proto/p2p/p2p.proto b/proto/p2p/p2p.proto index 6e43e8da2aba..4ba322243d2a 100644 --- a/proto/p2p/p2p.proto +++ b/proto/p2p/p2p.proto @@ -112,6 +112,8 @@ message Handshake { // Signature of the peer IP port pair at a provided timestamp with the BLS // key. bytes ip_bls_sig = 13; + // To avoid sending IPs that the client isn't interested in tracking, the + // server expects the client to confirm that it is tracking all subnets. bool all_subnets = 14; } diff --git a/proto/pb/p2p/p2p.pb.go b/proto/pb/p2p/p2p.pb.go index e2d97c73a82f..cd1873857523 100644 --- a/proto/pb/p2p/p2p.pb.go +++ b/proto/pb/p2p/p2p.pb.go @@ -678,8 +678,10 @@ type Handshake struct { KnownPeers *BloomFilter `protobuf:"bytes,12,opt,name=known_peers,json=knownPeers,proto3" json:"known_peers,omitempty"` // Signature of the peer IP port pair at a provided timestamp with the BLS // key. - IpBlsSig []byte `protobuf:"bytes,13,opt,name=ip_bls_sig,json=ipBlsSig,proto3" json:"ip_bls_sig,omitempty"` - AllSubnets bool `protobuf:"varint,14,opt,name=all_subnets,json=allSubnets,proto3" json:"all_subnets,omitempty"` + IpBlsSig []byte `protobuf:"bytes,13,opt,name=ip_bls_sig,json=ipBlsSig,proto3" json:"ip_bls_sig,omitempty"` + // To avoid sending IPs that the client isn't interested in tracking, the + // server expects the client to confirm that it is tracking all subnets. + AllSubnets bool `protobuf:"varint,14,opt,name=all_subnets,json=allSubnets,proto3" json:"all_subnets,omitempty"` } func (x *Handshake) Reset() { From 7cb89fb4b6aecf2cae4eff926f7347a8790aca6a Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 3 Jun 2024 18:39:15 -0400 Subject: [PATCH 49/62] nit --- network/network.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/network/network.go b/network/network.go index be14f2b53a25..12cd1353ba04 100644 --- a/network/network.go +++ b/network/network.go @@ -537,11 +537,11 @@ func (n *network) Peers( knownPeers *bloom.ReadFilter, salt []byte, ) []*ips.ClaimedIPPort { - areWeAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, n.config.MyNodeID) != 0 + areWeAPrimaryNetworkValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, n.config.MyNodeID) != 0 // Only return IPs for subnets that we are tracking. var allowedSubnets func(ids.ID) bool - if areWeAValidator { + if areWeAPrimaryNetworkValidator { allowedSubnets = func(ids.ID) bool { return true } } else { allowedSubnets = func(subnetID ids.ID) bool { @@ -549,8 +549,8 @@ func (n *network) Peers( } } - areTheyAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, peerID) != 0 - if areWeAValidator && requestAllPeers && areTheyAValidator { + areTheyAPrimaryNetworkValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, peerID) != 0 + if areWeAPrimaryNetworkValidator && requestAllPeers && areTheyAPrimaryNetworkValidator { // Return IPs for all subnets. return getGossipableIPs( n.ipTracker, From 895376d992d29f6d37348849ac6d03da8fda0d41 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 17 Jun 2024 15:04:34 -0400 Subject: [PATCH 50/62] nit --- network/network.go | 6 +++--- network/peer/peer.go | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/network/network.go b/network/network.go index 2f054b90527e..f7254e569733 100644 --- a/network/network.go +++ b/network/network.go @@ -481,7 +481,7 @@ func (n *network) AllowConnection(nodeID ids.NodeID) bool { } func (n *network) Track(claimedIPPorts []*ips.ClaimedIPPort) error { - iAmAValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, n.config.MyNodeID) != 0 + _, iAmAValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) for _, ip := range claimedIPPorts { if err := n.track(ip, iAmAValidator); err != nil { return err @@ -538,7 +538,7 @@ func (n *network) Peers( knownPeers *bloom.ReadFilter, salt []byte, ) []*ips.ClaimedIPPort { - areWeAPrimaryNetworkValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, n.config.MyNodeID) != 0 + _, areWeAPrimaryNetworkValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) // Only return IPs for subnets that we are tracking. var allowedSubnets func(ids.ID) bool @@ -550,7 +550,7 @@ func (n *network) Peers( } } - areTheyAPrimaryNetworkValidator := n.config.Validators.GetWeight(constants.PrimaryNetworkID, peerID) != 0 + _, areTheyAPrimaryNetworkValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, peerID) if areWeAPrimaryNetworkValidator && requestAllPeers && areTheyAPrimaryNetworkValidator { // Return IPs for all subnets. return getGossipableIPs( diff --git a/network/peer/peer.go b/network/peer/peer.go index efb1b47edb93..c424c1fbdccd 100644 --- a/network/peer/peer.go +++ b/network/peer/peer.go @@ -534,6 +534,7 @@ func (p *peer) writeMessages() { myVersion := p.VersionCompatibility.Version() knownPeersFilter, knownPeersSalt := p.Network.KnownPeers() + _, areWeAPrimaryNetworkValidator := p.Validators.GetValidator(constants.PrimaryNetworkID, p.MyNodeID) msg, err := p.MessageCreator.Handshake( p.NetworkID, p.Clock.Unix(), @@ -550,7 +551,7 @@ func (p *peer) writeMessages() { p.ObjectedACPs, knownPeersFilter, knownPeersSalt, - p.Validators.GetWeight(constants.PrimaryNetworkID, p.MyNodeID) != 0, + areWeAPrimaryNetworkValidator, ) if err != nil { p.Log.Error(failedToCreateMessageLog, @@ -644,10 +645,11 @@ func (p *peer) sendNetworkMessages() { select { case <-p.getPeerListChan: knownPeersFilter, knownPeersSalt := p.Config.Network.KnownPeers() + _, areWeAPrimaryNetworkValidator := p.Validators.GetValidator(constants.PrimaryNetworkID, p.MyNodeID) msg, err := p.Config.MessageCreator.GetPeerList( knownPeersFilter, knownPeersSalt, - p.Validators.GetWeight(constants.PrimaryNetworkID, p.MyNodeID) != 0, + areWeAPrimaryNetworkValidator, ) if err != nil { p.Log.Error(failedToCreateMessageLog, From 2b00ee7c316d6c98c3f507ffe6d0067a0181031b Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 17 Jun 2024 15:41:01 -0400 Subject: [PATCH 51/62] nits --- network/ip_tracker.go | 14 ++++++------ network/ip_tracker_test.go | 46 +++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/network/ip_tracker.go b/network/ip_tracker.go index c6ce4d42b69b..2d5f326e0ad9 100644 --- a/network/ip_tracker.go +++ b/network/ip_tracker.go @@ -87,9 +87,9 @@ func newIPTracker( type trackedNode struct { // manuallyTracked tracks if this node's connection was manually requested. manuallyTracked bool - // subnets contains all the subnets that this node is a validator of, - // including potentially the primary network. - subnets set.Set[ids.ID] + // validatedSubnets contains all the subnets that this node is a validator + // of, including potentially the primary network. + validatedSubnets set.Set[ids.ID] // subnets contains the subset of [subnets] that the local node also tracks, // including potentially the primary network. trackedSubnets set.Set[ids.ID] @@ -102,7 +102,7 @@ func (n *trackedNode) wantsConnection() bool { } func (n *trackedNode) canDelete() bool { - return !n.manuallyTracked && n.subnets.Len() == 0 + return !n.manuallyTracked && n.validatedSubnets.Len() == 0 } type connectedNode struct { @@ -326,7 +326,7 @@ func (i *ipTracker) AddIP(ip *ips.ClaimedIPPort) bool { // GetIP returns the most recent IP of the provided nodeID. Returns true if all // of the following conditions are met: // 1. There is currently an IP for the provided nodeID. -// 1. The provided IP is from a node whose connection is desired on a tracked +// 2. The provided IP is from a node whose connection is desired on a tracked // subnet. func (i *ipTracker) GetIP(nodeID ids.NodeID) (*ips.ClaimedIPPort, bool) { i.lock.RLock() @@ -427,7 +427,7 @@ func (i *ipTracker) addTrackableID(nodeID ids.NodeID, subnetID *ids.ID) { if subnetID == nil { nodeTracker.manuallyTracked = true } else { - nodeTracker.subnets.Add(*subnetID) + nodeTracker.validatedSubnets.Add(*subnetID) if *subnetID == constants.PrimaryNetworkID || i.trackedSubnets.Contains(*subnetID) { nodeTracker.trackedSubnets.Add(*subnetID) } @@ -512,7 +512,7 @@ func (i *ipTracker) OnValidatorRemoved(subnetID ids.ID, nodeID ids.NodeID, _ uin return } - trackedNode.subnets.Remove(subnetID) + trackedNode.validatedSubnets.Remove(subnetID) trackedNode.trackedSubnets.Remove(subnetID) if trackedNode.canDelete() { diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index ac032d989d42..31fd86270da5 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -170,9 +170,9 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { tracker.numTrackedPeers.Inc() tracker.numTrackedSubnets.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ - manuallyTracked: true, - subnets: set.Of(constants.PrimaryNetworkID), - trackedSubnets: set.Of(constants.PrimaryNetworkID), + manuallyTracked: true, + validatedSubnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), } tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ numGossipableIPs: tracker.numGossipableIPs, @@ -190,7 +190,7 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { tracker.numTrackedPeers.Inc() tracker.numTrackedSubnets.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ - subnets: set.Of(subnetID), + validatedSubnets: set.Of(subnetID), } tracker.subnet[subnetID] = &gossipableSubnet{ numGossipableIPs: tracker.numGossipableIPs, @@ -213,10 +213,10 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { tracker.numGossipableIPs.Inc() tracker.numTrackedSubnets.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ - manuallyTracked: true, - subnets: set.Of(constants.PrimaryNetworkID), - trackedSubnets: set.Of(constants.PrimaryNetworkID), - ip: ip, + manuallyTracked: true, + validatedSubnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, } tracker.bloomAdditions[ip.NodeID] = 1 tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ @@ -244,8 +244,8 @@ func TestIPTracker_ManuallyGossip(t *testing.T) { tracker.numTrackedPeers.Inc() tracker.numTrackedSubnets.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ - subnets: set.Of(subnetID), - ip: ip, + validatedSubnets: set.Of(subnetID), + ip: ip, } tracker.bloomAdditions[ip.NodeID] = 1 tracker.subnet[subnetID] = &gossipableSubnet{ @@ -887,7 +887,7 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { }, subnetID: constants.PrimaryNetworkID, expectedChange: func(tracker *ipTracker) { - tracker.tracked[ip.NodeID].subnets.Add(constants.PrimaryNetworkID) + tracker.tracked[ip.NodeID].validatedSubnets.Add(constants.PrimaryNetworkID) tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ numGossipableIPs: tracker.numGossipableIPs, @@ -907,7 +907,7 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { subnetID: constants.PrimaryNetworkID, expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Inc() - tracker.tracked[ip.NodeID].subnets.Add(constants.PrimaryNetworkID) + tracker.tracked[ip.NodeID].validatedSubnets.Add(constants.PrimaryNetworkID) tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ numGossipableIPs: tracker.numGossipableIPs, @@ -933,7 +933,7 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { subnetID: constants.PrimaryNetworkID, expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Inc() - tracker.tracked[ip.NodeID].subnets.Add(constants.PrimaryNetworkID) + tracker.tracked[ip.NodeID].validatedSubnets.Add(constants.PrimaryNetworkID) tracker.tracked[ip.NodeID].trackedSubnets.Add(constants.PrimaryNetworkID) tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ numGossipableIPs: tracker.numGossipableIPs, @@ -963,8 +963,8 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { subnetID: constants.PrimaryNetworkID, expectedChange: func(tracker *ipTracker) { tracker.tracked[ip.NodeID] = &trackedNode{ - subnets: set.Of(constants.PrimaryNetworkID), - trackedSubnets: set.Of(constants.PrimaryNetworkID), + validatedSubnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), } tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ numGossipableIPs: tracker.numGossipableIPs, @@ -984,9 +984,9 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { expectedChange: func(tracker *ipTracker) { tracker.numGossipableIPs.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ - subnets: set.Of(constants.PrimaryNetworkID), - trackedSubnets: set.Of(constants.PrimaryNetworkID), - ip: ip, + validatedSubnets: set.Of(constants.PrimaryNetworkID), + trackedSubnets: set.Of(constants.PrimaryNetworkID), + ip: ip, } tracker.bloomAdditions[ip.NodeID] = 1 tracker.subnet[constants.PrimaryNetworkID] = &gossipableSubnet{ @@ -1012,8 +1012,8 @@ func TestIPTracker_OnValidatorAdded(t *testing.T) { expectedChange: func(tracker *ipTracker) { tracker.numTrackedSubnets.Inc() tracker.tracked[ip.NodeID] = &trackedNode{ - subnets: set.Of(subnetID), - ip: ip, + validatedSubnets: set.Of(subnetID), + ip: ip, } tracker.bloomAdditions[ip.NodeID] = 1 tracker.subnet[subnetID] = &gossipableSubnet{ @@ -1075,7 +1075,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { tracker.numTrackedSubnets.Dec() node := tracker.tracked[ip.NodeID] - node.subnets.Remove(constants.PrimaryNetworkID) + node.validatedSubnets.Remove(constants.PrimaryNetworkID) node.trackedSubnets.Remove(constants.PrimaryNetworkID) delete(tracker.subnet, constants.PrimaryNetworkID) @@ -1096,7 +1096,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { tracker.numTrackedSubnets.Dec() node := tracker.tracked[ip.NodeID] - node.subnets.Remove(constants.PrimaryNetworkID) + node.validatedSubnets.Remove(constants.PrimaryNetworkID) node.trackedSubnets.Remove(constants.PrimaryNetworkID) delete(tracker.subnet, constants.PrimaryNetworkID) @@ -1126,7 +1126,7 @@ func TestIPTracker_OnValidatorRemoved(t *testing.T) { subnetID: subnetID, expectedChange: func(tracker *ipTracker) { tracker.numTrackedSubnets.Dec() - tracker.tracked[ip.NodeID].subnets.Remove(subnetID) + tracker.tracked[ip.NodeID].validatedSubnets.Remove(subnetID) delete(tracker.subnet, subnetID) }, }, From cf3d60d0fe55145729c5136905f1f21017c9fbfd Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 18 Jun 2024 11:33:59 -0400 Subject: [PATCH 52/62] standardize name --- network/network.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/network/network.go b/network/network.go index f7254e569733..031a3fc1e318 100644 --- a/network/network.go +++ b/network/network.go @@ -476,14 +476,14 @@ func (n *network) AllowConnection(nodeID ids.NodeID) bool { if !n.config.RequireValidatorToConnect { return true } - _, iAmAValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) - return iAmAValidator || n.ipTracker.WantsConnection(nodeID) + _, areWeAPrimaryNetworkAValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) + return areWeAPrimaryNetworkAValidator || n.ipTracker.WantsConnection(nodeID) } func (n *network) Track(claimedIPPorts []*ips.ClaimedIPPort) error { - _, iAmAValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) + _, areWeAPrimaryNetworkAValidator := n.config.Validators.GetValidator(constants.PrimaryNetworkID, n.config.MyNodeID) for _, ip := range claimedIPPorts { - if err := n.track(ip, iAmAValidator); err != nil { + if err := n.track(ip, areWeAPrimaryNetworkAValidator); err != nil { return err } } @@ -754,9 +754,9 @@ func (n *network) getPeers( continue } - _, isValidator := n.config.Validators.GetValidator(subnetID, nodeID) + _, areTheyAValidator := n.config.Validators.GetValidator(subnetID, nodeID) // check if the peer is allowed to connect to the subnet - if !allower.IsAllowed(nodeID, isValidator) { + if !allower.IsAllowed(nodeID, areTheyAValidator) { continue } @@ -797,9 +797,9 @@ func (n *network) samplePeers( return false } - _, isValidator := n.config.Validators.GetValidator(subnetID, peerID) + _, areTheyAValidator := n.config.Validators.GetValidator(subnetID, peerID) // check if the peer is allowed to connect to the subnet - if !allower.IsAllowed(peerID, isValidator) { + if !allower.IsAllowed(peerID, areTheyAValidator) { return false } @@ -808,7 +808,7 @@ func (n *network) samplePeers( return true } - if isValidator { + if areTheyAValidator { numValidatorsToSample-- return numValidatorsToSample >= 0 } From 3ed23a81cbc95532788f8d98e6208d3769f22eb2 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 18 Jun 2024 11:35:54 -0400 Subject: [PATCH 53/62] nit --- network/ip_tracker_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/network/ip_tracker_test.go b/network/ip_tracker_test.go index 31fd86270da5..99f1a616287e 100644 --- a/network/ip_tracker_test.go +++ b/network/ip_tracker_test.go @@ -43,7 +43,6 @@ func requireEqual(t *testing.T, expected, actual *ipTracker) { require.Equal(expected.bloomAdditions, actual.bloomAdditions) require.Equal(expected.maxBloomCount, actual.maxBloomCount) require.Equal(expected.connected, actual.connected) - require.Equal(expected.connected, actual.connected) require.Equal(expected.subnet, actual.subnet) } From a13ddf276322eab8a27a69678830ee5857a174e5 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 18 Jun 2024 11:36:49 -0400 Subject: [PATCH 54/62] nit --- network/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/README.md b/network/README.md index 1b9a4d3235bc..0408cbf7ebb5 100644 --- a/network/README.md +++ b/network/README.md @@ -55,7 +55,7 @@ A peer will then read the full message and attempt to parse it into either a net Upon connection to a new peer, a handshake is performed between the node attempting to establish the outbound connection to the peer and the peer receiving the inbound connection. -When attempting to establish the connection, the first message that the node sends is a `Handshake` message describing configuration of the node. If the `Handshake` message is successfully received and the peer decides that it will allow a connection with this node, it replies with a `PeerList` message that contains metadata about other peers that allows a node to connect to them. See [PeerList Gossip](#peerlist-gossip). +When attempting to establish the connection, the first message that the node sends is a `Handshake` message describing the configuration of the node. If the `Handshake` message is successfully received and the peer decides that it will allow a connection with this node, it replies with a `PeerList` message that contains metadata about other peers that allows a node to connect to them. See [PeerList Gossip](#peerlist-gossip). As an example, nodes that are attempting to connect with an incompatible version of AvalancheGo or a significantly skewed local clock are rejected. From 5df54ebdd8281ff82f4754a424d2d26a9448b335 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 25 Jun 2024 18:59:10 -0400 Subject: [PATCH 55/62] nit --- network/README.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/network/README.md b/network/README.md index 0408cbf7ebb5..80b8f2bdbd13 100644 --- a/network/README.md +++ b/network/README.md @@ -152,6 +152,15 @@ Peers are discovered by receiving [`PeerList`](#peerlist) messages during the [P To provide an eventual guarantee that all peers learn of one another, nodes periodically send a [`GetPeerList`](#getpeerlist) message to a randomly selected Primary Network validator with the node's current [Bloom Filter](#bloom-filter) and `Salt`. +#### Gossipable Peers + +The peers that a node may include into a [`GetPeerList`](#getpeerlist) message are considered `gossipable`. + + +#### Trackable Peers + +The peers that a node would attempt to connect to if included in a [`PeerList`](#peerlist) message are considered `trackable`. + #### Bloom Filter A [Bloom Filter](https://en.wikipedia.org/wiki/Bloom_filter) is used to track which nodes are known. @@ -175,6 +184,35 @@ A `GetPeerList` message contains the Bloom Filter of the currently known peers a - The node claiming the `IP:Port` pair is currently connected. - The node claiming the `IP:Port` pair is either in the default bootstrapper set, is a current Primary Network validator, is a validator of a tracked Subnet, or is a validator of a Subnet and the peer is a Primary Network validator. +#### Avoiding Persistent Network Traffic + +To avoid persistent network traffic, it must eventually hold that the set of [`gossipable peers`](#gossipable-peers) is a subset of the [`trackable peers`](#trackable-peers) for all nodes in the network. + + + +For example, say there are 3 nodes: `Rick`, `Morty`, and `Summer`. + +First we consider the case that `Rick` and `Morty` consider `Summer` [`gossipable`](#gossipable-peers) and [`trackable`](#trackable-peers), respectively. +```mermaid +sequenceDiagram + actor Morty + actor Rick + Note left of Morty: Not currently tracking Summer + Morty->>Rick: GetPeerList + Note right of Rick: Summer isn't in the bloom filter + Rick->>Morty: PeerList containing Summer + Note left of Morty: Bloom: [1, 0, 0] + Morty->>Rick: GetPeerList + Note right of Rick: Summer is in the bloom filter + Rick->>Morty: PeerList - Empty +``` + +1. Node `A` sends a `GetPeerList` message to node `B`. +2. Node `B` will respond with [`gossipable peers`](#gossipable-peers). +3. Node `A` will attempt to connect to the peers that it considers [`trackable`](#trackable-peers). + +If `B` includes `C` in the `PeerList`, and `A` is not tracking `C`, then `A` would ignore the information about `C`. By ignoring `C`, `A` would not add `C` to its bloom filter. If `A` were to ask `B` for peers again, `B` would then be willing to include `C` again. Thus if `A` is not tracking nodes that `B` considers gossipable, there can be persistent network traffic. + #### Example PeerList Gossip The following diagram shows an example of `Alice` repeatedly learning about new peers from `Bob`. From 612e21c0dc8e124ddc5a5608df6e4baa75dc2906 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 25 Jun 2024 18:59:57 -0400 Subject: [PATCH 56/62] nit --- network/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/README.md b/network/README.md index 80b8f2bdbd13..26ecbefa29c2 100644 --- a/network/README.md +++ b/network/README.md @@ -201,7 +201,7 @@ sequenceDiagram Morty->>Rick: GetPeerList Note right of Rick: Summer isn't in the bloom filter Rick->>Morty: PeerList containing Summer - Note left of Morty: Bloom: [1, 0, 0] + Note left of Morty: Track Summer and add to bloom filter Morty->>Rick: GetPeerList Note right of Rick: Summer is in the bloom filter Rick->>Morty: PeerList - Empty From 053cb007b80e236a316fa2378c8c7ae063b3acef Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 25 Jun 2024 19:00:39 -0400 Subject: [PATCH 57/62] nit --- network/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/README.md b/network/README.md index 26ecbefa29c2..5ddd4f3bd3a1 100644 --- a/network/README.md +++ b/network/README.md @@ -200,7 +200,7 @@ sequenceDiagram Note left of Morty: Not currently tracking Summer Morty->>Rick: GetPeerList Note right of Rick: Summer isn't in the bloom filter - Rick->>Morty: PeerList containing Summer + Rick->>Morty: PeerList - Contains Summer Note left of Morty: Track Summer and add to bloom filter Morty->>Rick: GetPeerList Note right of Rick: Summer is in the bloom filter From dbe32f6ebec2620a53fe5cb0c639f0516a063d21 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 25 Jun 2024 19:03:48 -0400 Subject: [PATCH 58/62] nit --- network/README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/network/README.md b/network/README.md index 5ddd4f3bd3a1..bf44949370ec 100644 --- a/network/README.md +++ b/network/README.md @@ -188,8 +188,6 @@ A `GetPeerList` message contains the Bloom Filter of the currently known peers a To avoid persistent network traffic, it must eventually hold that the set of [`gossipable peers`](#gossipable-peers) is a subset of the [`trackable peers`](#trackable-peers) for all nodes in the network. - - For example, say there are 3 nodes: `Rick`, `Morty`, and `Summer`. First we consider the case that `Rick` and `Morty` consider `Summer` [`gossipable`](#gossipable-peers) and [`trackable`](#trackable-peers), respectively. @@ -207,11 +205,21 @@ sequenceDiagram Rick->>Morty: PeerList - Empty ``` -1. Node `A` sends a `GetPeerList` message to node `B`. -2. Node `B` will respond with [`gossipable peers`](#gossipable-peers). -3. Node `A` will attempt to connect to the peers that it considers [`trackable`](#trackable-peers). - -If `B` includes `C` in the `PeerList`, and `A` is not tracking `C`, then `A` would ignore the information about `C`. By ignoring `C`, `A` would not add `C` to its bloom filter. If `A` were to ask `B` for peers again, `B` would then be willing to include `C` again. Thus if `A` is not tracking nodes that `B` considers gossipable, there can be persistent network traffic. +Now we consider the case that `Rick` considers `Summer` [`gossipable`](#gossipable-peers), but `Morty` does not consider `Summer` [`trackable`](#trackable-peers). +```mermaid +sequenceDiagram + actor Morty + actor Rick + Note left of Morty: Not currently tracking Summer + Morty->>Rick: GetPeerList + Note right of Rick: Summer isn't in the bloom filter + Rick->>Morty: PeerList - Contains Summer + Note left of Morty: Ignore Summer + Morty->>Rick: GetPeerList + Note right of Rick: Summer isn't in the bloom filter + Rick->>Morty: PeerList - Contains Summer +``` +This case is suboptimal, because `Rick` told `Morty` about `Summer` multiple times. If this case were to happen consistently, `Rick` may waste a significant amount of bandwidth trying to teach `Morty` about `Summer`. #### Example PeerList Gossip From 8238d60c05d15efcaed051f66737eb2bdd0b72aa Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 25 Jun 2024 19:04:45 -0400 Subject: [PATCH 59/62] nit --- network/README.md | 104 +++++++++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/network/README.md b/network/README.md index bf44949370ec..1f1724e388cc 100644 --- a/network/README.md +++ b/network/README.md @@ -33,15 +33,15 @@ Peers communicate by enqueuing messages between one another. Each peer on either ```mermaid sequenceDiagram - actor Alice - actor Bob + actor Morty + actor Rick loop - Alice->>Bob: Write outbound messages - Bob->>Alice: Read incoming messages + Morty->>Rick: Write outbound messages + Rick->>Morty: Read incoming messages end loop - Bob->>Alice: Write outbound messages - Alice->>Bob: Read incoming messages + Rick->>Morty: Write outbound messages + Morty->>Rick: Read incoming messages end ``` @@ -61,44 +61,44 @@ As an example, nodes that are attempting to connect with an incompatible version ```mermaid sequenceDiagram - actor Alice - actor Bob - Note over Alice,Bob: Connection Created + actor Morty + actor Rick + Note over Morty,Rick: Connection Created par - Alice->>Bob: AvalancheGo v1.0.0 + Morty->>Rick: AvalancheGo v1.0.0 and - Bob->>Alice: AvalancheGo v1.11.4 + Rick->>Morty: AvalancheGo v1.11.4 end - Note right of Bob: v1.0.0 is incompatible with v1.11.4. - Note left of Alice: v1.11.4 could be compatible with v1.0.0! + Note right of Rick: v1.0.0 is incompatible with v1.11.4. + Note left of Morty: v1.11.4 could be compatible with v1.0.0! par - Bob-->>Alice: Disconnect + Rick-->>Morty: Disconnect and - Alice-XBob: Peerlist + Morty-XRick: Peerlist end - Note over Alice,Bob: Handshake Failed + Note over Morty,Rick: Handshake Failed ``` Nodes that mutually desire the connection will both respond with `PeerList` messages and complete the handshake. ```mermaid sequenceDiagram - actor Alice - actor Bob - Note over Alice,Bob: Connection Created + actor Morty + actor Rick + Note over Morty,Rick: Connection Created par - Alice->>Bob: AvalancheGo v1.11.0 + Morty->>Rick: AvalancheGo v1.11.0 and - Bob->>Alice: AvalancheGo v1.11.4 + Rick->>Morty: AvalancheGo v1.11.4 end - Note right of Bob: v1.11.0 is compatible with v1.11.4! - Note left of Alice: v1.11.4 could be compatible with v1.11.0! + Note right of Rick: v1.11.0 is compatible with v1.11.4! + Note left of Morty: v1.11.4 could be compatible with v1.11.0! par - Bob->>Alice: Peerlist + Rick->>Morty: Peerlist and - Alice->>Bob: Peerlist + Morty->>Rick: Peerlist end - Note over Alice,Bob: Handshake Complete + Note over Morty,Rick: Handshake Complete ``` ### Ping-Pong Messages @@ -107,12 +107,12 @@ Peers periodically send `Ping` messages containing perceived uptime information. ```mermaid sequenceDiagram - actor Alice - actor Bob - Note left of Alice: Send Ping - Alice->>Bob: I think your uptime is 95% - Note right of Bob: Send Pong - Bob->>Alice: ACK + actor Morty + actor Rick + Note left of Morty: Send Ping + Morty->>Rick: I think your uptime is 95% + Note right of Rick: Send Pong + Rick->>Morty: ACK ``` ## Peer Discovery @@ -223,27 +223,27 @@ This case is suboptimal, because `Rick` told `Morty` about `Summer` multiple tim #### Example PeerList Gossip -The following diagram shows an example of `Alice` repeatedly learning about new peers from `Bob`. +The following diagram shows an example of `Morty` repeatedly learning about new peers from `Rick`. ```mermaid sequenceDiagram - actor Alice - actor Bob - Note left of Alice: Initialize Bloom Filter - Note left of Alice: Bloom: [0, 0, 0] - Alice->>Bob: GetPeerList [0, 0, 0] - Note right of Bob: Any peers can be sent. - Bob->>Alice: PeerList - Peer-1 - Note left of Alice: Bloom: [1, 0, 0] - Alice->>Bob: GetPeerList [1, 0, 0] - Note right of Bob: Either Peer-2 or Peer-3 can be sent. - Bob->>Alice: PeerList - Peer-3 - Note left of Alice: Bloom: [1, 0, 1] - Alice->>Bob: GetPeerList [1, 0, 1] - Note right of Bob: Only Peer-2 can be sent. - Bob->>Alice: PeerList - Peer-2 - Note left of Alice: Bloom: [1, 1, 1] - Alice->>Bob: GetPeerList [1, 1, 1] - Note right of Bob: There are no more peers left to send! - Bob->>Alice: PeerList - Empty + actor Morty + actor Rick + Note left of Morty: Initialize Bloom Filter + Note left of Morty: Bloom: [0, 0, 0] + Morty->>Rick: GetPeerList [0, 0, 0] + Note right of Rick: Any peers can be sent. + Rick->>Morty: PeerList - Peer-1 + Note left of Morty: Bloom: [1, 0, 0] + Morty->>Rick: GetPeerList [1, 0, 0] + Note right of Rick: Either Peer-2 or Peer-3 can be sent. + Rick->>Morty: PeerList - Peer-3 + Note left of Morty: Bloom: [1, 0, 1] + Morty->>Rick: GetPeerList [1, 0, 1] + Note right of Rick: Only Peer-2 can be sent. + Rick->>Morty: PeerList - Peer-2 + Note left of Morty: Bloom: [1, 1, 1] + Morty->>Rick: GetPeerList [1, 1, 1] + Note right of Rick: There are no more peers left to send! + Rick->>Morty: PeerList - Empty ``` From 8e59560e448dc7815b3607ea32aabab6906644f9 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 25 Jun 2024 19:07:14 -0400 Subject: [PATCH 60/62] nit --- network/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/network/README.md b/network/README.md index 1f1724e388cc..9f589d848c76 100644 --- a/network/README.md +++ b/network/README.md @@ -204,6 +204,7 @@ sequenceDiagram Note right of Rick: Summer is in the bloom filter Rick->>Morty: PeerList - Empty ``` +This case is ideal, as `Rick` only notifies `Morty` about `Summer` once, and never uses bandwidth for their connection again. Now we consider the case that `Rick` considers `Summer` [`gossipable`](#gossipable-peers), but `Morty` does not consider `Summer` [`trackable`](#trackable-peers). ```mermaid From be86c974d0e401d712406326d70134cd63411432 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 25 Jun 2024 19:08:49 -0400 Subject: [PATCH 61/62] remove redundent graph --- network/README.md | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/network/README.md b/network/README.md index 9f589d848c76..a6a2d1a751b6 100644 --- a/network/README.md +++ b/network/README.md @@ -220,31 +220,4 @@ sequenceDiagram Note right of Rick: Summer isn't in the bloom filter Rick->>Morty: PeerList - Contains Summer ``` -This case is suboptimal, because `Rick` told `Morty` about `Summer` multiple times. If this case were to happen consistently, `Rick` may waste a significant amount of bandwidth trying to teach `Morty` about `Summer`. - -#### Example PeerList Gossip - -The following diagram shows an example of `Morty` repeatedly learning about new peers from `Rick`. - -```mermaid -sequenceDiagram - actor Morty - actor Rick - Note left of Morty: Initialize Bloom Filter - Note left of Morty: Bloom: [0, 0, 0] - Morty->>Rick: GetPeerList [0, 0, 0] - Note right of Rick: Any peers can be sent. - Rick->>Morty: PeerList - Peer-1 - Note left of Morty: Bloom: [1, 0, 0] - Morty->>Rick: GetPeerList [1, 0, 0] - Note right of Rick: Either Peer-2 or Peer-3 can be sent. - Rick->>Morty: PeerList - Peer-3 - Note left of Morty: Bloom: [1, 0, 1] - Morty->>Rick: GetPeerList [1, 0, 1] - Note right of Rick: Only Peer-2 can be sent. - Rick->>Morty: PeerList - Peer-2 - Note left of Morty: Bloom: [1, 1, 1] - Morty->>Rick: GetPeerList [1, 1, 1] - Note right of Rick: There are no more peers left to send! - Rick->>Morty: PeerList - Empty -``` +This case is suboptimal, because `Rick` told `Morty` about `Summer` multiple times. If this case were to happen consistently, `Rick` may waste a significant amount of bandwidth trying to teach `Morty` about `Summer`. \ No newline at end of file From 009e4459937b2027b71e7e7ac2ecf2cd13d9d0f1 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Tue, 25 Jun 2024 19:25:31 -0400 Subject: [PATCH 62/62] nit --- network/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/network/README.md b/network/README.md index a6a2d1a751b6..3d3341050e60 100644 --- a/network/README.md +++ b/network/README.md @@ -16,6 +16,7 @@ - [Bloom Filter](#bloom-filter) - [GetPeerList](#getpeerlist) - [PeerList](#peerlist) + - [Avoiding Persistent Network Traffic](#avoiding-persistent-network-traffic) ## Overview