Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Batch Scenario Runs Into Single Test #10878

Merged
merged 6 commits into from
Jun 15, 2022
Merged

Batch Scenario Runs Into Single Test #10878

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
50619d8
batch scenarios
nisdas Jun 14, 2022
80a7653
fix
nisdas Jun 14, 2022
cc1cd7d
fix
nisdas Jun 14, 2022
a8485b8
Merge branch 'develop' into batchScenarios
nisdas Jun 14, 2022
10fbfd4
Update testing/endtoend/endtoend_test.go
nisdas Jun 14, 2022
2bb85e5
Merge branch 'develop' into batchScenarios
nisdas Jun 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion testing/endtoend/endtoend_setup_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func e2eMinimal(t *testing.T, cfgo ...types.E2EConfigOpt) *testRunner {
return newTestRunner(t, testConfig)
}

func e2eMainnet(t *testing.T, usePrysmSh, useMultiClient bool) *testRunner {
func e2eMainnet(t *testing.T, usePrysmSh, useMultiClient bool, cfgo ...types.E2EConfigOpt) *testRunner {
params.SetupTestConfigCleanup(t)
params.OverrideBeaconConfig(params.E2EMainnetTestConfig())
if useMultiClient {
Expand Down Expand Up @@ -155,6 +155,9 @@ func e2eMainnet(t *testing.T, usePrysmSh, useMultiClient bool) *testRunner {
EvalInterceptor: defaultInterceptor,
Seed: int64(seed),
}
for _, o := range cfgo {
o(testConfig)
}

return newTestRunner(t, testConfig)
}
Expand Down
162 changes: 78 additions & 84 deletions testing/endtoend/endtoend_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -563,24 +563,17 @@ func (r *testRunner) executeProvidedEvaluators(currentEpoch uint64, conns []*grp
wg.Wait()
}

func (r *testRunner) singleNodeOffline(epoch uint64, _ []*grpc.ClientConn) bool {
switch epoch {
case 9:
require.NoError(r.t, r.comHandler.beaconNodes.PauseAtIndex(0))
require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(0))
return true
case 10:
require.NoError(r.t, r.comHandler.beaconNodes.ResumeAtIndex(0))
require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(0))
return true
case 11, 12:
// Allow 2 epochs for the network to finalize again.
return true
}
return false
}

func (r *testRunner) singleNodeOfflineMulticlient(epoch uint64, _ []*grpc.ClientConn) bool {
// This interceptor will define the multi scenario run for our minimal tests.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

2) is missing from the comment

// 1) In the first scenario we will be taking a single prysm node and its validator offline.
// Along with that we will also take a single lighthouse node and its validator offline.
// After 1 epoch we will then attempt to bring it online again.
//
//
// 3) Then we will start testing optimistic sync by engaging our engine proxy.
nisdas marked this conversation as resolved.
Show resolved Hide resolved
// After the proxy has been sending `SYNCING` responses to the beacon node, we
// will test this with our optimistic sync evaluator to ensure everything works
// as expected.
func (r *testRunner) multiScenarioMulticlient(epoch uint64, conns []*grpc.ClientConn) bool {
switch epoch {
case 9:
require.NoError(r.t, r.comHandler.beaconNodes.PauseAtIndex(0))
Expand All @@ -594,48 +587,8 @@ func (r *testRunner) singleNodeOfflineMulticlient(epoch uint64, _ []*grpc.Client
require.NoError(r.t, r.comHandler.lighthouseBeaconNodes.ResumeAtIndex(0))
require.NoError(r.t, r.comHandler.lighthouseValidatorNodes.ResumeAtIndex(0))
return true
case 11, 12:
// Allow 2 epochs for the network to finalize again.
return true
}
return false
}

func (r *testRunner) eeOffline(epoch uint64, _ []*grpc.ClientConn) bool {
switch epoch {
case 9:
require.NoError(r.t, r.comHandler.eth1Miner.Pause())
return true
case 10:
require.NoError(r.t, r.comHandler.eth1Miner.Resume())
return true
case 11, 12:
// Allow 2 epochs for the network to finalize again.
return true
}
return false
}

func (r *testRunner) allValidatorsOffline(epoch uint64, _ []*grpc.ClientConn) bool {
switch epoch {
case 9:
require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(0))
require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(1))
return true
case 10:
require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(0))
require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(1))
return true
case 11, 12:
// Allow 2 epochs for the network to finalize again.
return true
}
return false
}

func (r *testRunner) optimisticSync(epoch uint64, conns []*grpc.ClientConn) bool {
switch epoch {
case 9:
case 14:
// Set it for prysm beacon node.
component, err := r.comHandler.eth1Proxy.ComponentAtIndex(0)
require.NoError(r.t, err)
component.(e2etypes.EngineProxy).AddRequestInterceptor("engine_newPayloadV1", func() interface{} {
Expand All @@ -646,8 +599,19 @@ func (r *testRunner) optimisticSync(epoch uint64, conns []*grpc.ClientConn) bool
}, func() bool {
return true
})
// Set it for lighthouse beacon node.
component, err = r.comHandler.eth1Proxy.ComponentAtIndex(2)
require.NoError(r.t, err)
component.(e2etypes.EngineProxy).AddRequestInterceptor("engine_newPayloadV1", func() interface{} {
return &enginev1.PayloadStatus{
Status: enginev1.PayloadStatus_SYNCING,
LatestValidHash: make([]byte, 32),
}
}, func() bool {
return true
})
return true
case 10:
case 15:
r.executeProvidedEvaluators(epoch, []*grpc.ClientConn{conns[0]}, []e2etypes.Evaluator{
ev.OptimisticSyncEnabled,
})
Expand All @@ -659,6 +623,29 @@ func (r *testRunner) optimisticSync(epoch uint64, conns []*grpc.ClientConn) bool
engineProxy.RemoveRequestInterceptor("engine_newPayloadV1")
engineProxy.ReleaseBackedUpRequests("engine_newPayloadV1")

// Remove for lighthouse too
component, err = r.comHandler.eth1Proxy.ComponentAtIndex(2)
require.NoError(r.t, err)
engineProxy, ok = component.(e2etypes.EngineProxy)
require.Equal(r.t, true, ok)
engineProxy.RemoveRequestInterceptor("engine_newPayloadV1")
engineProxy.ReleaseBackedUpRequests("engine_newPayloadV1")

return true
case 11, 12, 16, 17:
// Allow 2 epochs for the network to finalize again.
return true
}
return false
}

func (r *testRunner) eeOffline(epoch uint64, _ []*grpc.ClientConn) bool {
switch epoch {
case 9:
require.NoError(r.t, r.comHandler.eth1Miner.Pause())
return true
case 10:
require.NoError(r.t, r.comHandler.eth1Miner.Resume())
return true
case 11, 12:
// Allow 2 epochs for the network to finalize again.
Expand All @@ -667,10 +654,36 @@ func (r *testRunner) optimisticSync(epoch uint64, conns []*grpc.ClientConn) bool
return false
}

func (r *testRunner) optimisticSyncMulticlient(epoch uint64, conns []*grpc.ClientConn) bool {
// This interceptor will define the multi scenario run for our minimal tests.
// 1) In the first scenario we will be taking a single node and its validator offline.
// After 1 epoch we will then attempt to bring it online again.
//
// 2) In the second scenario we will be taking all validators offline. After 2
// epochs we will wait for the network to recover.
//
// 3) Then we will start testing optimistic sync by engaging our engine proxy.
// After the proxy has been sending `SYNCING` responses to the beacon node, we
// will test this with our optimistic sync evaluator to ensure everything works
// as expected.
func (r *testRunner) multiScenario(epoch uint64, conns []*grpc.ClientConn) bool {
switch epoch {
case 9:
// Set it for prysm beacon node.
require.NoError(r.t, r.comHandler.beaconNodes.PauseAtIndex(0))
require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(0))
return true
case 10:
require.NoError(r.t, r.comHandler.beaconNodes.ResumeAtIndex(0))
require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(0))
return true
case 14:
require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(0))
require.NoError(r.t, r.comHandler.validatorNodes.PauseAtIndex(1))
return true
case 15:
require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(0))
require.NoError(r.t, r.comHandler.validatorNodes.ResumeAtIndex(1))
return true
case 19:
component, err := r.comHandler.eth1Proxy.ComponentAtIndex(0)
require.NoError(r.t, err)
component.(e2etypes.EngineProxy).AddRequestInterceptor("engine_newPayloadV1", func() interface{} {
Expand All @@ -681,19 +694,8 @@ func (r *testRunner) optimisticSyncMulticlient(epoch uint64, conns []*grpc.Clien
}, func() bool {
return true
})
// Set it for lighthouse beacon node.
component, err = r.comHandler.eth1Proxy.ComponentAtIndex(2)
require.NoError(r.t, err)
component.(e2etypes.EngineProxy).AddRequestInterceptor("engine_newPayloadV1", func() interface{} {
return &enginev1.PayloadStatus{
Status: enginev1.PayloadStatus_SYNCING,
LatestValidHash: make([]byte, 32),
}
}, func() bool {
return true
})
return true
case 10:
case 20:
r.executeProvidedEvaluators(epoch, []*grpc.ClientConn{conns[0]}, []e2etypes.Evaluator{
ev.OptimisticSyncEnabled,
})
Expand All @@ -705,16 +707,8 @@ func (r *testRunner) optimisticSyncMulticlient(epoch uint64, conns []*grpc.Clien
engineProxy.RemoveRequestInterceptor("engine_newPayloadV1")
engineProxy.ReleaseBackedUpRequests("engine_newPayloadV1")

// Remove for lighthouse too
component, err = r.comHandler.eth1Proxy.ComponentAtIndex(2)
require.NoError(r.t, err)
engineProxy, ok = component.(e2etypes.EngineProxy)
require.Equal(r.t, true, ok)
engineProxy.RemoveRequestInterceptor("engine_newPayloadV1")
engineProxy.ReleaseBackedUpRequests("engine_newPayloadV1")

return true
case 11, 12:
case 11, 12, 16, 17, 21, 22:
// Allow 2 epochs for the network to finalize again.
return true
}
Expand Down
15 changes: 5 additions & 10 deletions testing/endtoend/mainnet_scenario_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,17 @@ package endtoend

import (
"testing"

"github.com/prysmaticlabs/prysm/testing/endtoend/types"
)

func TestEndToEnd_MainnetConfig_MultiClient(t *testing.T) {
e2eMainnet(t, false /*usePrysmSh*/, true /*useMultiClient*/).run()
}

func TestEndToEnd_ScenarioRun_BeaconOffline_Multiclient(t *testing.T) {
runner := e2eMainnet(t, false /*usePrysmSh*/, true /*useMultiClient*/)
runner.config.Evaluators = scenarioEvalsMulti()
runner.config.EvalInterceptor = runner.singleNodeOffline
runner.scenarioRunner()
}

func TestEndToEnd_ScenarioRun_OptimisticSync_Multiclient(t *testing.T) {
runner := e2eMainnet(t, false /*usePrysmSh*/, true /*useMultiClient*/)
func TestEndToEnd_MultiScenarioRun_Multiclient(t *testing.T) {
runner := e2eMainnet(t, false /*usePrysmSh*/, true /*useMultiClient*/, types.WithEpochs(22))
runner.config.Evaluators = scenarioEvalsMulti()
runner.config.EvalInterceptor = runner.optimisticSyncMulticlient
runner.config.EvalInterceptor = runner.multiScenarioMulticlient
runner.scenarioRunner()
}
26 changes: 7 additions & 19 deletions testing/endtoend/minimal_scenario_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,16 @@
package endtoend

import "testing"
import (
"testing"

func TestEndToEnd_ScenarioRun_BeaconOffline(t *testing.T) {
runner := e2eMinimal(t)

runner.config.Evaluators = scenarioEvals()
runner.config.EvalInterceptor = runner.singleNodeOffline
runner.scenarioRunner()
}
"github.com/prysmaticlabs/prysm/testing/endtoend/types"
)

func TestEndToEnd_ScenarioRun_AllvalidatorsOffline(t *testing.T) {
runner := e2eMinimal(t)
func TestEndToEnd_MultiScenarioRun(t *testing.T) {
runner := e2eMinimal(t, types.WithEpochs(22))

runner.config.Evaluators = scenarioEvals()
runner.config.EvalInterceptor = runner.allValidatorsOffline
runner.config.EvalInterceptor = runner.multiScenario
runner.scenarioRunner()
}

Expand All @@ -26,11 +22,3 @@ func TestEndToEnd_ScenarioRun_EEOffline(t *testing.T) {
runner.config.EvalInterceptor = runner.eeOffline
runner.scenarioRunner()
}

func TestEndToEnd_ScenarioRun_OptimisticSync(t *testing.T) {
runner := e2eMinimal(t)

runner.config.Evaluators = scenarioEvals()
runner.config.EvalInterceptor = runner.optimisticSync
runner.scenarioRunner()
}
6 changes: 6 additions & 0 deletions testing/endtoend/types/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ func WithExtraEpochs(extra uint64) E2EConfigOpt {
}
}

func WithEpochs(e uint64) E2EConfigOpt {
return func(cfg *E2EConfig) {
cfg.EpochsToRun = e
}
}

func WithRemoteSigner() E2EConfigOpt {
return func(cfg *E2EConfig) {
cfg.UseWeb3RemoteSigner = true
Expand Down