From 4b91ae2800729afe1a6f0fd1817d23f637bde31b Mon Sep 17 00:00:00 2001 From: Bogdan Kanivets Date: Wed, 8 Feb 2023 23:27:02 -0800 Subject: [PATCH] tests: trigger raftBeforeFollowerSend failpoint in linearizability test raftBeforeFollowerSend can only be triggered on Follower and won't work on 1 node cluster. Had to split RandomFailpoint into RandomOneNodeClusterFailpoint and RandomMultiNodeClusterFailpoint Signed-off-by: Bogdan Kanivets --- tests/linearizability/failpoints.go | 21 ++++++++++--------- tests/linearizability/linearizability_test.go | 4 ++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/linearizability/failpoints.go b/tests/linearizability/failpoints.go index 675e18b164b..1792c173e7e 100644 --- a/tests/linearizability/failpoints.go +++ b/tests/linearizability/failpoints.go @@ -55,7 +55,7 @@ var ( RaftBeforeLeaderSendPanic Failpoint = goPanicFailpoint{"raftBeforeLeaderSend", nil, Leader} BlackholePeerNetwork Failpoint = blackholePeerNetworkFailpoint{duration: time.Second} DelayPeerNetwork Failpoint = delayPeerNetworkFailpoint{duration: time.Second, baseLatency: 75 * time.Millisecond, randomizedLatency: 50 * time.Millisecond} - RandomFailpoint Failpoint = randomFailpoint{[]Failpoint{ + oneNodeClusterFailpoints = []Failpoint{ KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic, DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic, @@ -67,17 +67,18 @@ var ( RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork, - }} - RaftBeforeApplySnapPanic Failpoint = goPanicFailpoint{"raftBeforeApplySnap", triggerBlackholeUntilSnapshot, Follower} - RaftAfterApplySnapPanic Failpoint = goPanicFailpoint{"raftAfterApplySnap", triggerBlackholeUntilSnapshot, Follower} - RaftAfterWALReleasePanic Failpoint = goPanicFailpoint{"raftAfterWALRelease", triggerBlackholeUntilSnapshot, Follower} - RaftBeforeSaveSnapPanic Failpoint = goPanicFailpoint{"raftBeforeSaveSnap", triggerBlackholeUntilSnapshot, Follower} - RaftAfterSaveSnapPanic Failpoint = goPanicFailpoint{"raftAfterSaveSnap", triggerBlackholeUntilSnapshot, Follower} - RandomSnapshotFailpoint Failpoint = randomFailpoint{[]Failpoint{ + } + RandomOneNodeClusterFailpoint Failpoint = randomFailpoint{oneNodeClusterFailpoints} + RaftBeforeFollowerSendPanic Failpoint = goPanicFailpoint{"raftBeforeFollowerSend", nil, Follower} + RandomMultiNodeClusterFailpoint Failpoint = randomFailpoint{append(oneNodeClusterFailpoints, RaftBeforeFollowerSendPanic)} + RaftBeforeApplySnapPanic Failpoint = goPanicFailpoint{"raftBeforeApplySnap", triggerBlackholeUntilSnapshot, Follower} + RaftAfterApplySnapPanic Failpoint = goPanicFailpoint{"raftAfterApplySnap", triggerBlackholeUntilSnapshot, Follower} + RaftAfterWALReleasePanic Failpoint = goPanicFailpoint{"raftAfterWALRelease", triggerBlackholeUntilSnapshot, Follower} + RaftBeforeSaveSnapPanic Failpoint = goPanicFailpoint{"raftBeforeSaveSnap", triggerBlackholeUntilSnapshot, Follower} + RaftAfterSaveSnapPanic Failpoint = goPanicFailpoint{"raftAfterSaveSnap", triggerBlackholeUntilSnapshot, Follower} + RandomSnapshotFailpoint Failpoint = randomFailpoint{[]Failpoint{ RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic, RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, }} - // TODO: Figure out how to reliably trigger below failpoints and add them to RandomFailpoint - raftBeforeFollowerSendPanic Failpoint = goPanicFailpoint{"raftBeforeFollowerSend", nil, AnyMember} ) type Failpoint interface { diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index e5e7b3786b6..ab377ec6bdd 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -96,7 +96,7 @@ func TestLinearizability(t *testing.T) { for _, traffic := range trafficList { scenarios = append(scenarios, scenario{ name: "ClusterOfSize1/" + traffic.name, - failpoint: RandomFailpoint, + failpoint: RandomOneNodeClusterFailpoint, traffic: &traffic, config: *e2e.NewConfig( e2e.WithClusterSize(1), @@ -107,7 +107,7 @@ func TestLinearizability(t *testing.T) { }) scenarios = append(scenarios, scenario{ name: "ClusterOfSize3/" + traffic.name, - failpoint: RandomFailpoint, + failpoint: RandomMultiNodeClusterFailpoint, traffic: &traffic, config: *e2e.NewConfig( e2e.WithSnapshotCount(100),