From a114d913593e630b6e1acd2cac46ce5a66ed06ac Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Fri, 10 Feb 2023 00:00:15 +0100 Subject: [PATCH] Reproduce #15271 Signed-off-by: Marek Siarkowicz --- .github/workflows/linearizability-template.yaml | 2 +- tests/linearizability/failpoints.go | 14 +++++++++----- tests/linearizability/linearizability_test.go | 3 +-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linearizability-template.yaml b/.github/workflows/linearizability-template.yaml index 2c7d3119f83..4fd8c009cda 100644 --- a/.github/workflows/linearizability-template.yaml +++ b/.github/workflows/linearizability-template.yaml @@ -46,7 +46,7 @@ jobs: - name: test-linearizability run: | # Use --failfast to avoid overriding report generated by failed test - EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability + EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run TestLinearizability/Snapshot' RESULTS_DIR=/tmp/linearizability make test-linearizability - uses: actions/upload-artifact@v2 if: always() with: diff --git a/tests/linearizability/failpoints.go b/tests/linearizability/failpoints.go index 08f8a4e1acb..ead806233b0 100644 --- a/tests/linearizability/failpoints.go +++ b/tests/linearizability/failpoints.go @@ -29,7 +29,7 @@ import ( ) const ( - triggerTimeout = 5 * time.Second + triggerTimeout = 20 * time.Second ) var ( @@ -53,7 +53,8 @@ var ( CompactBeforeCommitBatchPanic Failpoint = goPanicFailpoint{"compactBeforeCommitBatch", triggerCompact, AnyMember} CompactAfterCommitBatchPanic Failpoint = goPanicFailpoint{"compactAfterCommitBatch", triggerCompact, AnyMember} RaftBeforeLeaderSendPanic Failpoint = goPanicFailpoint{"raftBeforeLeaderSend", nil, Leader} - BlackholePeerNetwork Failpoint = blackholePeerNetworkFailpoint{} + BlackholePeerNetwork Failpoint = blackholePeerNetworkFailpoint{waitTillSnapshot: false} + BlackholeUntilSnapshot Failpoint = blackholePeerNetworkFailpoint{waitTillSnapshot: true} DelayPeerNetwork Failpoint = delayPeerNetworkFailpoint{duration: time.Second, baseLatency: 75 * time.Millisecond, randomizedLatency: 50 * time.Millisecond} oneNodeClusterFailpoints = []Failpoint{ KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, @@ -77,7 +78,8 @@ var ( RaftBeforeSaveSnapPanic Failpoint = goPanicFailpoint{"raftBeforeSaveSnap", triggerBlackholeUntilSnapshot, Follower} RaftAfterSaveSnapPanic Failpoint = goPanicFailpoint{"raftAfterSaveSnap", triggerBlackholeUntilSnapshot, Follower} RandomSnapshotFailpoint Failpoint = randomFailpoint{[]Failpoint{ - RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic, RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, + //RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic, RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, + BlackholeUntilSnapshot, }} ) @@ -308,11 +310,13 @@ func (f randomFailpoint) Available(e2e.EtcdProcess) bool { return true } -type blackholePeerNetworkFailpoint struct{} +type blackholePeerNetworkFailpoint struct { + waitTillSnapshot bool +} func (f blackholePeerNetworkFailpoint) Trigger(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster) error { member := clus.Procs[rand.Int()%len(clus.Procs)] - return triggerBlackhole(t, ctx, member, clus, false) + return triggerBlackhole(t, ctx, member, clus, f.waitTillSnapshot) } func triggerBlackhole(t *testing.T, ctx context.Context, member e2e.EtcdProcess, clus *e2e.EtcdProcessCluster, shouldWaitTillSnapshot bool) error { diff --git a/tests/linearizability/linearizability_test.go b/tests/linearizability/linearizability_test.go index be327677578..940f5a602bf 100644 --- a/tests/linearizability/linearizability_test.go +++ b/tests/linearizability/linearizability_test.go @@ -64,9 +64,8 @@ var ( largePutSize: 32769, leaseTTL: DefaultLeaseTTL, writes: []requestChance{ - {operation: Put, chance: 85}, + {operation: Put, chance: 90}, {operation: MultiOpTxn, chance: 10}, - {operation: LargePut, chance: 5}, }, }, }