Try reproduce etcd-io#17247

Change-Id: Ieed5f16ded627ac27f6acae99fea23211e6f718d Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
serathius · Jan 16, 2024 · c9d515a · c9d515a
1 parent 4bc55e5
commit c9d515a
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 107 deletions.
diff --git a/.github/workflows/robustness.yaml b/.github/workflows/robustness.yaml
@@ -7,15 +7,15 @@ jobs:
     uses: ./.github/workflows/robustness-template.yaml
     with:
       etcdBranch: main
-      count: 12
-      testTimeout: 30m
+      count: 80
+      testTimeout: 200m
       artifactName: main
       runs-on: "['ubuntu-latest-8-cores']"
   main-arm64:
     uses: ./.github/workflows/robustness-template.yaml
     with:
       etcdBranch: main
-      count: 12
-      testTimeout: 30m
+      count: 80
+      testTimeout: 200m
       artifactName: main-arm64
       runs-on: "['actuated-arm64-8cpu-8gb']"
diff --git a/tests/robustness/failpoint/failpoint.go b/tests/robustness/failpoint/failpoint.go
@@ -37,20 +37,22 @@ const (
 
 var (
 	allFailpoints = []Failpoint{
-		KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
-		DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
-		BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic,
-		BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
-		CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic,
-		CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork,
-		RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic,
-		RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot,
-		BeforeApplyOneConfChangeSleep,
-		MemberReplace,
-		DropPeerNetwork,
-		RaftBeforeSaveSleep,
-		RaftAfterSaveSleep,
-		ApplyBeforeOpenSnapshot,
+		//KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic,
+		//DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic,
+		//BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic,
+		//BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic,
+		//CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic,
+		//CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork,
+		//RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic,
+		//RaftBeforeSaveSnapPanic,
+		RaftAfterSaveSnapPanic,
+		//BlackholeUntilSnapshot,
+		//BeforeApplyOneConfChangeSleep,
+		//MemberReplace,
+		//DropPeerNetwork,
+		//RaftBeforeSaveSleep,
+		//RaftAfterSaveSleep,
+		//ApplyBeforeOpenSnapshot,
 	}
 )
 

diff --git a/tests/robustness/scenarios.go b/tests/robustness/scenarios.go
@@ -32,18 +32,6 @@ type TrafficProfile struct {
 }
 
 var trafficProfiles = []TrafficProfile{
-	{
-		Traffic: traffic.EtcdPut,
-		Profile: traffic.HighTrafficProfile,
-	},
-	{
-		Traffic: traffic.EtcdPutDeleteLease,
-		Profile: traffic.LowTraffic,
-	},
-	{
-		Traffic: traffic.Kubernetes,
-		Profile: traffic.HighTrafficProfile,
-	},
 	{
 		Traffic: traffic.Kubernetes,
 		Profile: traffic.LowTraffic,
@@ -64,7 +52,6 @@ func scenarios(t *testing.T) []testScenario {
 	if err != nil {
 		t.Fatalf("Failed checking etcd version binary, binary: %q, err: %v", e2e.BinPath.Etcd, err)
 	}
-	enableLazyFS := e2e.BinPath.LazyFSAvailable()
 	randomizableOptions := []e2e.EPClusterOption{
 		options.WithClusterOptionGroups(
 			options.ClusterOptions{options.WithTickMs(29), options.WithElectionMs(271)},
@@ -73,29 +60,13 @@ func scenarios(t *testing.T) []testScenario {
 	}
 
 	baseOptions := []e2e.EPClusterOption{
-		options.WithSnapshotCount(50, 100, 1000),
+		options.WithSnapshotCount(50),
 		options.WithSubsetOptions(randomizableOptions...),
 		e2e.WithGoFailEnabled(true),
 		e2e.WithCompactionBatchLimit(100),
 		e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond),
 	}
 	scenarios := []testScenario{}
-	for _, tp := range trafficProfiles {
-		name := filepath.Join(tp.Traffic.Name(), tp.Profile.Name, "ClusterOfSize1")
-		clusterOfSize1Options := baseOptions
-		clusterOfSize1Options = append(clusterOfSize1Options, e2e.WithClusterSize(1))
-		// Add LazyFS only for traffic with lower QPS as it uses a lot of CPU lowering minimal QPS.
-		if enableLazyFS && tp.Profile.MinimalQPS <= 100 {
-			clusterOfSize1Options = append(clusterOfSize1Options, e2e.WithLazyFSEnabled(true))
-			name = filepath.Join(name, "LazyFS")
-		}
-		scenarios = append(scenarios, testScenario{
-			name:    name,
-			traffic: tp.Traffic,
-			profile: tp.Profile,
-			cluster: *e2e.NewConfig(clusterOfSize1Options...),
-		})
-	}
 
 	for _, tp := range trafficProfiles {
 		name := filepath.Join(tp.Traffic.Name(), tp.Profile.Name, "ClusterOfSize3")
@@ -112,64 +83,5 @@ func scenarios(t *testing.T) []testScenario {
 			cluster: *e2e.NewConfig(clusterOfSize3Options...),
 		})
 	}
-	scenarios = append(scenarios, testScenario{
-		name:      "Issue14370",
-		failpoint: failpoint.RaftBeforeSavePanic,
-		profile:   traffic.LowTraffic,
-		traffic:   traffic.EtcdPutDeleteLease,
-		cluster: *e2e.NewConfig(
-			options.WithSubsetOptions(randomizableOptions...),
-			e2e.WithClusterSize(1),
-			e2e.WithGoFailEnabled(true),
-		),
-	})
-	scenarios = append(scenarios, testScenario{
-		name:      "Issue14685",
-		failpoint: failpoint.DefragBeforeCopyPanic,
-		profile:   traffic.LowTraffic,
-		traffic:   traffic.EtcdPutDeleteLease,
-		cluster: *e2e.NewConfig(
-			options.WithSubsetOptions(randomizableOptions...),
-			e2e.WithClusterSize(1),
-			e2e.WithGoFailEnabled(true),
-		),
-	})
-	scenarios = append(scenarios, testScenario{
-		name:      "Issue13766",
-		failpoint: failpoint.KillFailpoint,
-		profile:   traffic.HighTrafficProfile,
-		traffic:   traffic.EtcdPut,
-		cluster: *e2e.NewConfig(
-			e2e.WithSnapshotCount(100),
-		),
-	})
-	scenarios = append(scenarios, testScenario{
-		name: "Issue15220",
-		watch: watchConfig{
-			requestProgress: true,
-		},
-		profile: traffic.LowTraffic,
-		traffic: traffic.EtcdPutDeleteLease,
-		cluster: *e2e.NewConfig(
-			options.WithSubsetOptions(randomizableOptions...),
-			e2e.WithClusterSize(1),
-		),
-	})
-	// TODO: Deflake waiting for waiting until snapshot for etcd versions that don't support setting snapshot catchup entries.
-	if v.Compare(version.V3_6) >= 0 {
-		scenarios = append(scenarios, testScenario{
-			name:      "Issue15271",
-			failpoint: failpoint.BlackholeUntilSnapshot,
-			profile:   traffic.HighTrafficProfile,
-			traffic:   traffic.EtcdPut,
-			cluster: *e2e.NewConfig(
-				options.WithSubsetOptions(randomizableOptions...),
-				e2e.WithSnapshotCatchUpEntries(100),
-				e2e.WithSnapshotCount(100),
-				e2e.WithPeerProxy(true),
-				e2e.WithIsPeerTLS(true),
-			),
-		})
-	}
 	return scenarios
 }