Change SubmitBackup to only reboot in Attrition workload for UpgradeAndBackupRestore-1 [release-7.2]

fdbserver/include/fdbserver/TesterInterface.actor.h

@@ -85,6 +85,7 @@ struct WorkloadRequest {
 	int clientId; // the "id" of the client receiving the request (0 indexed)
 	int clientCount; // the total number of test clients participating in the workload
 	ReplyPromise<struct WorkloadInterface> reply;
+	std::vector<std::string> disabledFailureInjectionWorkloads;
 
 	template <class Ar>
 	void serialize(Ar& ar) {
@@ -100,6 +101,7 @@ struct WorkloadRequest {
 		           reply,
 		           defaultTenant,
 		           runFailureWorkloads,
+		           disabledFailureInjectionWorkloads,
 		           arena);
 	}
 };

fdbserver/include/fdbserver/workloads/workloads.actor.h

@@ -335,6 +335,7 @@ class TestSpec {
 	ISimulator::BackupAgentType simDrAgents;
 
 	KnobKeyValuePairs overrideKnobs;
+	std::vector<std::string> disabledFailureInjectionWorkloads;
 };
 
 ACTOR Future<DistributedTestResults> runWorkload(Database cx,

fdbserver/tester.actor.cpp

@@ -45,6 +45,7 @@
 #include "fdbserver/Knobs.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbrpc/SimulatorProcessInfo.h"
+#include "flow/Platform.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 FDB_DEFINE_BOOLEAN_PARAM(UntrustedMode);
@@ -402,8 +403,18 @@ void CompoundWorkload::addFailureInjection(WorkloadRequest& work) {
 		if (disabledWorkloads.count(workload->description()) > 0) {
 			continue;
 		}
+		if (std::count(work.disabledFailureInjectionWorkloads.begin(),
+		               work.disabledFailureInjectionWorkloads.end(),
+		               workload->description()) > 0) {
+			continue;
+		}
 		while (shouldInjectFailure(random, work, workload)) {
 			workload->initFailureInjectionMode(random);
+			TraceEvent("AddFailureInjectionWorkload")
+			    .detail("Name", workload->description())
+			    .detail("ClientID", work.clientId)
+			    .detail("ClientCount", clientCount)
+			    .detail("Title", work.title);
 			failureInjection.push_back(workload);
 			workload = factory->create(*this);
 		}
@@ -986,6 +997,7 @@ ACTOR Future<DistributedTestResults> runWorkload(Database cx,
 		req.clientCount = testers.size();
 		req.sharedRandomNumber = sharedRandom;
 		req.defaultTenant = defaultTenant.castTo<TenantNameRef>();
+		req.disabledFailureInjectionWorkloads = spec.disabledFailureInjectionWorkloads;
 		workRequests.push_back(testers[i].recruitments.getReply(req));
 	}
 
@@ -1414,6 +1426,18 @@ std::map<std::string, std::function<void(const std::string& value, TestSpec* spe
 	  } },
 	{ "runFailureWorkloads",
 	  [](const std::string& value, TestSpec* spec) { spec->runFailureWorkloads = (value == "true"); } },
+	{ "disabledFailureInjectionWorkloads",
+	  [](const std::string& value, TestSpec* spec) {
+	      std::stringstream ss(value);
+	      while (ss.good()) {
+		      std::string substr;
+		      getline(ss, substr, ',');
+		      substr = removeWhitespace(substr);
+		      if (!substr.empty()) {
+			      spec->disabledFailureInjectionWorkloads.push_back(substr);
+		      }
+	      }
+	  } },
 };
 
 std::vector<TestSpec> readTests(std::ifstream& ifs) {

tests/CMakeLists.txt

@@ -276,8 +276,8 @@ if(WITH_PYTHON)
     TEST_FILES restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-1.txt
     restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt)
   add_fdb_test(
-    TEST_FILES restarting/from_7.0.0/UpgradeAndBackupRestore-1.toml
-    restarting/from_7.0.0/UpgradeAndBackupRestore-2.toml)
+    TEST_FILES restarting/from_7.0.0_until_7.2.0/UpgradeAndBackupRestore-1.toml
+    restarting/from_7.0.0_until_7.2.0/UpgradeAndBackupRestore-2.toml)
   add_fdb_test(
     TEST_FILES restarting/to_7.1.0/CycleTestRestart-1.toml
     restarting/to_7.1.0/CycleTestRestart-2.toml)
@@ -308,6 +308,9 @@ if(WITH_PYTHON)
   add_fdb_test(
     TEST_FILES restarting/from_7.2.0/DrUpgradeRestart-1.txt
     restarting/from_7.2.0/DrUpgradeRestart-2.txt)
+  add_fdb_test(
+    TEST_FILES restarting/from_7.2.4/UpgradeAndBackupRestore-1.toml
+    restarting/from_7.2.4/UpgradeAndBackupRestore-2.toml)
 
 
   add_fdb_test(TEST_FILES slow/ApiCorrectness.toml)

tests/restarting/from_7.0.0_until_7.2.0/UpgradeAndBackupRestore-2.toml

@@ -0,0 +1,61 @@
+[[test]]
+testTitle = 'SecondCycleTest'
+simBackupAgents = 'BackupToFile'
+clearAfterTest=false
+runConsistencyCheck=false
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 30000
+    transactionsPerSecond = 2500.0
+    testDuration = 30.0
+    expectedRate = 0
+    keyPrefix = 'AfterRestart'
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 90.0
+
+    [[test.workload]]
+    testName = 'Rollback'
+    meanDelay = 90.0
+    testDuration = 90.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 90.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 90.0
+
+[[test]]
+testTitle= 'RestoreBackup'
+simBackupAgents = 'BackupToFile'
+clearAfterTest=false
+
+    [[test.workload]]
+    testName = 'RestoreBackup'
+    tag = 'default'
+
+[[test]]
+testTitle = 'CheckCycles'
+checkOnly=true
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount=30000
+    keyPrefix = 'AfterRestart'
+    expectedRate=0
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 30000
+    keyPrefix= 'BeforeRestart'
+    expectedRate = 0

tests/restarting/from_7.2.4/UpgradeAndBackupRestore-1.toml

@@ -0,0 +1,67 @@
+[configuration]
+storageEngineExcludeTypes=[3]
+
+[[test]]
+testTitle = 'SubmitBackup'
+simBackupAgents= 'BackupToFile'
+clearAfterTest = false
+runConsistencyCheck=false
+disabledFailureInjectionWorkloads = 'Attrition'
+
+    [[test.workload]]
+    testName = 'SubmitBackup'
+    delayFor = 0
+    stopWhenDone = false
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 30.0
+
+[[test]]
+testTitle = 'FirstCycleTest'
+clearAfterTest=false
+runConsistencyCheck = false
+disabledFailureInjectionWorkloads = 'Attrition'
+
+    [[test.workload]]
+    testName = 'Cycle'
+    nodeCount = 30000
+    transactionsPerSecond = 2500.0
+    testDuration = 30.0
+    expectedRate = 0
+    keyPrefix = 'BeforeRestart'
+
+    [[test.workload]]
+    testName = 'RandomClogging'
+    testDuration = 90.0
+
+    [[test.workload]]
+    testName = 'Rollback'
+    meanDelay = 90.0
+    testDuration = 90.0
+
+    [[test.workload]]
+    testName = 'Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 90.0
+
+    [[test.workload]]
+    testName='Attrition'
+    machinesToKill = 10
+    machinesToLeave = 3
+    reboot = true
+    testDuration = 90.0
+
+[[test]]
+testTitle = 'SaveDatabase'
+clearAfterTest = false
+
+    [[test.workload]]
+    testName = 'SaveAndKill'
+    restartInfoLocation = 'simfdb/restartInfo.ini'
+    testDuration=30.0