Enhance TKG repeat feature (#356)

- add TKG_ITERATIONS and AOT_ITERATIONS options
- add option to continue or stop after failure is detected

Signed-off-by: renfeiw <renfeiw@ca.ibm.com>

featureSettings.mk

@@ -12,16 +12,16 @@
 # limitations under the License.
 ##############################################################################
 
-ifneq (,$(findstring AOT,$(TEST_FLAG)))
-export TR_silentEnv=1
-export TR_Options=forceAOT
-export TR_OptionsAOT=forceAOT
-AOT_OPTIONS = -Xshareclasses:name=test_aot -Xscmx400M -Xscmaxaot256m
-ifndef TEST_ITERATIONS
-TEST_ITERATIONS = 2
-endif
+ifndef TKG_ITERATIONS
+	export TKG_ITERATIONS=1
 endif
 
-ifndef TEST_ITERATIONS
-TEST_ITERATIONS = 1
+ifneq (,$(findstring AOT,$(TEST_FLAG)))
+	export TR_silentEnv=1
+	export TR_Options=forceAOT
+	export TR_OptionsAOT=forceAOT
+	AOT_OPTIONS = -Xshareclasses:name=test_aot -Xscmx400M -Xscmaxaot256m
+	ifndef AOT_ITERATIONS
+		export AOT_ITERATIONS=2
+	endif
 endif

makeGen.mk

@@ -42,7 +42,7 @@ autoconfig:
 	perl scripts$(D)configure.pl
 
 autogen: autoconfig
-	${TEST_JDK_HOME}/bin/java -cp $(Q)$(TKG_JAR)$(P)$(JSON_SIMPLE)$(Q) org.testKitGen.MainRunner --mode=$(MODE) --spec=$(SPEC) --microArch=$(MICROARCH) --jdkVersion=$(JDK_VERSION) --impl=$(JDK_IMPL) --vendor=$(Q)$(JDK_VENDOR)$(Q) --buildList=${BUILD_LIST} --iterations=$(TEST_ITERATIONS) --testFlag=$(TEST_FLAG) --testTarget=$(TESTTARGET) --testList=$(TESTLIST) --numOfMachines=$(NUM_MACHINES) --testTime=$(TEST_TIME) --TRSSURL=$(TRSS_URL) $(OPTS)
+	${TEST_JDK_HOME}/bin/java -cp $(Q)$(TKG_JAR)$(P)$(JSON_SIMPLE)$(Q) org.testKitGen.MainRunner --mode=$(MODE) --spec=$(SPEC) --microArch=$(MICROARCH) --jdkVersion=$(JDK_VERSION) --impl=$(JDK_IMPL) --vendor=$(Q)$(JDK_VENDOR)$(Q) --buildList=${BUILD_LIST} --iterations=$(TKG_ITERATIONS) --aotIterations=$(AOT_ITERATIONS) --testFlag=$(TEST_FLAG) --testTarget=$(TESTTARGET) --testList=$(TESTLIST) --numOfMachines=$(NUM_MACHINES) --testTime=$(TEST_TIME) --TRSSURL=$(TRSS_URL) $(OPTS)
 
 AUTOGEN_FILES = $(wildcard $(CURRENT_DIR)$(D)jvmTest.mk)
 AUTOGEN_FILES += $(wildcard $(CURRENT_DIR)$(D)machineConfigure.mk)

settings.mk

@@ -132,6 +132,17 @@ JVM_TEST_ROOT = $(BUILD_ROOT)
 TEST_GROUP=level.*
 
 
+TEST_REPEAT = false
+
+ifneq ($(TKG_ITERATIONS), 1)
+	TEST_REPEAT = true
+endif
+
+ifdef AOT_ITERATIONS
+	ifneq ($(AOT_ITERATIONS), 1)
+		TEST_REPEAT = true
+	endif
+endif
 
 #######################################
 # Set OS, ARCH and BITS based on SPEC
@@ -190,10 +201,10 @@ ifndef UNIQUEID
 	export UNIQUEID := $(shell perl $(GETID) -v)
 endif
 TESTOUTPUT := $(TEST_ROOT)$(D)TKG$(D)output_$(UNIQUEID)
-ifeq ($(TEST_ITERATIONS), 1)
-	REPORTDIR_NQ = $(TESTOUTPUT)$(D)$@
-else
+ifeq ($(TEST_REPEAT), true)
 	REPORTDIR_NQ = $(TESTOUTPUT)$(D)$@_ITER_$$itercnt
+else
+	REPORTDIR_NQ = $(TESTOUTPUT)$(D)$@
 endif
 REPORTDIR = $(Q)$(REPORTDIR_NQ)$(Q)
 
@@ -205,10 +216,32 @@ KEEP_REPORTDIR?=true
 ifeq ($(KEEP_REPORTDIR), false)
 	RM_REPORTDIR=$(RM) -r $(REPORTDIR);
 endif
-ifeq ($(TEST_ITERATIONS), 1) 
-	TEST_STATUS=if [ $$? -eq 0 ] ; then $(ECHO) $(Q)$(Q); $(ECHO) $(Q)$@$(Q)$(Q)_PASSED$(Q); $(ECHO) $(Q)$(Q); $(CD) $(TEST_ROOT); $(RM_REPORTDIR) else $(ECHO) $(Q)$(Q); $(ECHO) $(Q)$@$(Q)$(Q)_FAILED$(Q); $(ECHO) $(Q)$(Q); fi
+ECHO_SPACE=$(ECHO) $(Q)$(Q)
+ECHO_PASSED=$(ECHO_SPACE); $(ECHO) $(Q)$@$(Q)$(Q)_PASSED$(Q); $(ECHO_SPACE)
+ECHO_FAILED=$(ECHO_SPACE); $(ECHO) $(Q)$@$(Q)$(Q)_FAILED$(Q); $(ECHO_SPACE)
+ifneq ($(TEST_REPEAT), true) 
+	TEST_STATUS=if [ $$? -eq 0 ] ; then $(ECHO_PASSED); $(CD) $(TEST_ROOT); $(RM_REPORTDIR) else $(ECHO_FAILED); fi
 else
-	TEST_STATUS=if [ $$? -eq 0 ] ; then $(ECHO) $(Q)$(Q); $(ECHO) $(Q)$@$(Q)$(Q)_PASSED(ITER_$$itercnt)$(Q); $(ECHO) $(Q)$(Q); $(CD) $(TEST_ROOT); $(RM_REPORTDIR) if [ $$itercnt -eq $(TEST_ITERATIONS) ] ; then $(ECHO) $(Q)$(Q); $(ECHO) $(Q)$@$(Q)$(Q)_PASSED$(Q); $(ECHO) $(Q)$(Q); fi else $(ECHO) $(Q)$(Q); $(ECHO) $(Q)$@$(Q)$(Q)_FAILED(ITER_$$itercnt)$(Q); $(ECHO) $(Q)$(Q); $(ECHO) $(Q)$(Q); $(ECHO) $(Q)$@$(Q)$(Q)_FAILED$(Q); $(ECHO) $(Q)$(Q); exit 1; fi
+	ECHO_PASSED_ITER=$(ECHO_SPACE); $(ECHO) $(Q)$@$(Q)$(Q)_PASSED(ITER_$$itercnt)$(Q); $(ECHO_SPACE)
+	ECHO_FAILED_ITER=$(ECHO_SPACE); $(ECHO) $(Q)$@$(Q)$(Q)_FAILED(ITER_$$itercnt)$(Q); $(ECHO_SPACE)
+	HAS_FAILURE=[ $$success -ne $$itercnt ]
+	EXIT_CONDITION=[ $$itercnt -eq $(TEST_ITERATIONS) ]
+	ifeq ($(EXIT_FAILURE), true)
+		EXIT_CONDITION+=|| $(HAS_FAILURE)
+	endif
+	TEST_STATUS=\
+		if [ $$? -eq 0 ] ; \
+		then $(ECHO_PASSED_ITER); success=$$((success+1)); $(CD) $(TEST_ROOT); $(RM_REPORTDIR) \
+		else $(ECHO_FAILED_ITER); \
+		fi; \
+		if $(EXIT_CONDITION); \
+		then \
+			$(ECHO) $(Q)(success rate: $$success/$$itercnt)$(Q); \
+			if $(HAS_FAILURE); \
+			then $(ECHO_FAILED); exit 1;\
+			else $(ECHO_PASSED); \
+			fi; \
+		fi
 endif
 
 ifneq ($(DEBUG),)

src/org/testKitGen/Arguments.java

@@ -34,7 +34,8 @@ enum Mode {
 	private String vendor = "";
 	private String projectRootDir = System.getProperty("user.dir") + "/..";
 	private String buildList = "";
-	private String iterations = "";
+	private Integer iterations = null;
+	private Integer aotIterations = null;
 	private String testFlag = "";
 	private Integer numOfMachines = null;
 	private Integer testTime = null;
@@ -58,14 +59,16 @@ enum Mode {
 			+ "                              Defaults to the parent folder of TKG \n"
 			+ "    --buildList=<paths>       Comma separated project paths (relative to projectRootDir) to search for playlist.xml\n"
 			+ "                              Defaults to projectRootDir\n"
-			+ "    --iterations=<number>     Repeatedly generate test command based on iteration number\n"
+			+ "    --iterations=<number>     Repeatedly generate test command based on iterations number\n"
 			+ "                              Defaults to 1\n"
+			+ "    --aotIterations=<number>  Repeatedly generate aot test command based on aotIterations number.\n"
+			+ "                              When both iterations and aotIterations is set, the larger parameter will be applied on AOT applicable tests. AOT explicit tests will be repeated based on the iterations parameter.\n"
+			+ "                              Defaults to false\n"
 			+ "    --testFlag=<string>       Comma separated string to specify different test flags\n"
 			+ "                              Defaults to \"\"\n"
 			+ "    --testTarget=<string>     Test target to execute\n"
 			+ "                              Defaults to all\n"
 			+ "    --numOfMachines=<number>  Specify number of machines for mode parallelList \n"
-			+ "                              Defaults to 1\n"
 			+ "    --testTime=<number>       Specify expected length of test running time (minutes) on each machines for mode parallelList, this option will be suppressed if numOfMachines is given\n"
 			+ "                              If testTime and numOfMachines are not provided, default numOfMachines will be used\n"
 			+ "    --TRSSURL=<serverURL>     Specify the TRSS server URL for mode parallelList\n"
@@ -124,10 +127,14 @@ public String getBuildList() {
 		return buildList;
 	}
 
-	public String getIterations() {
+	public Integer getIterations() {
 		return iterations;
 	}
 
+	public Integer getAotIterations() {
+		return aotIterations;
+	}
+
 	public List<String> getTestFlag() {
 		return Arrays.asList(testFlag.split(","));
 	}
@@ -152,6 +159,20 @@ public String getTestList() {
 		return testList;
 	}
 
+	private Integer parsePositiveArgOrDefault(String arg, Integer defaultValue) {
+		String str = arg.substring(arg.indexOf("=") + 1);
+		Integer result = defaultValue;
+		if (!str.isEmpty()) {
+			result = Integer.valueOf(str);
+			if (result <= 0) {
+				System.err.println("Invalid option: " + arg);
+				System.err.println("The value needs to be greater than 0");
+				System.exit(1);
+			}
+		}
+		return result;
+	}
+
 	private String spec2Plat(String spec) {
 		String plat = "";
 		try (FileReader platReader = new FileReader(Constants.BUILDPLAT_JSON)) {
@@ -211,7 +232,9 @@ public void parse(String[] args) {
 				// buildList is case sensitive
 				buildList = arg.substring(arg.indexOf("=") + 1);
 			} else if (arglc.startsWith("--iterations=")) {
-				iterations = arglc.substring(arg.indexOf("=") + 1);
+				iterations = parsePositiveArgOrDefault(arg, 1);
+			} else if  (arglc.startsWith("--aotiterations=")) {
+				aotIterations = parsePositiveArgOrDefault(arg, 1);
 			} else if (arglc.startsWith("--testflag=")) {
 				testFlag = arglc.substring(arg.indexOf("=") + 1);
 			} else if (arglc.startsWith("--testlist=")) {
@@ -225,25 +248,9 @@ public void parse(String[] args) {
 				TRSSURL = arg.substring(arg.indexOf("=") + 1);
 				TRSSURL = TRSSURL.replaceAll("/$", "");
 			} else if (arglc.startsWith("--numofmachines")) {
-				String numOfMachinesStr = arg.substring(arg.indexOf("=") + 1);
-				if (!numOfMachinesStr.isEmpty()) {
-					numOfMachines = Integer.valueOf(numOfMachinesStr);
-					if (numOfMachines <= 0) {
-						System.err.println("Invalid option: " + arg);
-						System.err.println("Num of machines need to be greater than 0");
-						System.exit(1);
-					}
-				}
+				numOfMachines = parsePositiveArgOrDefault(arg, null);
 			} else if (arglc.startsWith("--testtime")) {
-				String testTimeStr = arg.substring(arg.indexOf("=") + 1);
-				if (!testTimeStr.isEmpty()) {
-					testTime = Integer.valueOf(testTimeStr);
-					if (testTime <= 0) {
-						System.err.println("Invalid option: " + arg);
-						System.err.println("Test time needs to be greater than 0");
-						System.exit(1);
-					}
-				}
+				testTime = parsePositiveArgOrDefault(arg, null);
 			} else {
 				System.err.println("Invalid option: " + arg);
 				System.err.println(usage);

src/org/testKitGen/MkGen.java

@@ -132,9 +132,9 @@ private void writeSingleTest(List<String> testsInPlaylist, TestInfo testInfo, Fi
 					f.write(indent
 							+ "@echo \"JVM_OPTIONS: $(JVM_OPTIONS)\" | tee -a $(Q)$(TESTOUTPUT)$(D)TestTargetResult$(Q);\n");
 
-					f.write(indent + "{ ");
+					f.write(indent + "{ success=0; \\\n");
 					for (int k = 1; k <= testInfo.getIterations(); k++) {
-						f.write("itercnt=" + k + "; \\\n" + indent + "$(MKTREE) $(REPORTDIR); \\\n" + indent
+						f.write(indent + "itercnt=" + k + "; \\\n" + indent + "$(MKTREE) $(REPORTDIR); \\\n" + indent
 								+ "$(CD) $(REPORTDIR); \\\n");
 						f.write(indent + testInfo.getCommand() + ";");
 						if (k != testInfo.getIterations()) {

src/org/testKitGen/TestInfo.java

@@ -43,7 +43,7 @@ public TestInfo(Arguments arg) {
 		this.platformRequirementsList = new ArrayList<String>();
 		this.vars = new ArrayList<Variation>();
 		this.aotOptions = "";
-		this.iterations = Integer.parseInt(arg.getIterations());
+		this.iterations = arg.getIterations();
 		this.capabilities = new HashMap<String, String>();
 		this.levelStr = "";
 		this.features = new HashMap<String, String>();

src/org/testKitGen/TestInfoParser.java

@@ -97,6 +97,8 @@ public TestInfo parse() {
 			} else if (entry.getValue().equalsIgnoreCase("applicable")) {
 				if (testFlags.contains("aot") && (entry.getKey().equalsIgnoreCase("aot") || entry.getKey().equalsIgnoreCase("all"))) {
 					ti.setAotOptions("$(AOT_OPTIONS) ");
+					// When TEST_FLAG contains AOT, the times of repetition is the max of aotIterations and iterations parameters.
+					ti.setIterations(Math.max(arg.getAotIterations(), arg.getIterations()));
 				}
 			} else if (entry.getValue().equalsIgnoreCase("nonapplicable")) {
 				// Do not generate make target if the test is not applicable for one feature defined in TEST_FLAG
@@ -105,8 +107,8 @@ public TestInfo parse() {
 				}
 			} else if (entry.getValue().equalsIgnoreCase("explicit")) {
 				if (testFlags.contains("aot") && entry.getKey().equalsIgnoreCase("aot")) {
-					// When test tagged with AOT explicit, its test runs explicitly without any special treatment
-					ti.setIterations(1);
+					// When test tagged with AOT explicit, its test runs explicitly without any special treatment. The times of repetition is depending on iterations parameter.
+					ti.setIterations(arg.getIterations());
 				}
 			} else {
 				System.err.println("Error: Please provide a valid feature parameter in test " + ti.getTestCaseName() + ". The valid string is <feature_name>:[required|applicable|nonapplicable|explicit].");