[GITHUB-2897] Not exception but warning if some (not all) of the give…

CHANGES.txt

@@ -1,4 +1,5 @@
 Current
+New: GITHUB-2897: Not exception but warning if some (not all) of the given test names are not found in suite files. (Bruce Wen)
 Fixed: GITHUB-2888: Skipped Tests with DataProvider appear as failed (Joaquin Moreira)
 Fixed: GITHUB-2884: Discrepancies with DataProvider and Retry of failed tests (Krishnan Mahadevan)
 Fixed: GITHUB-2879: Test listeners specified in parent testng.xml file are not included in testng-failed.xml file (Krishnan Mahadevan)
@@ -1294,7 +1295,7 @@ Fixed: Wasn't parsing <selector-class-name> correctly
 Fixed: Annotation Transformers now work on class-level annotations
 Fixed: Some class-level @Test attributes were not always honored
 Added: Clean separation between @Test invocation events and @Configuration invocation events
-  (see also TESTNG-111)
+  (see also TESTNG-111)
 Added: Test instances created by @Factory now run in multiple threads in parallel mode
 Fixed: @Before/@AfterGroups invocation order
 Fixed: TESTNG-27: Parameters are not used on <test> level anymore
@@ -1462,8 +1463,8 @@ Fixed: TestNGAntTask was hardcoding m_haltOnFSP to true
 Fixed: Passing a null parameter caused an NPE in the reports
 Added: "listener" to the ant task (and documentation)
 Added: if patch-testng-sourcedir.properties is found in the classpath
-  with a property "sourcedir" containing a ; separated list of
-  directories, this list will override -sourcedir.
+  with a property "sourcedir" containing a ; separated list of
+  directories, this list will override -sourcedir.
 
 ===========================================================================
 
@@ -1473,14 +1474,14 @@ Added: if patch-testng-sourcedir.properties is found in the classpath
 Added: Maven 2 plug-in
 Fixed: Message formattings in TestNG assertion utility class
 Fixed: @Factory methods were counted as @Test as well
-  https://jira.opensymphony.com/browse/TESTNG-51
+  https://jira.opensymphony.com/browse/TESTNG-51
 Fixed: All DataProvider parameters were shown in the HTML report
 Fixed: Bug in testng-failed.xml generation
 Fixed: <packages> bug when using a jar file to load the test classes
 Added: alwaysRun for before @Configuration methods
-  https://jira.opensymphony.com/browse/TESTNG-35
+  https://jira.opensymphony.com/browse/TESTNG-35
 Fixed: groupless @Configurations were not invoked if a method depends on a group
-  https://jira.opensymphony.com/browse/TESTNG-45
+  https://jira.opensymphony.com/browse/TESTNG-45
 Added: beforeGroups/afterGroups to @Configuration
 
 Eclipse plugin:
@@ -1621,7 +1622,7 @@ Fixed: Bug in <package> when directories contain spaces in their names
 Fixed: Introduced a JDK5 dependency in the JDK1.4 build (getEnclosingClass())
 Fixed: Output directory in ant task was not honored if it didn't exist
 Fixed: Problem with timeout according to
-  https://forums.opensymphony.com/thread.jspa?threadID=6707
+  https://forums.opensymphony.com/thread.jspa?threadID=6707
 
 Eclipse plug-in:
 
@@ -1631,7 +1632,7 @@ Added: Quick Fix for JUnit conversion (Annotations and JavaDoc)
 Fixed: Methods Run as TestNG test
 Added: Package level Run as TestNG test
 Fixed: Resources from the linked directories are using a wrong path when
-  passed to command line TestNG
+  passed to command line TestNG
 
 IDEA plug-in:
 

testng-ant/src/main/java/org/testng/TestNGAntTask.java

@@ -161,6 +161,7 @@ public class TestNGAntTask extends Task {
  private String m_methods;
  private Mode mode = Mode.testng;
  private boolean forkJvm = true;
+ private boolean m_ignoreMissedTestNames;
 
  public enum Mode {
  // lower-case to better look in build scripts
@@ -360,6 +361,10 @@ public void setTestNames(String testNames) {
  m_testNames = testNames;
  }
 
+ public void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) {
+ m_ignoreMissedTestNames = ignoreMissedTestNames;
+ }
+
  /**
  * Sets the suite runner class to invoke
  *
@@ -578,6 +583,7 @@ protected List<String> createArguments() {
  addStringIfNotBlank(argv, CommandLineArgs.SUITE_NAME, m_suiteName);
  addStringIfNotBlank(argv, CommandLineArgs.TEST_NAME, m_testName);
  addStringIfNotBlank(argv, CommandLineArgs.TEST_NAMES, m_testNames);
+ addBooleanIfTrue(argv, CommandLineArgs.IGNORE_MISSED_TEST_NAMES, m_ignoreMissedTestNames);
  addStringIfNotBlank(argv, CommandLineArgs.METHODS, m_methods);
  addReporterConfigs(argv);
  addIntegerIfNotNull(argv, CommandLineArgs.SUITE_THREAD_POOL_SIZE, m_suiteThreadPoolSize);

testng-core/src/main/java/org/testng/CommandLineArgs.java

@@ -142,6 +142,14 @@ public class CommandLineArgs {
  @Parameter(names = TEST_NAMES, description = "The list of test names to run")
  public String testNames;
 
+ public static final String IGNORE_MISSED_TEST_NAMES = "-ignoreMissedTestNames";
+
+ @Parameters(
+ names = IGNORE_MISSED_TEST_NAMES,
+ description =
+ "Ignore missed test names given by '-testnames' and continue to run existing tests, if any.")
+ public boolean ignoreMissedTestNames = false;
+
  public static final String TEST_JAR = "-testjar";
 
  @Parameter(names = TEST_JAR, description = "A jar file containing the tests")

testng-core/src/main/java/org/testng/JarFileUtils.java

@@ -23,6 +23,7 @@
 class JarFileUtils {
  private final IPostProcessor processor;
  private final String xmlPathInJar;
+ private final boolean ignoreMissedTestNames;
  private final List<String> testNames;
  private final List<XmlSuite> suites = Lists.newLinkedList();
  private final XmlSuite.ParallelMode mode;
@@ -36,10 +37,20 @@ class JarFileUtils {
  String xmlPathInJar,
  List<String> testNames,
  XmlSuite.ParallelMode mode) {
+ this(processor, xmlPathInJar, testNames, mode, false);
+ }
+
+ JarFileUtils(
+ IPostProcessor processor,
+ String xmlPathInJar,
+ List<String> testNames,
+ XmlSuite.ParallelMode mode,
+ final boolean ignoreMissedTestNames) {
  this.processor = processor;
  this.xmlPathInJar = xmlPathInJar;
  this.testNames = testNames;
  this.mode = mode == null ? XmlSuite.ParallelMode.NONE : mode;
+ this.ignoreMissedTestNames = ignoreMissedTestNames;
  }
 
  List<XmlSuite> extractSuitesFrom(File jarFile) {
@@ -96,7 +107,7 @@ private boolean testngXmlExistsInJar(File jarFile, List<String> classes) throws
  // If test names were specified, only run these test names
  if (testNames != null) {
  TestNamesMatcher testNamesMatcher = new TestNamesMatcher(suite, testNames);
- testNamesMatcher.validateMissMatchedTestNames();
+ testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
  suites.addAll(testNamesMatcher.getSuitesMatchingTestNames());
  } else {
  suites.add(suite);

testng-core/src/main/java/org/testng/TestNG.java

@@ -354,7 +354,7 @@ private Collection<XmlSuite> processCommandLineArgs(Collection<XmlSuite> allSuit
  }
  // If test names were specified, only run these test names
  TestNamesMatcher testNamesMatcher = new TestNamesMatcher(s, m_testNames);
- testNamesMatcher.validateMissMatchedTestNames();
+ testNamesMatcher.validateMissMatchedTestNames(m_ignoreMissedTestNames);
  result.addAll(testNamesMatcher.getSuitesMatchingTestNames());
  }
 
@@ -799,6 +799,8 @@ public List<ISuiteListener> getSuiteListeners() {
  /** The list of test names to run from the given suite */
  private List<String> m_testNames;
 
+ private boolean m_ignoreMissedTestNames;
+
  private Integer m_suiteThreadPoolSize = CommandLineArgs.SUITE_THREAD_POOL_SIZE_DEFAULT;
 
  private boolean m_randomizeSuites = Boolean.FALSE;
@@ -1475,6 +1477,7 @@ protected void configure(CommandLineArgs cla) {
 
  if (cla.testNames != null) {
  setTestNames(Arrays.asList(cla.testNames.split(",")));
+ setIgnoreMissedTestNames(cla.ignoreMissedTestNames);
  }
 
  // Note: can't use a Boolean field here because we are allowing a boolean
@@ -1574,6 +1577,10 @@ protected void configure(CommandLineArgs cla) {
  alwaysRunListeners(cla.alwaysRunListeners);
  }
 
+ private void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) {
+ m_ignoreMissedTestNames = ignoreMissedTestNames;
+ }
+
  public void setSuiteThreadPoolSize(Integer suiteThreadPoolSize) {
  m_suiteThreadPoolSize = suiteThreadPoolSize;
  }
@@ -1641,6 +1648,9 @@ public void configure(Map cmdLineArgs) {
  String testNames = (String) cmdLineArgs.get(CommandLineArgs.TEST_NAMES);
  if (testNames != null) {
  result.testNames = testNames;
+ result.ignoreMissedTestNames =
+ (Boolean)
+ cmdLineArgs.getOrDefault(CommandLineArgs.IGNORE_MISSED_TEST_NAMES, Boolean.FALSE);
  }
 
  String useDefaultListeners = (String) cmdLineArgs.get(CommandLineArgs.USE_DEFAULT_LISTENERS);

testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java

@@ -3,12 +3,15 @@
 import java.util.List;
 import org.testng.TestNGException;
 import org.testng.collections.Lists;
+import org.testng.log4testng.Logger;
 import org.testng.xml.XmlSuite;
 import org.testng.xml.XmlTest;
 
 /** The class to work with "-testnames" */
 public final class TestNamesMatcher {
 
+ private static final Logger LOGGER = Logger.getLogger(TestNamesMatcher.class);
+
  private final List<XmlSuite> cloneSuites = Lists.newArrayList();
  private final List<String> matchedTestNames = Lists.newArrayList();
  private final List<XmlTest> matchedTests = Lists.newArrayList();
@@ -43,13 +46,29 @@ public List<XmlSuite> getSuitesMatchingTestNames() {
  return cloneSuites;
  }
 
- public void validateMissMatchedTestNames() {
+ /**
+ * Do validation for testNames and notify users if any testNames are missed in suite.
+ *
+ * @param ignoreMissedTestNames if true print warning message otherwise throw TestNGException for
+ * missed testNames.
+ */
+ public void validateMissMatchedTestNames(final boolean ignoreMissedTestNames) {
+ final List<String> tmpTestNames = getMissedTestNames();
+ if (!tmpTestNames.isEmpty()) {
+ final String errMsg = "The test(s) <" + tmpTestNames + "> cannot be found in suite.";
+ if (ignoreMissedTestNames) {
+ LOGGER.warn(errMsg);
+ } else {
+ throw new TestNGException(errMsg);
+ }
+ }
+ }
+
+ public List<String> getMissedTestNames() {
  List<String> tmpTestNames = Lists.newArrayList();
  tmpTestNames.addAll(testNames);
  tmpTestNames.removeIf(matchedTestNames::contains);
- if (!tmpTestNames.isEmpty()) {
- throw new TestNGException("The test(s) <" + tmpTestNames + "> cannot be found in suite.");
- }
+ return tmpTestNames;
  }
 
  public List<XmlTest> getMatchedTests() {

testng-core/src/test/java/org/testng/xml/internal/TestNamesMatcherTest.java

@@ -90,6 +90,38 @@ public void testCloneIfContainsTestsWithNamesMatchingAnyWithoutMatch() {
  }
  }
 
+ @Test(description = "GITHUB-2897, No exception thrown when ignoreMissedTestNames enabled.")
+ public void testNoExceptionFromValidateWhenIgnoreMissedTestNamesEnabled() {
+ final boolean ignoreMissedTestNames = true;
+ XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2");
+ TestNamesMatcher testNamesMatcher =
+ new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"));
+ testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
+ }
+
+ @Test(
+ description = "GITHUB-2897, Expected exception thrown when ignoreMissedTestNames disabled.",
+ expectedExceptions = TestNGException.class,
+ expectedExceptionsMessageRegExp =
+ "\nThe test\\(s\\) \\<\\[test3\\]\\> cannot be found in suite.")
+ public void testNoExceptionFromValidateWhenIgnoreMissedTestNamesDisabled() {
+ final boolean ignoreMissedTestNames = false;
+ XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2");
+ TestNamesMatcher testNamesMatcher =
+ new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"));
+ testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
+ }
+
+ @Test(description = "GITHUB-2897, Missed test names are found as expected.")
+ public void testMissedTestNamesFound() {
+ XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2");
+ final String expectedMissedTestNames = "test3";
+ TestNamesMatcher testNamesMatcher =
+ new TestNamesMatcher(xmlSuite, Collections.singletonList(expectedMissedTestNames));
+ List<String> missedTestNames = testNamesMatcher.getMissedTestNames();
+ assertThat(missedTestNames).hasSameElementsAs(Arrays.asList(expectedMissedTestNames));
+ }
+
  @DataProvider(name = "getTestnames")
  public Object[][] getTestnameToSearchFor() {
  return new Object[][] {