[GITHUB-2897] Not exception but warning if some (not all) of the give…

CHANGES.txt

@@ -1,4 +1,5 @@
 Current
+New: GITHUB-2897: Not exception but warning if some (not all) of the given test names are not found in suite files. (Bruce Wen)
 New: Added assertListContains and assertListContainsObject methods to check if specific object present in List (Dmytro Budym)
 Fixed: GITHUB-2888: Skipped Tests with DataProvider appear as failed (Joaquin Moreira)
 Fixed: GITHUB-2884: Discrepancies with DataProvider and Retry of failed tests (Krishnan Mahadevan)

testng-ant/src/main/java/org/testng/TestNGAntTask.java

@@ -161,6 +161,7 @@ public class TestNGAntTask extends Task {
   private String m_methods;
   private Mode mode = Mode.testng;
   private boolean forkJvm = true;
+  private boolean m_ignoreMissedTestNames;
 
   public enum Mode {
     // lower-case to better look in build scripts
@@ -360,6 +361,10 @@ public void setTestNames(String testNames) {
     m_testNames = testNames;
   }
 
+  public void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) {
+    m_ignoreMissedTestNames = ignoreMissedTestNames;
+  }
+
   /**
    * Sets the suite runner class to invoke
    *
@@ -578,6 +583,7 @@ protected List<String> createArguments() {
     addStringIfNotBlank(argv, CommandLineArgs.SUITE_NAME, m_suiteName);
     addStringIfNotBlank(argv, CommandLineArgs.TEST_NAME, m_testName);
     addStringIfNotBlank(argv, CommandLineArgs.TEST_NAMES, m_testNames);
+    addBooleanIfTrue(argv, CommandLineArgs.IGNORE_MISSED_TEST_NAMES, m_ignoreMissedTestNames);
     addStringIfNotBlank(argv, CommandLineArgs.METHODS, m_methods);
     addReporterConfigs(argv);
     addIntegerIfNotNull(argv, CommandLineArgs.SUITE_THREAD_POOL_SIZE, m_suiteThreadPoolSize);

testng-collections/src/main/java/org/testng/util/Strings.java

@@ -1,5 +1,6 @@
 package org.testng.util;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -18,6 +19,23 @@ public static boolean isNotNullAndNotEmpty(String string) {
     return !isNullOrEmpty(string);
   }
 
+  /**
+   * Check if the given string list is null or empty or all elements are null or empty or blank.
+   *
+   * @param list A list instance with String elements.
+   * @return true if the given string list is null or empty or all elements are null or empty or
+   *     blank; otherwise false.
+   */
+  public static boolean isBlankStringList(List<String> list) {
+    if (list == null) {
+      return true;
+    }
+    if (list.isEmpty()) {
+      return true;
+    }
+    return list.stream().allMatch(t -> t == null || t.isBlank());
+  }
+
   private static final Map<String, String> ESCAPE_HTML_MAP = Maps.newLinkedHashMap();
 
   static {

testng-core/src/main/java/org/testng/CommandLineArgs.java

@@ -142,6 +142,14 @@ public class CommandLineArgs {
   @Parameter(names = TEST_NAMES, description = "The list of test names to run")
   public String testNames;
 
+  public static final String IGNORE_MISSED_TEST_NAMES = "-ignoreMissedTestNames";
+
+  @Parameter(
+      names = IGNORE_MISSED_TEST_NAMES,
+      description =
+          "Ignore missed test names given by '-testnames' and continue to run existing tests, if any.")
+  public boolean ignoreMissedTestNames = false;
+
   public static final String TEST_JAR = "-testjar";
 
   @Parameter(names = TEST_JAR, description = "A jar file containing the tests")

testng-core/src/main/java/org/testng/JarFileUtils.java

@@ -23,6 +23,7 @@
 class JarFileUtils {
   private final IPostProcessor processor;
   private final String xmlPathInJar;
+  private final boolean ignoreMissedTestNames;
   private final List<String> testNames;
   private final List<XmlSuite> suites = Lists.newLinkedList();
   private final XmlSuite.ParallelMode mode;
@@ -36,10 +37,28 @@ class JarFileUtils {
       String xmlPathInJar,
       List<String> testNames,
       XmlSuite.ParallelMode mode) {
+    this(processor, xmlPathInJar, testNames, mode, false);
+  }
+
+  JarFileUtils(
+      IPostProcessor processor,
+      String xmlPathInJar,
+      List<String> testNames,
+      boolean ignoreMissedTestNames) {
+    this(processor, xmlPathInJar, testNames, XmlSuite.ParallelMode.NONE, ignoreMissedTestNames);
+  }
+
+  JarFileUtils(
+      IPostProcessor processor,
+      String xmlPathInJar,
+      List<String> testNames,
+      XmlSuite.ParallelMode mode,
+      boolean ignoreMissedTestNames) {
     this.processor = processor;
     this.xmlPathInJar = xmlPathInJar;
     this.testNames = testNames;
     this.mode = mode == null ? XmlSuite.ParallelMode.NONE : mode;
+    this.ignoreMissedTestNames = ignoreMissedTestNames;
   }
 
   List<XmlSuite> extractSuitesFrom(File jarFile) {
@@ -69,6 +88,7 @@ private boolean testngXmlExistsInJar(File jarFile, List<String> classes) throws
       Enumeration<JarEntry> entries = jf.entries();
       File file = java.nio.file.Files.createTempDirectory("testngXmlPathInJar-").toFile();
       String suitePath = null;
+
       while (entries.hasMoreElements()) {
         JarEntry je = entries.nextElement();
         String jeName = je.getName();
@@ -87,24 +107,34 @@ private boolean testngXmlExistsInJar(File jarFile, List<String> classes) throws
           classes.add(constructClassName(je));
         }
       }
+
       if (Strings.isNullOrEmpty(suitePath)) {
+        Utils.log("TestNG", 1, String.format("Not found '%s' in '%s'.", xmlPathInJar, jarFile));
         return false;
       }
+
       Collection<XmlSuite> parsedSuites = Parser.parse(suitePath, processor);
       delete(file);
+      boolean addedSuite = false;
       for (XmlSuite suite : parsedSuites) {
-        // If test names were specified, only run these test names
-        if (testNames != null) {
-          TestNamesMatcher testNamesMatcher = new TestNamesMatcher(suite, testNames);
-          testNamesMatcher.validateMissMatchedTestNames();
-          suites.addAll(testNamesMatcher.getSuitesMatchingTestNames());
-        } else {
+        if (testNames == null) {
           suites.add(suite);
+          addedSuite = true;
+        } else {
+          TestNamesMatcher testNamesMatcher =
+              new TestNamesMatcher(suite, testNames, ignoreMissedTestNames);
+          boolean validationResult = testNamesMatcher.validateMissMatchedTestNames();
+          if (validationResult) {
+            suites.addAll(testNamesMatcher.getSuitesMatchingTestNames());
+            addedSuite = true;
+          } else {
+            Utils.error(String.format("None of '%s' found in '%s'.", testNames, suite));
+          }
         }
-        return true;
       }
+
+      return addedSuite;
     }
-    return false;
   }
 
   private void delete(File f) throws IOException {

testng-core/src/main/java/org/testng/TestNG.java

@@ -353,7 +353,8 @@ private Collection<XmlSuite> processCommandLineArgs(Collection<XmlSuite> allSuit
         continue;
       }
       // If test names were specified, only run these test names
-      TestNamesMatcher testNamesMatcher = new TestNamesMatcher(s, m_testNames);
+      TestNamesMatcher testNamesMatcher =
+          new TestNamesMatcher(s, m_testNames, m_ignoreMissedTestNames);
       testNamesMatcher.validateMissMatchedTestNames();
       result.addAll(testNamesMatcher.getSuitesMatchingTestNames());
     }
@@ -417,7 +418,8 @@ public void initializeSuitesAndJarFile() {
     File jarFile = new File(m_jarPath);
 
     JarFileUtils utils =
-        new JarFileUtils(getProcessor(), m_xmlPathInJar, m_testNames, m_parallelMode);
+        new JarFileUtils(
+            getProcessor(), m_xmlPathInJar, m_testNames, m_parallelMode, m_ignoreMissedTestNames);
 
     Collection<XmlSuite> allSuites = utils.extractSuitesFrom(jarFile);
     allSuites.forEach(this::processParallelModeCommandLineArgs);
@@ -799,6 +801,8 @@ public List<ISuiteListener> getSuiteListeners() {
   /** The list of test names to run from the given suite */
   private List<String> m_testNames;
 
+  private boolean m_ignoreMissedTestNames;
+
   private Integer m_suiteThreadPoolSize = CommandLineArgs.SUITE_THREAD_POOL_SIZE_DEFAULT;
 
   private boolean m_randomizeSuites = Boolean.FALSE;
@@ -1475,6 +1479,7 @@ protected void configure(CommandLineArgs cla) {
 
     if (cla.testNames != null) {
       setTestNames(Arrays.asList(cla.testNames.split(",")));
+      setIgnoreMissedTestNames(cla.ignoreMissedTestNames);
     }
 
     // Note: can't use a Boolean field here because we are allowing a boolean
@@ -1574,6 +1579,10 @@ protected void configure(CommandLineArgs cla) {
     alwaysRunListeners(cla.alwaysRunListeners);
   }
 
+  private void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) {
+    m_ignoreMissedTestNames = ignoreMissedTestNames;
+  }
+
   public void setSuiteThreadPoolSize(Integer suiteThreadPoolSize) {
     m_suiteThreadPoolSize = suiteThreadPoolSize;
   }

testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java

@@ -3,19 +3,35 @@
 import java.util.List;
 import org.testng.TestNGException;
 import org.testng.collections.Lists;
+import org.testng.log4testng.Logger;
+import org.testng.util.Strings;
 import org.testng.xml.XmlSuite;
 import org.testng.xml.XmlTest;
 
-/** The class to work with "-testnames" */
+/**
+ * The class to work with "-testnames", "-ignoreMissedTestNames", and VM argument
+ * "-Dtestng.ignore.missed.testnames". If both "-ignoreMissedTestNames" and VM argument
+ * "-Dtestng.ignore.missed.testnames" are set, then either of them has "true" value will enable the
+ * feature to ingore partially missed test names and run those existing test names.
+ */
 public final class TestNamesMatcher {
 
+  private static final Logger LOGGER = Logger.getLogger(TestNamesMatcher.class);
+
   private final List<XmlSuite> cloneSuites = Lists.newArrayList();
   private final List<String> matchedTestNames = Lists.newArrayList();
   private final List<XmlTest> matchedTests = Lists.newArrayList();
   private final List<String> testNames;
+  private final boolean ignoreMissedTestNames;
 
   public TestNamesMatcher(XmlSuite xmlSuite, List<String> testNames) {
+    this(xmlSuite, testNames, false);
+  }
+
+  public TestNamesMatcher(
+      XmlSuite xmlSuite, List<String> testNames, boolean ignoreMissedTestNames) {
     this.testNames = testNames;
+    this.ignoreMissedTestNames = ignoreMissedTestNames;
     cloneIfContainsTestsWithNamesMatchingAny(xmlSuite, this.testNames);
   }
 
@@ -26,7 +42,7 @@ public TestNamesMatcher(XmlSuite xmlSuite, List<String> testNames) {
    * @param testNames The list of testnames to iterate through
    */
   private void cloneIfContainsTestsWithNamesMatchingAny(XmlSuite xmlSuite, List<String> testNames) {
-    if (testNames == null || testNames.isEmpty()) {
+    if (Strings.isBlankStringList(testNames)) {
       throw new TestNGException("Please provide a valid list of names to check.");
     }
 
@@ -43,13 +59,38 @@ public List<XmlSuite> getSuitesMatchingTestNames() {
     return cloneSuites;
   }
 
-  public void validateMissMatchedTestNames() {
-    List<String> tmpTestNames = Lists.newArrayList();
-    tmpTestNames.addAll(testNames);
-    tmpTestNames.removeIf(matchedTestNames::contains);
-    if (!tmpTestNames.isEmpty()) {
-      throw new TestNGException("The test(s) <" + tmpTestNames + "> cannot be found in suite.");
+  /**
+   * Do validation for testNames and notify users if any testNames are missed in suite. This method
+   * is also used to decide how to run test suite when test names are given. In legacy logic, if
+   * test names are given and exist in suite, then run them; if any of them do not exist in suite,
+   * then throw exception and exit. After ignoreMissedTestNames is introduced, if
+   * ignoreMissedTestNames is enabled, then any of the given test names exist in suite will be run,
+   * and print warning message to tell those test names do not exist in suite.
+   *
+   * @return boolean if ignoreMissedTestNames disabled, then return true if no missed test names in
+   *     suite, otherwise throw TestNGException; if ignoreMissedTestNames enabled, then return true
+   *     if any test names exist in suite, otehrwise (all given test names are missed) throw
+   *     TestNGException.
+   */
+  public boolean validateMissMatchedTestNames() {
+    final List<String> missedTestNames = getMissedTestNames();
+    if (!missedTestNames.isEmpty()) {
+      final String errMsg = "The test(s) <" + missedTestNames + "> cannot be found in suite.";
+      if (ignoreMissedTestNames && !matchedTestNames.isEmpty()) {
+        LOGGER.warn(errMsg);
+        return true;
+      } else {
+        throw new TestNGException(errMsg);
+      }
     }
+    return missedTestNames.isEmpty() && !matchedTestNames.isEmpty();
+  }
+
+  public List<String> getMissedTestNames() {
+    List<String> missedTestNames = Lists.newArrayList();
+    missedTestNames.addAll(testNames);
+    missedTestNames.removeIf(matchedTestNames::contains);
+    return missedTestNames;
   }
 
   public List<XmlTest> getMatchedTests() {