diff --git a/CHANGES.txt b/CHANGES.txt index 01803e313c..3901cdcf6e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ Current +New: GITHUB-2897: Not exception but warning if some (not all) of the given test names are not found in suite files. (Bruce Wen) New: Added assertListContains and assertListContainsObject methods to check if specific object present in List (Dmytro Budym) Fixed: GITHUB-2888: Skipped Tests with DataProvider appear as failed (Joaquin Moreira) Fixed: GITHUB-2884: Discrepancies with DataProvider and Retry of failed tests (Krishnan Mahadevan) diff --git a/testng-ant/src/main/java/org/testng/TestNGAntTask.java b/testng-ant/src/main/java/org/testng/TestNGAntTask.java index 412b796ef9..6ef5967d94 100644 --- a/testng-ant/src/main/java/org/testng/TestNGAntTask.java +++ b/testng-ant/src/main/java/org/testng/TestNGAntTask.java @@ -161,6 +161,7 @@ public class TestNGAntTask extends Task { private String m_methods; private Mode mode = Mode.testng; private boolean forkJvm = true; + private boolean m_ignoreMissedTestNames; public enum Mode { // lower-case to better look in build scripts @@ -360,6 +361,10 @@ public void setTestNames(String testNames) { m_testNames = testNames; } + public void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) { + m_ignoreMissedTestNames = ignoreMissedTestNames; + } + /** * Sets the suite runner class to invoke * @@ -578,6 +583,7 @@ protected List createArguments() { addStringIfNotBlank(argv, CommandLineArgs.SUITE_NAME, m_suiteName); addStringIfNotBlank(argv, CommandLineArgs.TEST_NAME, m_testName); addStringIfNotBlank(argv, CommandLineArgs.TEST_NAMES, m_testNames); + addBooleanIfTrue(argv, CommandLineArgs.IGNORE_MISSED_TEST_NAMES, m_ignoreMissedTestNames); addStringIfNotBlank(argv, CommandLineArgs.METHODS, m_methods); addReporterConfigs(argv); addIntegerIfNotNull(argv, CommandLineArgs.SUITE_THREAD_POOL_SIZE, m_suiteThreadPoolSize); diff --git a/testng-collections/src/main/java/org/testng/util/Strings.java b/testng-collections/src/main/java/org/testng/util/Strings.java index 8f8f3f55a5..452f56f29d 100644 --- a/testng-collections/src/main/java/org/testng/util/Strings.java +++ b/testng-collections/src/main/java/org/testng/util/Strings.java @@ -1,5 +1,6 @@ package org.testng.util; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -18,6 +19,23 @@ public static boolean isNotNullAndNotEmpty(String string) { return !isNullOrEmpty(string); } + /** + * Check if the given string list is null or empty or all elements are null or empty or blank. + * + * @param list A list instance with String elements. + * @return true if the given string list is null or empty or all elements are null or empty or + * blank; otherwise false. + */ + public static boolean isBlankStringList(List list) { + if (list == null) { + return true; + } + if (list.isEmpty()) { + return true; + } + return list.stream().allMatch(t -> t == null || t.isBlank()); + } + private static final Map ESCAPE_HTML_MAP = Maps.newLinkedHashMap(); static { diff --git a/testng-core/src/main/java/org/testng/CommandLineArgs.java b/testng-core/src/main/java/org/testng/CommandLineArgs.java index 03866b2f29..f696de1d04 100644 --- a/testng-core/src/main/java/org/testng/CommandLineArgs.java +++ b/testng-core/src/main/java/org/testng/CommandLineArgs.java @@ -142,6 +142,14 @@ public class CommandLineArgs { @Parameter(names = TEST_NAMES, description = "The list of test names to run") public String testNames; + public static final String IGNORE_MISSED_TEST_NAMES = "-ignoreMissedTestNames"; + + @Parameter( + names = IGNORE_MISSED_TEST_NAMES, + description = + "Ignore missed test names given by '-testnames' and continue to run existing tests, if any.") + public boolean ignoreMissedTestNames = false; + public static final String TEST_JAR = "-testjar"; @Parameter(names = TEST_JAR, description = "A jar file containing the tests") diff --git a/testng-core/src/main/java/org/testng/JarFileUtils.java b/testng-core/src/main/java/org/testng/JarFileUtils.java index 176881bc2f..377ee3f700 100644 --- a/testng-core/src/main/java/org/testng/JarFileUtils.java +++ b/testng-core/src/main/java/org/testng/JarFileUtils.java @@ -23,6 +23,7 @@ class JarFileUtils { private final IPostProcessor processor; private final String xmlPathInJar; + private final boolean ignoreMissedTestNames; private final List testNames; private final List suites = Lists.newLinkedList(); private final XmlSuite.ParallelMode mode; @@ -36,10 +37,28 @@ class JarFileUtils { String xmlPathInJar, List testNames, XmlSuite.ParallelMode mode) { + this(processor, xmlPathInJar, testNames, mode, false); + } + + JarFileUtils( + IPostProcessor processor, + String xmlPathInJar, + List testNames, + boolean ignoreMissedTestNames) { + this(processor, xmlPathInJar, testNames, XmlSuite.ParallelMode.NONE, ignoreMissedTestNames); + } + + JarFileUtils( + IPostProcessor processor, + String xmlPathInJar, + List testNames, + XmlSuite.ParallelMode mode, + boolean ignoreMissedTestNames) { this.processor = processor; this.xmlPathInJar = xmlPathInJar; this.testNames = testNames; this.mode = mode == null ? XmlSuite.ParallelMode.NONE : mode; + this.ignoreMissedTestNames = ignoreMissedTestNames; } List extractSuitesFrom(File jarFile) { @@ -69,6 +88,7 @@ private boolean testngXmlExistsInJar(File jarFile, List classes) throws Enumeration entries = jf.entries(); File file = java.nio.file.Files.createTempDirectory("testngXmlPathInJar-").toFile(); String suitePath = null; + while (entries.hasMoreElements()) { JarEntry je = entries.nextElement(); String jeName = je.getName(); @@ -87,24 +107,34 @@ private boolean testngXmlExistsInJar(File jarFile, List classes) throws classes.add(constructClassName(je)); } } + if (Strings.isNullOrEmpty(suitePath)) { + Utils.log("TestNG", 1, String.format("Not found '%s' in '%s'.", xmlPathInJar, jarFile)); return false; } + Collection parsedSuites = Parser.parse(suitePath, processor); delete(file); + boolean addedSuite = false; for (XmlSuite suite : parsedSuites) { - // If test names were specified, only run these test names - if (testNames != null) { - TestNamesMatcher testNamesMatcher = new TestNamesMatcher(suite, testNames); - testNamesMatcher.validateMissMatchedTestNames(); - suites.addAll(testNamesMatcher.getSuitesMatchingTestNames()); - } else { + if (testNames == null) { suites.add(suite); + addedSuite = true; + } else { + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(suite, testNames, ignoreMissedTestNames); + boolean validationResult = testNamesMatcher.validateMissMatchedTestNames(); + if (validationResult) { + suites.addAll(testNamesMatcher.getSuitesMatchingTestNames()); + addedSuite = true; + } else { + Utils.error(String.format("None of '%s' found in '%s'.", testNames, suite)); + } } - return true; } + + return addedSuite; } - return false; } private void delete(File f) throws IOException { diff --git a/testng-core/src/main/java/org/testng/TestNG.java b/testng-core/src/main/java/org/testng/TestNG.java index 2fa11a71c7..339b1b6bfe 100644 --- a/testng-core/src/main/java/org/testng/TestNG.java +++ b/testng-core/src/main/java/org/testng/TestNG.java @@ -353,7 +353,8 @@ private Collection processCommandLineArgs(Collection allSuit continue; } // If test names were specified, only run these test names - TestNamesMatcher testNamesMatcher = new TestNamesMatcher(s, m_testNames); + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(s, m_testNames, m_ignoreMissedTestNames); testNamesMatcher.validateMissMatchedTestNames(); result.addAll(testNamesMatcher.getSuitesMatchingTestNames()); } @@ -417,7 +418,8 @@ public void initializeSuitesAndJarFile() { File jarFile = new File(m_jarPath); JarFileUtils utils = - new JarFileUtils(getProcessor(), m_xmlPathInJar, m_testNames, m_parallelMode); + new JarFileUtils( + getProcessor(), m_xmlPathInJar, m_testNames, m_parallelMode, m_ignoreMissedTestNames); Collection allSuites = utils.extractSuitesFrom(jarFile); allSuites.forEach(this::processParallelModeCommandLineArgs); @@ -799,6 +801,8 @@ public List getSuiteListeners() { /** The list of test names to run from the given suite */ private List m_testNames; + private boolean m_ignoreMissedTestNames; + private Integer m_suiteThreadPoolSize = CommandLineArgs.SUITE_THREAD_POOL_SIZE_DEFAULT; private boolean m_randomizeSuites = Boolean.FALSE; @@ -1475,6 +1479,7 @@ protected void configure(CommandLineArgs cla) { if (cla.testNames != null) { setTestNames(Arrays.asList(cla.testNames.split(","))); + setIgnoreMissedTestNames(cla.ignoreMissedTestNames); } // Note: can't use a Boolean field here because we are allowing a boolean @@ -1574,6 +1579,10 @@ protected void configure(CommandLineArgs cla) { alwaysRunListeners(cla.alwaysRunListeners); } + private void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) { + m_ignoreMissedTestNames = ignoreMissedTestNames; + } + public void setSuiteThreadPoolSize(Integer suiteThreadPoolSize) { m_suiteThreadPoolSize = suiteThreadPoolSize; } diff --git a/testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java b/testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java index 9d0967f854..507f358699 100644 --- a/testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java +++ b/testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java @@ -3,19 +3,35 @@ import java.util.List; import org.testng.TestNGException; import org.testng.collections.Lists; +import org.testng.log4testng.Logger; +import org.testng.util.Strings; import org.testng.xml.XmlSuite; import org.testng.xml.XmlTest; -/** The class to work with "-testnames" */ +/** + * The class to work with "-testnames", "-ignoreMissedTestNames", and VM argument + * "-Dtestng.ignore.missed.testnames". If both "-ignoreMissedTestNames" and VM argument + * "-Dtestng.ignore.missed.testnames" are set, then either of them has "true" value will enable the + * feature to ingore partially missed test names and run those existing test names. + */ public final class TestNamesMatcher { + private static final Logger LOGGER = Logger.getLogger(TestNamesMatcher.class); + private final List cloneSuites = Lists.newArrayList(); private final List matchedTestNames = Lists.newArrayList(); private final List matchedTests = Lists.newArrayList(); private final List testNames; + private final boolean ignoreMissedTestNames; public TestNamesMatcher(XmlSuite xmlSuite, List testNames) { + this(xmlSuite, testNames, false); + } + + public TestNamesMatcher( + XmlSuite xmlSuite, List testNames, boolean ignoreMissedTestNames) { this.testNames = testNames; + this.ignoreMissedTestNames = ignoreMissedTestNames; cloneIfContainsTestsWithNamesMatchingAny(xmlSuite, this.testNames); } @@ -26,7 +42,7 @@ public TestNamesMatcher(XmlSuite xmlSuite, List testNames) { * @param testNames The list of testnames to iterate through */ private void cloneIfContainsTestsWithNamesMatchingAny(XmlSuite xmlSuite, List testNames) { - if (testNames == null || testNames.isEmpty()) { + if (Strings.isBlankStringList(testNames)) { throw new TestNGException("Please provide a valid list of names to check."); } @@ -43,13 +59,38 @@ public List getSuitesMatchingTestNames() { return cloneSuites; } - public void validateMissMatchedTestNames() { - List tmpTestNames = Lists.newArrayList(); - tmpTestNames.addAll(testNames); - tmpTestNames.removeIf(matchedTestNames::contains); - if (!tmpTestNames.isEmpty()) { - throw new TestNGException("The test(s) <" + tmpTestNames + "> cannot be found in suite."); + /** + * Do validation for testNames and notify users if any testNames are missed in suite. This method + * is also used to decide how to run test suite when test names are given. In legacy logic, if + * test names are given and exist in suite, then run them; if any of them do not exist in suite, + * then throw exception and exit. After ignoreMissedTestNames is introduced, if + * ignoreMissedTestNames is enabled, then any of the given test names exist in suite will be run, + * and print warning message to tell those test names do not exist in suite. + * + * @return boolean if ignoreMissedTestNames disabled, then return true if no missed test names in + * suite, otherwise throw TestNGException; if ignoreMissedTestNames enabled, then return true + * if any test names exist in suite, otehrwise (all given test names are missed) throw + * TestNGException. + */ + public boolean validateMissMatchedTestNames() { + final List missedTestNames = getMissedTestNames(); + if (!missedTestNames.isEmpty()) { + final String errMsg = "The test(s) <" + missedTestNames + "> cannot be found in suite."; + if (ignoreMissedTestNames && !matchedTestNames.isEmpty()) { + LOGGER.warn(errMsg); + return true; + } else { + throw new TestNGException(errMsg); + } } + return missedTestNames.isEmpty() && !matchedTestNames.isEmpty(); + } + + public List getMissedTestNames() { + List missedTestNames = Lists.newArrayList(); + missedTestNames.addAll(testNames); + missedTestNames.removeIf(matchedTestNames::contains); + return missedTestNames; } public List getMatchedTests() { diff --git a/testng-core/src/test/java/org/testng/JarFileUtilsTest.java b/testng-core/src/test/java/org/testng/JarFileUtilsTest.java index 5e3816ec9a..3d3afee6eb 100644 --- a/testng-core/src/test/java/org/testng/JarFileUtilsTest.java +++ b/testng-core/src/test/java/org/testng/JarFileUtilsTest.java @@ -21,6 +21,7 @@ import org.testng.xml.XmlTest; public class JarFileUtilsTest { + private static File jar = null; @BeforeClass @@ -65,6 +66,42 @@ public void testWithInvalidTestNames() throws MalformedURLException { new String[] {"org.testng.jarfileutils.org.testng.SampleTest1"}); } + @Test( + description = + "GITHUB-2897, Have TestNGException thrown when ignoreMissedTestNames enabled and ALL given test names are invalid.", + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = + "\nThe test\\(s\\) <\\[testng-tests-child11\\]> cannot be found in suite.") + public void testWithAllInvalidTestNamesNoExceptionIfIgnoreMissedTestNamesEnabled() + throws MalformedURLException { + boolean ignoreMissedTestNames = true; + JarFileUtils utils = + newJarFileUtils(Collections.singletonList("testng-tests-child11"), ignoreMissedTestNames); + runTest(utils, 1, null, null, "Jar suite"); + } + + @Test( + description = + "GITHUB-2897, No TestNGException thrown when ignoreMissedTestNames enabled and partial given test names are invalid and partial valid test names are expected to run.") + public void testWithPartialInvalidTestNamesNoExceptionIfIgnoreMissedTestNamesEnabled() + throws MalformedURLException { + String[] expectedTestNames = + new String[] {"testng-tests-child2", "testng-tests-child4", "testng-tests-child5"}; + String[] expectedClassNames = + new String[] { + "org.testng.jarfileutils.org.testng.SampleTest2", + "org.testng.jarfileutils.org.testng.SampleTest4", + "org.testng.jarfileutils.org.testng.SampleTest5" + }; + List testNames = + Arrays.asList( + "testng-tests-child2", "testng-tests-child4", "testng-tests-child5", "invalid"); + boolean ignoreMissedTestNames = true; + JarFileUtils utils = newJarFileUtils(testNames, ignoreMissedTestNames); + // 3 tests from 3 suites, the first suite has one test is given + runTest(utils, 1, 3, expectedTestNames, expectedClassNames, "testng-tests-suite"); + } + @Test public void testWithInvalidXmlFile() throws MalformedURLException { JarFileUtils utils = @@ -89,11 +126,32 @@ public void testWithInvalidXmlFile() throws MalformedURLException { */ @Test public void ensureThatExceptionAreNotThrown() throws MalformedURLException { + TestNG testNg = new TestNG(false); + List testNames = + Arrays.asList("testng-tests-child2", "testng-tests-child4", "testng-tests-child5"); + testNg.setTestNames(testNames); + testNg.setXmlPathInJar("jarfileutils/testng-tests.xml"); + testNg.setTestJar(jar.getAbsolutePath()); + testNg.initializeSuitesAndJarFile(); + // "testng-tests-child2", "testng-tests-child4", "testng-tests-child5" are from 3 different test + // suites + Assert.assertEquals(testNg.m_suites.size(), 3); + } + + /** + * Test to ensure that exception is thrown for invalid test name. + * + * @throws MalformedURLException + */ + @Test( + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = "\nThe test\\(s\\) <\\[dummy\\]> cannot be found in suite.") + public void ensureThatExceptionAreThrown() throws MalformedURLException { TestNG testNg = new TestNG(false); List testNames = Arrays.asList("testng-tests-child2", "testng-tests-child4", "testng-tests-child5", "dummy"); testNg.setTestNames(testNames); - testNg.setXmlPathInJar(jar.getAbsolutePath()); + testNg.setXmlPathInJar("jarfileutils/testng-tests.xml"); testNg.setTestJar(jar.getAbsolutePath()); testNg.initializeSuitesAndJarFile(); Assert.assertEquals(testNg.m_suites.size(), 1); @@ -118,20 +176,20 @@ public void testWithValidTestNamesFromMultiChildSuites() throws MalformedURLExce assertThat(suite.getName()).isEqualTo("testng-tests-suite"); List testNames = new LinkedList<>(); List classNames = new LinkedList<>(); - for (XmlSuite xmlSuite : suites) { - extractClassNames(xmlSuite, testNames, classNames); - } + extractClassNames(suites, testNames, classNames); assertThat(testNames).containsExactly(expectedTestNames); assertThat(classNames).contains(expectedClassNames); } private static void extractClassNames( - XmlSuite xmlSuite, List testNames, List classNames) { - for (XmlTest xmlTest : xmlSuite.getTests()) { - testNames.add(xmlTest.getName()); - for (XmlClass xmlClass : xmlTest.getXmlClasses()) { - classNames.add(xmlClass.getName()); + List xmlSuites, List testNames, List classNames) { + for (XmlSuite xmlSuite : xmlSuites) { + for (XmlTest xmlTest : xmlSuite.getTests()) { + testNames.add(xmlTest.getName()); + for (XmlClass xmlClass : xmlTest.getXmlClasses()) { + classNames.add(xmlClass.getName()); + } } } } @@ -150,18 +208,30 @@ private static void runTest( String[] expectedTestNames, String[] expectedClassNames, String expectedSuiteName) { + runTest(utils, numberOfTests, 1, expectedTestNames, expectedClassNames, expectedSuiteName); + } + + private static void runTest( + JarFileUtils utils, + int numberOfTests, + int expectedSuiteTotal, + String[] expectedTestNames, + String[] expectedClassNames, + String expectedSuiteName) { List suites = utils.extractSuitesFrom(jar); - assertThat(suites).hasSize(1); + assertThat(suites).hasSize(expectedSuiteTotal); XmlSuite suite = suites.get(0); assertThat(suite.getName()).isEqualTo(expectedSuiteName); assertThat(suite.getTests()).hasSize(numberOfTests); List testNames = new LinkedList<>(); List classNames = new LinkedList<>(); - extractClassNames(suite, testNames, classNames); + extractClassNames(suites, testNames, classNames); if (expectedTestNames != null) { assertThat(testNames).containsExactly(expectedTestNames); } - assertThat(classNames).contains(expectedClassNames); + if (expectedClassNames != null) { + assertThat(classNames).contains(expectedClassNames); + } } public static class FakeProcessor implements IPostProcessor { @@ -173,15 +243,26 @@ public Collection process(Collection suites) { } private static JarFileUtils newJarFileUtils(List testNames) throws MalformedURLException { - return newJarFileUtils("jarfileutils/testng-tests.xml", testNames); + return newJarFileUtils(testNames, false); + } + + private static JarFileUtils newJarFileUtils(List testNames, boolean ignoreMissedTestNames) + throws MalformedURLException { + return newJarFileUtils("jarfileutils/testng-tests.xml", testNames, ignoreMissedTestNames); } private static JarFileUtils newJarFileUtils(String suiteXmlName, List testNames) throws MalformedURLException { + return newJarFileUtils(suiteXmlName, testNames, false); + } + + private static JarFileUtils newJarFileUtils( + String suiteXmlName, List testNames, boolean ignoreMissedTestNames) + throws MalformedURLException { URL url = jar.toURI().toURL(); URLClassLoader classLoader = new URLClassLoader(new URL[] {url}, ClassLoader.getSystemClassLoader()); Thread.currentThread().setContextClassLoader(classLoader); - return new JarFileUtils(new FakeProcessor(), suiteXmlName, testNames); + return new JarFileUtils(new FakeProcessor(), suiteXmlName, testNames, ignoreMissedTestNames); } } diff --git a/testng-core/src/test/java/org/testng/xml/internal/TestNamesMatcherTest.java b/testng-core/src/test/java/org/testng/xml/internal/TestNamesMatcherTest.java index 712fa6ce00..7e168576b9 100644 --- a/testng-core/src/test/java/org/testng/xml/internal/TestNamesMatcherTest.java +++ b/testng-core/src/test/java/org/testng/xml/internal/TestNamesMatcherTest.java @@ -57,6 +57,17 @@ public void testCloneIfContainsTestsWithNamesMatchingAnyNegativeCondition( TestNamesMatcher testNamesHelper = new TestNamesMatcher(xmlSuite, names); } + @Test( + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = "\nPlease provide a valid list of names to check.", + dataProvider = "getData") + public void + testCloneIfContainsTestsWithNamesMatchingAnyNegativeConditionWithIgnoreMissedTestNamesEnabled( + XmlSuite xmlSuite, List names) { + boolean ignoreMissedTestNames = true; + TestNamesMatcher testNamesHelper = new TestNamesMatcher(xmlSuite, names, ignoreMissedTestNames); + } + @Test public void testIfTestnamesComesFromDifferentSuite() { XmlSuite parentSuite = createDummySuiteWithTestNamesAs("test1", "test2"); @@ -90,6 +101,69 @@ public void testCloneIfContainsTestsWithNamesMatchingAnyWithoutMatch() { } } + @Test( + description = + "GITHUB-2897, No TestNGException thrown when ignoreMissedTestNames enabled and only partial of the given test names are invalid.") + public void testNoExceptionWhenIgnoreMissedTestNamesEnabled() { + final boolean ignoreMissedTestNames = true; + XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2"); + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(xmlSuite, Arrays.asList("test2", "test3"), ignoreMissedTestNames); + testNamesMatcher.validateMissMatchedTestNames(); + } + + @Test( + description = + "GITHUB-2897, Expected TestNGException thrown when ignoreMissedTestNames enabled and all given test names are invalid.", + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = + "\nThe test\\(s\\) \\<\\[test3\\]\\> cannot be found in suite.") + public void testHaveExceptionWhenIgnoreMissedTestNamesEnabledWithAllInvalidTestNames() { + final boolean ignoreMissedTestNames = true; + XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2"); + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"), ignoreMissedTestNames); + testNamesMatcher.validateMissMatchedTestNames(); + } + + @Test( + description = + "GITHUB-2897, Expected TestNGException thrown when ignoreMissedTestNames disabled and all given test names are invalid.", + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = + "\nThe test\\(s\\) \\<\\[test3\\]\\> cannot be found in suite.") + public void testHaveExceptionWhenIgnoreMissedTestNamesDisabledWithAllInvalidTestNames() { + final boolean ignoreMissedTestNames = false; + XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2"); + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"), ignoreMissedTestNames); + testNamesMatcher.validateMissMatchedTestNames(); + } + + @Test( + description = + "GITHUB-2897, Expected TestNGException thrown when ignoreMissedTestNames disabled and partial of given test names are invalid.", + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = + "\nThe test\\(s\\) \\<\\[test3\\]\\> cannot be found in suite.") + public void testHaveExceptionWhenIgnoreMissedTestNamesDisabledWithPartialInvalidTestNames() { + final boolean ignoreMissedTestNames = false; + XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2"); + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(xmlSuite, Arrays.asList("test2", "test3"), ignoreMissedTestNames); + testNamesMatcher.validateMissMatchedTestNames(); + } + + @Test(description = "GITHUB-2897, Missed test names are found as expected.") + public void testMissedTestNamesFound() { + XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2"); + final String expectedMissedTestNames = "test3"; + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(xmlSuite, Collections.singletonList(expectedMissedTestNames)); + List missedTestNames = testNamesMatcher.getMissedTestNames(); + assertThat(missedTestNames).hasSameElementsAs(Arrays.asList(expectedMissedTestNames)); + } + @DataProvider(name = "getTestnames") public Object[][] getTestnameToSearchFor() { return new Object[][] { @@ -103,7 +177,10 @@ public Object[][] getTestnameToSearchFor() { public Object[][] getTestData() { return new Object[][] { {new XmlSuite(), null}, - {new XmlSuite(), Collections.emptyList()} + {new XmlSuite(), Collections.emptyList()}, + {new XmlSuite(), Collections.singletonList("")}, + {new XmlSuite(), Collections.singletonList(" ")}, + {new XmlSuite(), Arrays.asList(null, "", " ")} }; } }