From 4c7d9030f38621d915e18f8fde5a99f345a65ec4 Mon Sep 17 00:00:00 2001 From: Bruce Wen Date: Sat, 6 May 2023 05:05:22 +0200 Subject: [PATCH] Support running partial test names from suite file Closes #2897 1. Add new boolean option '-ignoreMissedTestNames' to work with the option '-testnames'. 2. When -testnames is given, and '-ignoreMissedTestNames true' is also given, then in case any missed test names not found in the suite, only warning message will be printed, TestNG will continue to run other test names which are existing in the suite. 3. Users who are going to use the new option '-ignoreMissedTestNames' should be aware of that the logging level should be properly configured to make sure the warning message is visible in output or console, rather than missed the notification of the missed test names, if any. --- CHANGES.txt | 1 + .../main/java/org/testng/TestNGAntTask.java | 6 + .../main/java/org/testng/util/Strings.java | 18 +++ .../main/java/org/testng/CommandLineArgs.java | 8 ++ .../main/java/org/testng/JarFileUtils.java | 46 ++++++-- .../src/main/java/org/testng/TestNG.java | 13 ++- .../testng/xml/internal/TestNamesMatcher.java | 57 +++++++-- .../java/org/testng/JarFileUtilsTest.java | 109 +++++++++++++++--- .../xml/internal/TestNamesMatcherTest.java | 79 ++++++++++++- 9 files changed, 304 insertions(+), 33 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 01803e313c..3901cdcf6e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ Current +New: GITHUB-2897: Not exception but warning if some (not all) of the given test names are not found in suite files. (Bruce Wen) New: Added assertListContains and assertListContainsObject methods to check if specific object present in List (Dmytro Budym) Fixed: GITHUB-2888: Skipped Tests with DataProvider appear as failed (Joaquin Moreira) Fixed: GITHUB-2884: Discrepancies with DataProvider and Retry of failed tests (Krishnan Mahadevan) diff --git a/testng-ant/src/main/java/org/testng/TestNGAntTask.java b/testng-ant/src/main/java/org/testng/TestNGAntTask.java index 412b796ef9..6ef5967d94 100644 --- a/testng-ant/src/main/java/org/testng/TestNGAntTask.java +++ b/testng-ant/src/main/java/org/testng/TestNGAntTask.java @@ -161,6 +161,7 @@ public class TestNGAntTask extends Task { private String m_methods; private Mode mode = Mode.testng; private boolean forkJvm = true; + private boolean m_ignoreMissedTestNames; public enum Mode { // lower-case to better look in build scripts @@ -360,6 +361,10 @@ public void setTestNames(String testNames) { m_testNames = testNames; } + public void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) { + m_ignoreMissedTestNames = ignoreMissedTestNames; + } + /** * Sets the suite runner class to invoke * @@ -578,6 +583,7 @@ protected List createArguments() { addStringIfNotBlank(argv, CommandLineArgs.SUITE_NAME, m_suiteName); addStringIfNotBlank(argv, CommandLineArgs.TEST_NAME, m_testName); addStringIfNotBlank(argv, CommandLineArgs.TEST_NAMES, m_testNames); + addBooleanIfTrue(argv, CommandLineArgs.IGNORE_MISSED_TEST_NAMES, m_ignoreMissedTestNames); addStringIfNotBlank(argv, CommandLineArgs.METHODS, m_methods); addReporterConfigs(argv); addIntegerIfNotNull(argv, CommandLineArgs.SUITE_THREAD_POOL_SIZE, m_suiteThreadPoolSize); diff --git a/testng-collections/src/main/java/org/testng/util/Strings.java b/testng-collections/src/main/java/org/testng/util/Strings.java index 8f8f3f55a5..452f56f29d 100644 --- a/testng-collections/src/main/java/org/testng/util/Strings.java +++ b/testng-collections/src/main/java/org/testng/util/Strings.java @@ -1,5 +1,6 @@ package org.testng.util; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -18,6 +19,23 @@ public static boolean isNotNullAndNotEmpty(String string) { return !isNullOrEmpty(string); } + /** + * Check if the given string list is null or empty or all elements are null or empty or blank. + * + * @param list A list instance with String elements. + * @return true if the given string list is null or empty or all elements are null or empty or + * blank; otherwise false. + */ + public static boolean isBlankStringList(List list) { + if (list == null) { + return true; + } + if (list.isEmpty()) { + return true; + } + return list.stream().allMatch(t -> t == null || t.isBlank()); + } + private static final Map ESCAPE_HTML_MAP = Maps.newLinkedHashMap(); static { diff --git a/testng-core/src/main/java/org/testng/CommandLineArgs.java b/testng-core/src/main/java/org/testng/CommandLineArgs.java index 03866b2f29..f696de1d04 100644 --- a/testng-core/src/main/java/org/testng/CommandLineArgs.java +++ b/testng-core/src/main/java/org/testng/CommandLineArgs.java @@ -142,6 +142,14 @@ public class CommandLineArgs { @Parameter(names = TEST_NAMES, description = "The list of test names to run") public String testNames; + public static final String IGNORE_MISSED_TEST_NAMES = "-ignoreMissedTestNames"; + + @Parameter( + names = IGNORE_MISSED_TEST_NAMES, + description = + "Ignore missed test names given by '-testnames' and continue to run existing tests, if any.") + public boolean ignoreMissedTestNames = false; + public static final String TEST_JAR = "-testjar"; @Parameter(names = TEST_JAR, description = "A jar file containing the tests") diff --git a/testng-core/src/main/java/org/testng/JarFileUtils.java b/testng-core/src/main/java/org/testng/JarFileUtils.java index 176881bc2f..377ee3f700 100644 --- a/testng-core/src/main/java/org/testng/JarFileUtils.java +++ b/testng-core/src/main/java/org/testng/JarFileUtils.java @@ -23,6 +23,7 @@ class JarFileUtils { private final IPostProcessor processor; private final String xmlPathInJar; + private final boolean ignoreMissedTestNames; private final List testNames; private final List suites = Lists.newLinkedList(); private final XmlSuite.ParallelMode mode; @@ -36,10 +37,28 @@ class JarFileUtils { String xmlPathInJar, List testNames, XmlSuite.ParallelMode mode) { + this(processor, xmlPathInJar, testNames, mode, false); + } + + JarFileUtils( + IPostProcessor processor, + String xmlPathInJar, + List testNames, + boolean ignoreMissedTestNames) { + this(processor, xmlPathInJar, testNames, XmlSuite.ParallelMode.NONE, ignoreMissedTestNames); + } + + JarFileUtils( + IPostProcessor processor, + String xmlPathInJar, + List testNames, + XmlSuite.ParallelMode mode, + boolean ignoreMissedTestNames) { this.processor = processor; this.xmlPathInJar = xmlPathInJar; this.testNames = testNames; this.mode = mode == null ? XmlSuite.ParallelMode.NONE : mode; + this.ignoreMissedTestNames = ignoreMissedTestNames; } List extractSuitesFrom(File jarFile) { @@ -69,6 +88,7 @@ private boolean testngXmlExistsInJar(File jarFile, List classes) throws Enumeration entries = jf.entries(); File file = java.nio.file.Files.createTempDirectory("testngXmlPathInJar-").toFile(); String suitePath = null; + while (entries.hasMoreElements()) { JarEntry je = entries.nextElement(); String jeName = je.getName(); @@ -87,24 +107,34 @@ private boolean testngXmlExistsInJar(File jarFile, List classes) throws classes.add(constructClassName(je)); } } + if (Strings.isNullOrEmpty(suitePath)) { + Utils.log("TestNG", 1, String.format("Not found '%s' in '%s'.", xmlPathInJar, jarFile)); return false; } + Collection parsedSuites = Parser.parse(suitePath, processor); delete(file); + boolean addedSuite = false; for (XmlSuite suite : parsedSuites) { - // If test names were specified, only run these test names - if (testNames != null) { - TestNamesMatcher testNamesMatcher = new TestNamesMatcher(suite, testNames); - testNamesMatcher.validateMissMatchedTestNames(); - suites.addAll(testNamesMatcher.getSuitesMatchingTestNames()); - } else { + if (testNames == null) { suites.add(suite); + addedSuite = true; + } else { + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(suite, testNames, ignoreMissedTestNames); + boolean validationResult = testNamesMatcher.validateMissMatchedTestNames(); + if (validationResult) { + suites.addAll(testNamesMatcher.getSuitesMatchingTestNames()); + addedSuite = true; + } else { + Utils.error(String.format("None of '%s' found in '%s'.", testNames, suite)); + } } - return true; } + + return addedSuite; } - return false; } private void delete(File f) throws IOException { diff --git a/testng-core/src/main/java/org/testng/TestNG.java b/testng-core/src/main/java/org/testng/TestNG.java index 2fa11a71c7..339b1b6bfe 100644 --- a/testng-core/src/main/java/org/testng/TestNG.java +++ b/testng-core/src/main/java/org/testng/TestNG.java @@ -353,7 +353,8 @@ private Collection processCommandLineArgs(Collection allSuit continue; } // If test names were specified, only run these test names - TestNamesMatcher testNamesMatcher = new TestNamesMatcher(s, m_testNames); + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(s, m_testNames, m_ignoreMissedTestNames); testNamesMatcher.validateMissMatchedTestNames(); result.addAll(testNamesMatcher.getSuitesMatchingTestNames()); } @@ -417,7 +418,8 @@ public void initializeSuitesAndJarFile() { File jarFile = new File(m_jarPath); JarFileUtils utils = - new JarFileUtils(getProcessor(), m_xmlPathInJar, m_testNames, m_parallelMode); + new JarFileUtils( + getProcessor(), m_xmlPathInJar, m_testNames, m_parallelMode, m_ignoreMissedTestNames); Collection allSuites = utils.extractSuitesFrom(jarFile); allSuites.forEach(this::processParallelModeCommandLineArgs); @@ -799,6 +801,8 @@ public List getSuiteListeners() { /** The list of test names to run from the given suite */ private List m_testNames; + private boolean m_ignoreMissedTestNames; + private Integer m_suiteThreadPoolSize = CommandLineArgs.SUITE_THREAD_POOL_SIZE_DEFAULT; private boolean m_randomizeSuites = Boolean.FALSE; @@ -1475,6 +1479,7 @@ protected void configure(CommandLineArgs cla) { if (cla.testNames != null) { setTestNames(Arrays.asList(cla.testNames.split(","))); + setIgnoreMissedTestNames(cla.ignoreMissedTestNames); } // Note: can't use a Boolean field here because we are allowing a boolean @@ -1574,6 +1579,10 @@ protected void configure(CommandLineArgs cla) { alwaysRunListeners(cla.alwaysRunListeners); } + private void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) { + m_ignoreMissedTestNames = ignoreMissedTestNames; + } + public void setSuiteThreadPoolSize(Integer suiteThreadPoolSize) { m_suiteThreadPoolSize = suiteThreadPoolSize; } diff --git a/testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java b/testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java index 9d0967f854..507f358699 100644 --- a/testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java +++ b/testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java @@ -3,19 +3,35 @@ import java.util.List; import org.testng.TestNGException; import org.testng.collections.Lists; +import org.testng.log4testng.Logger; +import org.testng.util.Strings; import org.testng.xml.XmlSuite; import org.testng.xml.XmlTest; -/** The class to work with "-testnames" */ +/** + * The class to work with "-testnames", "-ignoreMissedTestNames", and VM argument + * "-Dtestng.ignore.missed.testnames". If both "-ignoreMissedTestNames" and VM argument + * "-Dtestng.ignore.missed.testnames" are set, then either of them has "true" value will enable the + * feature to ingore partially missed test names and run those existing test names. + */ public final class TestNamesMatcher { + private static final Logger LOGGER = Logger.getLogger(TestNamesMatcher.class); + private final List cloneSuites = Lists.newArrayList(); private final List matchedTestNames = Lists.newArrayList(); private final List matchedTests = Lists.newArrayList(); private final List testNames; + private final boolean ignoreMissedTestNames; public TestNamesMatcher(XmlSuite xmlSuite, List testNames) { + this(xmlSuite, testNames, false); + } + + public TestNamesMatcher( + XmlSuite xmlSuite, List testNames, boolean ignoreMissedTestNames) { this.testNames = testNames; + this.ignoreMissedTestNames = ignoreMissedTestNames; cloneIfContainsTestsWithNamesMatchingAny(xmlSuite, this.testNames); } @@ -26,7 +42,7 @@ public TestNamesMatcher(XmlSuite xmlSuite, List testNames) { * @param testNames The list of testnames to iterate through */ private void cloneIfContainsTestsWithNamesMatchingAny(XmlSuite xmlSuite, List testNames) { - if (testNames == null || testNames.isEmpty()) { + if (Strings.isBlankStringList(testNames)) { throw new TestNGException("Please provide a valid list of names to check."); } @@ -43,13 +59,38 @@ public List getSuitesMatchingTestNames() { return cloneSuites; } - public void validateMissMatchedTestNames() { - List tmpTestNames = Lists.newArrayList(); - tmpTestNames.addAll(testNames); - tmpTestNames.removeIf(matchedTestNames::contains); - if (!tmpTestNames.isEmpty()) { - throw new TestNGException("The test(s) <" + tmpTestNames + "> cannot be found in suite."); + /** + * Do validation for testNames and notify users if any testNames are missed in suite. This method + * is also used to decide how to run test suite when test names are given. In legacy logic, if + * test names are given and exist in suite, then run them; if any of them do not exist in suite, + * then throw exception and exit. After ignoreMissedTestNames is introduced, if + * ignoreMissedTestNames is enabled, then any of the given test names exist in suite will be run, + * and print warning message to tell those test names do not exist in suite. + * + * @return boolean if ignoreMissedTestNames disabled, then return true if no missed test names in + * suite, otherwise throw TestNGException; if ignoreMissedTestNames enabled, then return true + * if any test names exist in suite, otehrwise (all given test names are missed) throw + * TestNGException. + */ + public boolean validateMissMatchedTestNames() { + final List missedTestNames = getMissedTestNames(); + if (!missedTestNames.isEmpty()) { + final String errMsg = "The test(s) <" + missedTestNames + "> cannot be found in suite."; + if (ignoreMissedTestNames && !matchedTestNames.isEmpty()) { + LOGGER.warn(errMsg); + return true; + } else { + throw new TestNGException(errMsg); + } } + return missedTestNames.isEmpty() && !matchedTestNames.isEmpty(); + } + + public List getMissedTestNames() { + List missedTestNames = Lists.newArrayList(); + missedTestNames.addAll(testNames); + missedTestNames.removeIf(matchedTestNames::contains); + return missedTestNames; } public List getMatchedTests() { diff --git a/testng-core/src/test/java/org/testng/JarFileUtilsTest.java b/testng-core/src/test/java/org/testng/JarFileUtilsTest.java index 5e3816ec9a..3d3afee6eb 100644 --- a/testng-core/src/test/java/org/testng/JarFileUtilsTest.java +++ b/testng-core/src/test/java/org/testng/JarFileUtilsTest.java @@ -21,6 +21,7 @@ import org.testng.xml.XmlTest; public class JarFileUtilsTest { + private static File jar = null; @BeforeClass @@ -65,6 +66,42 @@ public void testWithInvalidTestNames() throws MalformedURLException { new String[] {"org.testng.jarfileutils.org.testng.SampleTest1"}); } + @Test( + description = + "GITHUB-2897, Have TestNGException thrown when ignoreMissedTestNames enabled and ALL given test names are invalid.", + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = + "\nThe test\\(s\\) <\\[testng-tests-child11\\]> cannot be found in suite.") + public void testWithAllInvalidTestNamesNoExceptionIfIgnoreMissedTestNamesEnabled() + throws MalformedURLException { + boolean ignoreMissedTestNames = true; + JarFileUtils utils = + newJarFileUtils(Collections.singletonList("testng-tests-child11"), ignoreMissedTestNames); + runTest(utils, 1, null, null, "Jar suite"); + } + + @Test( + description = + "GITHUB-2897, No TestNGException thrown when ignoreMissedTestNames enabled and partial given test names are invalid and partial valid test names are expected to run.") + public void testWithPartialInvalidTestNamesNoExceptionIfIgnoreMissedTestNamesEnabled() + throws MalformedURLException { + String[] expectedTestNames = + new String[] {"testng-tests-child2", "testng-tests-child4", "testng-tests-child5"}; + String[] expectedClassNames = + new String[] { + "org.testng.jarfileutils.org.testng.SampleTest2", + "org.testng.jarfileutils.org.testng.SampleTest4", + "org.testng.jarfileutils.org.testng.SampleTest5" + }; + List testNames = + Arrays.asList( + "testng-tests-child2", "testng-tests-child4", "testng-tests-child5", "invalid"); + boolean ignoreMissedTestNames = true; + JarFileUtils utils = newJarFileUtils(testNames, ignoreMissedTestNames); + // 3 tests from 3 suites, the first suite has one test is given + runTest(utils, 1, 3, expectedTestNames, expectedClassNames, "testng-tests-suite"); + } + @Test public void testWithInvalidXmlFile() throws MalformedURLException { JarFileUtils utils = @@ -89,11 +126,32 @@ public void testWithInvalidXmlFile() throws MalformedURLException { */ @Test public void ensureThatExceptionAreNotThrown() throws MalformedURLException { + TestNG testNg = new TestNG(false); + List testNames = + Arrays.asList("testng-tests-child2", "testng-tests-child4", "testng-tests-child5"); + testNg.setTestNames(testNames); + testNg.setXmlPathInJar("jarfileutils/testng-tests.xml"); + testNg.setTestJar(jar.getAbsolutePath()); + testNg.initializeSuitesAndJarFile(); + // "testng-tests-child2", "testng-tests-child4", "testng-tests-child5" are from 3 different test + // suites + Assert.assertEquals(testNg.m_suites.size(), 3); + } + + /** + * Test to ensure that exception is thrown for invalid test name. + * + * @throws MalformedURLException + */ + @Test( + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = "\nThe test\\(s\\) <\\[dummy\\]> cannot be found in suite.") + public void ensureThatExceptionAreThrown() throws MalformedURLException { TestNG testNg = new TestNG(false); List testNames = Arrays.asList("testng-tests-child2", "testng-tests-child4", "testng-tests-child5", "dummy"); testNg.setTestNames(testNames); - testNg.setXmlPathInJar(jar.getAbsolutePath()); + testNg.setXmlPathInJar("jarfileutils/testng-tests.xml"); testNg.setTestJar(jar.getAbsolutePath()); testNg.initializeSuitesAndJarFile(); Assert.assertEquals(testNg.m_suites.size(), 1); @@ -118,20 +176,20 @@ public void testWithValidTestNamesFromMultiChildSuites() throws MalformedURLExce assertThat(suite.getName()).isEqualTo("testng-tests-suite"); List testNames = new LinkedList<>(); List classNames = new LinkedList<>(); - for (XmlSuite xmlSuite : suites) { - extractClassNames(xmlSuite, testNames, classNames); - } + extractClassNames(suites, testNames, classNames); assertThat(testNames).containsExactly(expectedTestNames); assertThat(classNames).contains(expectedClassNames); } private static void extractClassNames( - XmlSuite xmlSuite, List testNames, List classNames) { - for (XmlTest xmlTest : xmlSuite.getTests()) { - testNames.add(xmlTest.getName()); - for (XmlClass xmlClass : xmlTest.getXmlClasses()) { - classNames.add(xmlClass.getName()); + List xmlSuites, List testNames, List classNames) { + for (XmlSuite xmlSuite : xmlSuites) { + for (XmlTest xmlTest : xmlSuite.getTests()) { + testNames.add(xmlTest.getName()); + for (XmlClass xmlClass : xmlTest.getXmlClasses()) { + classNames.add(xmlClass.getName()); + } } } } @@ -150,18 +208,30 @@ private static void runTest( String[] expectedTestNames, String[] expectedClassNames, String expectedSuiteName) { + runTest(utils, numberOfTests, 1, expectedTestNames, expectedClassNames, expectedSuiteName); + } + + private static void runTest( + JarFileUtils utils, + int numberOfTests, + int expectedSuiteTotal, + String[] expectedTestNames, + String[] expectedClassNames, + String expectedSuiteName) { List suites = utils.extractSuitesFrom(jar); - assertThat(suites).hasSize(1); + assertThat(suites).hasSize(expectedSuiteTotal); XmlSuite suite = suites.get(0); assertThat(suite.getName()).isEqualTo(expectedSuiteName); assertThat(suite.getTests()).hasSize(numberOfTests); List testNames = new LinkedList<>(); List classNames = new LinkedList<>(); - extractClassNames(suite, testNames, classNames); + extractClassNames(suites, testNames, classNames); if (expectedTestNames != null) { assertThat(testNames).containsExactly(expectedTestNames); } - assertThat(classNames).contains(expectedClassNames); + if (expectedClassNames != null) { + assertThat(classNames).contains(expectedClassNames); + } } public static class FakeProcessor implements IPostProcessor { @@ -173,15 +243,26 @@ public Collection process(Collection suites) { } private static JarFileUtils newJarFileUtils(List testNames) throws MalformedURLException { - return newJarFileUtils("jarfileutils/testng-tests.xml", testNames); + return newJarFileUtils(testNames, false); + } + + private static JarFileUtils newJarFileUtils(List testNames, boolean ignoreMissedTestNames) + throws MalformedURLException { + return newJarFileUtils("jarfileutils/testng-tests.xml", testNames, ignoreMissedTestNames); } private static JarFileUtils newJarFileUtils(String suiteXmlName, List testNames) throws MalformedURLException { + return newJarFileUtils(suiteXmlName, testNames, false); + } + + private static JarFileUtils newJarFileUtils( + String suiteXmlName, List testNames, boolean ignoreMissedTestNames) + throws MalformedURLException { URL url = jar.toURI().toURL(); URLClassLoader classLoader = new URLClassLoader(new URL[] {url}, ClassLoader.getSystemClassLoader()); Thread.currentThread().setContextClassLoader(classLoader); - return new JarFileUtils(new FakeProcessor(), suiteXmlName, testNames); + return new JarFileUtils(new FakeProcessor(), suiteXmlName, testNames, ignoreMissedTestNames); } } diff --git a/testng-core/src/test/java/org/testng/xml/internal/TestNamesMatcherTest.java b/testng-core/src/test/java/org/testng/xml/internal/TestNamesMatcherTest.java index 712fa6ce00..7e168576b9 100644 --- a/testng-core/src/test/java/org/testng/xml/internal/TestNamesMatcherTest.java +++ b/testng-core/src/test/java/org/testng/xml/internal/TestNamesMatcherTest.java @@ -57,6 +57,17 @@ public void testCloneIfContainsTestsWithNamesMatchingAnyNegativeCondition( TestNamesMatcher testNamesHelper = new TestNamesMatcher(xmlSuite, names); } + @Test( + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = "\nPlease provide a valid list of names to check.", + dataProvider = "getData") + public void + testCloneIfContainsTestsWithNamesMatchingAnyNegativeConditionWithIgnoreMissedTestNamesEnabled( + XmlSuite xmlSuite, List names) { + boolean ignoreMissedTestNames = true; + TestNamesMatcher testNamesHelper = new TestNamesMatcher(xmlSuite, names, ignoreMissedTestNames); + } + @Test public void testIfTestnamesComesFromDifferentSuite() { XmlSuite parentSuite = createDummySuiteWithTestNamesAs("test1", "test2"); @@ -90,6 +101,69 @@ public void testCloneIfContainsTestsWithNamesMatchingAnyWithoutMatch() { } } + @Test( + description = + "GITHUB-2897, No TestNGException thrown when ignoreMissedTestNames enabled and only partial of the given test names are invalid.") + public void testNoExceptionWhenIgnoreMissedTestNamesEnabled() { + final boolean ignoreMissedTestNames = true; + XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2"); + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(xmlSuite, Arrays.asList("test2", "test3"), ignoreMissedTestNames); + testNamesMatcher.validateMissMatchedTestNames(); + } + + @Test( + description = + "GITHUB-2897, Expected TestNGException thrown when ignoreMissedTestNames enabled and all given test names are invalid.", + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = + "\nThe test\\(s\\) \\<\\[test3\\]\\> cannot be found in suite.") + public void testHaveExceptionWhenIgnoreMissedTestNamesEnabledWithAllInvalidTestNames() { + final boolean ignoreMissedTestNames = true; + XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2"); + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"), ignoreMissedTestNames); + testNamesMatcher.validateMissMatchedTestNames(); + } + + @Test( + description = + "GITHUB-2897, Expected TestNGException thrown when ignoreMissedTestNames disabled and all given test names are invalid.", + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = + "\nThe test\\(s\\) \\<\\[test3\\]\\> cannot be found in suite.") + public void testHaveExceptionWhenIgnoreMissedTestNamesDisabledWithAllInvalidTestNames() { + final boolean ignoreMissedTestNames = false; + XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2"); + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"), ignoreMissedTestNames); + testNamesMatcher.validateMissMatchedTestNames(); + } + + @Test( + description = + "GITHUB-2897, Expected TestNGException thrown when ignoreMissedTestNames disabled and partial of given test names are invalid.", + expectedExceptions = TestNGException.class, + expectedExceptionsMessageRegExp = + "\nThe test\\(s\\) \\<\\[test3\\]\\> cannot be found in suite.") + public void testHaveExceptionWhenIgnoreMissedTestNamesDisabledWithPartialInvalidTestNames() { + final boolean ignoreMissedTestNames = false; + XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2"); + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(xmlSuite, Arrays.asList("test2", "test3"), ignoreMissedTestNames); + testNamesMatcher.validateMissMatchedTestNames(); + } + + @Test(description = "GITHUB-2897, Missed test names are found as expected.") + public void testMissedTestNamesFound() { + XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2"); + final String expectedMissedTestNames = "test3"; + TestNamesMatcher testNamesMatcher = + new TestNamesMatcher(xmlSuite, Collections.singletonList(expectedMissedTestNames)); + List missedTestNames = testNamesMatcher.getMissedTestNames(); + assertThat(missedTestNames).hasSameElementsAs(Arrays.asList(expectedMissedTestNames)); + } + @DataProvider(name = "getTestnames") public Object[][] getTestnameToSearchFor() { return new Object[][] { @@ -103,7 +177,10 @@ public Object[][] getTestnameToSearchFor() { public Object[][] getTestData() { return new Object[][] { {new XmlSuite(), null}, - {new XmlSuite(), Collections.emptyList()} + {new XmlSuite(), Collections.emptyList()}, + {new XmlSuite(), Collections.singletonList("")}, + {new XmlSuite(), Collections.singletonList(" ")}, + {new XmlSuite(), Arrays.asList(null, "", " ")} }; } }