From 09aced894df3077c8f929cdba753d39ca915ad99 Mon Sep 17 00:00:00 2001 From: Henry Coles Date: Mon, 19 Apr 2021 11:25:37 +0100 Subject: [PATCH] check for mutations before running coverage --- .../java/org/pitest/coverage/NoCoverage.java | 57 ++++++++++++ .../build/MutationTestBuilder.java | 35 +++---- .../incremental/NullHistoryStore.java | 0 .../tooling/MutationCoverage.java | 93 +++++++++++++------ .../main/java/org/pitest/util/Timings.java | 8 +- .../tooling/MutationCoverageReportTest.java | 38 +++++++- 6 files changed, 177 insertions(+), 54 deletions(-) create mode 100644 pitest-entry/src/main/java/org/pitest/coverage/NoCoverage.java rename pitest-entry/src/{test => main}/java/org/pitest/mutationtest/incremental/NullHistoryStore.java (100%) diff --git a/pitest-entry/src/main/java/org/pitest/coverage/NoCoverage.java b/pitest-entry/src/main/java/org/pitest/coverage/NoCoverage.java new file mode 100644 index 000000000..548930b05 --- /dev/null +++ b/pitest-entry/src/main/java/org/pitest/coverage/NoCoverage.java @@ -0,0 +1,57 @@ +package org.pitest.coverage; + +import org.pitest.classinfo.ClassInfo; +import org.pitest.classinfo.ClassName; + +import java.math.BigInteger; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +public class NoCoverage implements CoverageDatabase { + @Override + public Collection getClassInfo(Collection classes) { + return Collections.emptyList(); + } + + @Override + public int getNumberOfCoveredLines(Collection clazz) { + return 0; + } + + @Override + public Collection getTestsForClass(ClassName clazz) { + return Collections.emptyList(); + } + + @Override + public Collection getTestsForInstructionLocation(InstructionLocation location) { + return Collections.emptyList(); + } + + @Override + public Collection getTestsForClassLine(ClassLine classLine) { + return Collections.emptyList(); + } + + @Override + public BigInteger getCoverageIdForClass(ClassName clazz) { + return BigInteger.ZERO; + } + + @Override + public Collection getClassesForFile(String sourceFile, String packageName) { + return Collections.emptyList(); + } + + @Override + public CoverageSummary createSummary() { + return new CoverageSummary(0,0); + } + + @Override + public Map> getInstructionCoverage() { + return Collections.emptyMap(); + } +} diff --git a/pitest-entry/src/main/java/org/pitest/mutationtest/build/MutationTestBuilder.java b/pitest-entry/src/main/java/org/pitest/mutationtest/build/MutationTestBuilder.java index 81adee4ba..f955211a4 100644 --- a/pitest-entry/src/main/java/org/pitest/mutationtest/build/MutationTestBuilder.java +++ b/pitest-entry/src/main/java/org/pitest/mutationtest/build/MutationTestBuilder.java @@ -14,7 +14,14 @@ */ package org.pitest.mutationtest.build; -import static java.util.Comparator.comparing; +import org.pitest.classinfo.ClassName; +import org.pitest.coverage.TestInfo; +import org.pitest.functional.FCollection; +import org.pitest.functional.prelude.Prelude; +import org.pitest.mutationtest.DetectionStatus; +import org.pitest.mutationtest.MutationAnalyser; +import org.pitest.mutationtest.MutationResult; +import org.pitest.mutationtest.engine.MutationDetails; import java.util.ArrayList; import java.util.Collection; @@ -25,14 +32,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import org.pitest.classinfo.ClassName; -import org.pitest.coverage.TestInfo; -import org.pitest.functional.FCollection; -import org.pitest.functional.prelude.Prelude; -import org.pitest.mutationtest.DetectionStatus; -import org.pitest.mutationtest.MutationAnalyser; -import org.pitest.mutationtest.MutationResult; -import org.pitest.mutationtest.engine.MutationDetails; +import static java.util.Comparator.comparing; public class MutationTestBuilder { @@ -42,8 +42,9 @@ public class MutationTestBuilder { private final MutationGrouper grouper; public MutationTestBuilder(final WorkerFactory workerFactory, - final MutationAnalyser analyser, final MutationSource mutationSource, - final MutationGrouper grouper) { + final MutationAnalyser analyser, + final MutationSource mutationSource, + final MutationGrouper grouper) { this.mutationSource = mutationSource; this.analyser = analyser; @@ -55,8 +56,7 @@ public List createMutationTestUnits( final Collection codeClasses) { final List tus = new ArrayList<>(); - final List mutations = FCollection.flatMap(codeClasses, - classToMutations()); + final List mutations = FCollection.flatMap(codeClasses, mutationSource::createMutations); mutations.sort(comparing(MutationDetails::getId)); @@ -65,7 +65,7 @@ public List createMutationTestUnits( final Collection needAnalysis = analysedMutations.stream() .filter(statusNotKnown()) - .map(resultToDetails()) + .map(MutationResult::getDetails) .collect(Collectors.toList()); final List analysed = FCollection.filter(analysedMutations, @@ -86,9 +86,6 @@ public List createMutationTestUnits( return tus; } - private Function> classToMutations() { - return a -> MutationTestBuilder.this.mutationSource.createMutations(a); - } private MutationAnalysisUnit makePreAnalysedUnit( final List analysed) { @@ -105,10 +102,6 @@ private MutationAnalysisUnit makeUnanalysedUnit( this.workerFactory); } - private static Function resultToDetails() { - return a -> a.getDetails(); - } - private static Predicate statusNotKnown() { return a -> a.getStatus() == DetectionStatus.NOT_STARTED; } diff --git a/pitest-entry/src/test/java/org/pitest/mutationtest/incremental/NullHistoryStore.java b/pitest-entry/src/main/java/org/pitest/mutationtest/incremental/NullHistoryStore.java similarity index 100% rename from pitest-entry/src/test/java/org/pitest/mutationtest/incremental/NullHistoryStore.java rename to pitest-entry/src/main/java/org/pitest/mutationtest/incremental/NullHistoryStore.java diff --git a/pitest-entry/src/main/java/org/pitest/mutationtest/tooling/MutationCoverage.java b/pitest-entry/src/main/java/org/pitest/mutationtest/tooling/MutationCoverage.java index e91b46eae..68b368da8 100644 --- a/pitest-entry/src/main/java/org/pitest/mutationtest/tooling/MutationCoverage.java +++ b/pitest-entry/src/main/java/org/pitest/mutationtest/tooling/MutationCoverage.java @@ -23,6 +23,8 @@ import org.pitest.classpath.CodeSource; import org.pitest.coverage.CoverageDatabase; import org.pitest.coverage.CoverageGenerator; +import org.pitest.coverage.CoverageSummary; +import org.pitest.coverage.NoCoverage; import org.pitest.coverage.TestInfo; import org.pitest.functional.FCollection; import org.pitest.help.Help; @@ -48,6 +50,8 @@ import org.pitest.mutationtest.incremental.DefaultCodeHistory; import org.pitest.mutationtest.incremental.HistoryListener; import org.pitest.mutationtest.incremental.IncrementalAnalyser; +import org.pitest.mutationtest.incremental.NullHistoryStore; +import org.pitest.mutationtest.statistics.MutationStatistics; import org.pitest.mutationtest.statistics.MutationStatisticsListener; import org.pitest.mutationtest.statistics.Score; import org.pitest.util.Log; @@ -66,6 +70,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import static java.util.Collections.emptyList; + public class MutationCoverage { private static final int MB = 1024 * 1024; @@ -112,7 +118,34 @@ public CombinedStatistics runReport() throws IOException { checkExcludedRunners(); - final CoverageDatabase coverageData = coverage().calculateCoverage(); + final EngineArguments args = EngineArguments.arguments() + .withExcludedMethods(this.data.getExcludedMethods()) + .withMutators(this.data.getMutators()); + final MutationEngine engine = this.strategies.factory().createEngine(args); + + List preScanMutations = findMutations(engine, args); + LOG.info("Created " + preScanMutations.size() + " mutation test units in pre scan"); + + // throw error if configured to do so + checkMutationsFound(preScanMutations); + + if (preScanMutations.isEmpty()) { + LOG.info("Skipping coverage and analysis as no mutations found" ); + return emptyStatistics(); + } + + return runAnalysis(runtime, t0, args, engine); + + } + + private CombinedStatistics emptyStatistics() { + MutationStatistics mutationStatistics = new MutationStatistics(emptyList(),0,0,0,0); + return new CombinedStatistics(mutationStatistics, new CoverageSummary(0,0)); + } + + private CombinedStatistics runAnalysis(Runtime runtime, long t0, EngineArguments args, MutationEngine engine) { + CoverageDatabase coverageData = coverage().calculateCoverage(); + HistoryStore history = this.strategies.history(); LOG.fine("Used memory after coverage calculation " + ((runtime.totalMemory() - runtime.freeMemory()) / MB) + " mb"); @@ -121,31 +154,24 @@ public CombinedStatistics runReport() throws IOException { final MutationStatisticsListener stats = new MutationStatisticsListener(); - final EngineArguments args = EngineArguments.arguments() - .withExcludedMethods(this.data.getExcludedMethods()) - .withMutators(this.data.getMutators()); - final MutationEngine engine = this.strategies.factory().createEngine(args); - - final List config = createConfig(t0, coverageData, - stats, engine); - - history().initialize(); + history.initialize(); this.timings.registerStart(Timings.Stage.BUILD_MUTATION_TESTS); - final List tus = buildMutationTests(coverageData, - engine, args); + final List tus = buildMutationTests(coverageData, history, + engine, args); this.timings.registerEnd(Timings.Stage.BUILD_MUTATION_TESTS); LOG.info("Created " + tus.size() + " mutation test units"); - checkMutationsFound(tus); - recordClassPath(coverageData); + recordClassPath(history, coverageData); LOG.fine("Used memory before analysis start " + ((runtime.totalMemory() - runtime.freeMemory()) / MB) + " mb"); LOG.fine("Free Memory before analysis start " + (runtime.freeMemory() / MB) + " mb"); + final List config = createConfig(t0, coverageData, history, + stats, engine); final MutationAnalysisExecutor mae = new MutationAnalysisExecutor( numberOfThreads(), config); this.timings.registerStart(Timings.Stage.RUN_MUTATION_TESTS); @@ -158,9 +184,22 @@ public CombinedStatistics runReport() throws IOException { return new CombinedStatistics(stats.getStatistics(), coverageData.createSummary()); + } + private List findMutations(MutationEngine engine, EngineArguments args) { + // Run mutant discovery without coverage data or history. + // Ideally we'd ony discover mutants once, but the process is currently tightly + // entangled with coverage data. Generating coverage data is expensive for + // some projects, while discovery usually takes less than 1 second. By doing + // an initial run here we are able to skip coverage generation when no mutants + // are found, e.g if pitest is being run against diffs. + this.timings.registerStart(Timings.Stage.MUTATION_PRE_SCAN); + List mutants = buildMutationTests(new NoCoverage(), new NullHistoryStore(), engine, args); + this.timings.registerEnd(Timings.Stage.MUTATION_PRE_SCAN); + return mutants; } + private void checkExcludedRunners() { final Collection excludedRunners = this.data.getExcludedRunners(); if (!excludedRunners.isEmpty()) { @@ -178,9 +217,11 @@ private int numberOfThreads() { return Math.max(1, this.data.getNumberOfThreads()); } - private List createConfig(final long t0, - final CoverageDatabase coverageData, - final MutationStatisticsListener stats, final MutationEngine engine) { + private List createConfig(long t0, + CoverageDatabase coverageData, + HistoryStore history, + MutationStatisticsListener stats, + MutationEngine engine) { final List ls = new ArrayList<>(); ls.add(stats); @@ -193,7 +234,7 @@ private List createConfig(final long t0, .listenerFactory().getListener(this.data.getFreeFormProperties(), args); ls.add(mutationReportListener); - ls.add(new HistoryListener(history())); + ls.add(new HistoryListener(history)); if (!this.data.isVerbose()) { ls.add(new SpinnerListener(System.out)); @@ -201,11 +242,11 @@ private List createConfig(final long t0, return ls; } - private void recordClassPath(final CoverageDatabase coverageData) { + private void recordClassPath(HistoryStore history, CoverageDatabase coverageData) { final Set allClassNames = getAllClassesAndTests(coverageData); final Collection ids = FCollection.map( this.code.getClassInfo(allClassNames), ClassInfo.toFullClassId()); - history().recordClassPath(ids, coverageData); + history.recordClassPath(ids, coverageData); } private Set getAllClassesAndTests( @@ -245,8 +286,10 @@ private void printStats(final MutationStatisticsListener stats) { stats.getStatistics().report(ps); } - private List buildMutationTests( - final CoverageDatabase coverageData, final MutationEngine engine, EngineArguments args) { + private List buildMutationTests(CoverageDatabase coverageData, + HistoryStore history, + MutationEngine engine, + EngineArguments args) { final MutationConfig mutationConfig = new MutationConfig(engine, coverage() .getLaunchOptions()); @@ -264,7 +307,7 @@ private List buildMutationTests( final MutationSource source = new MutationSource(mutationConfig, testPrioritiser, bas, interceptor); final MutationAnalyser analyser = new IncrementalAnalyser( - new DefaultCodeHistory(this.code, history()), coverageData); + new DefaultCodeHistory(this.code, history), coverageData); final WorkerFactory wf = new WorkerFactory(this.baseDir, coverage() .getConfiguration(), mutationConfig, args, @@ -299,10 +342,6 @@ private CoverageGenerator coverage() { return this.strategies.coverage(); } - private HistoryStore history() { - return this.strategies.history(); - } - // For reasons not yet understood classes from rt.jar are not resolved for some // projects during static analysis phase. For now fall back to the classloader when // a class not provided by project classpath diff --git a/pitest-entry/src/main/java/org/pitest/util/Timings.java b/pitest-entry/src/main/java/org/pitest/util/Timings.java index 531947cd1..7ebe79d37 100644 --- a/pitest-entry/src/main/java/org/pitest/util/Timings.java +++ b/pitest-entry/src/main/java/org/pitest/util/Timings.java @@ -22,9 +22,11 @@ public class Timings { public enum Stage { - BUILD_MUTATION_TESTS("build mutation tests"), RUN_MUTATION_TESTS( - "run mutation analysis"), SCAN_CLASS_PATH("scan classpath"), COVERAGE( - "coverage and dependency analysis"); + MUTATION_PRE_SCAN("pres scan for mutations"), + BUILD_MUTATION_TESTS("build mutation tests"), + RUN_MUTATION_TESTS("run mutation analysis"), + SCAN_CLASS_PATH("scan classpath"), + COVERAGE("coverage and dependency analysis"); private final String description; diff --git a/pitest-entry/src/test/java/org/pitest/mutationtest/tooling/MutationCoverageReportTest.java b/pitest-entry/src/test/java/org/pitest/mutationtest/tooling/MutationCoverageReportTest.java index 9f29c39e4..d56a8d540 100644 --- a/pitest-entry/src/test/java/org/pitest/mutationtest/tooling/MutationCoverageReportTest.java +++ b/pitest-entry/src/test/java/org/pitest/mutationtest/tooling/MutationCoverageReportTest.java @@ -18,13 +18,17 @@ import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.anyCollection; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.pitest.mutationtest.LocationMother.aLocation; +import static org.pitest.mutationtest.LocationMother.aMutationId; import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.List; import org.junit.Before; import org.junit.Ignore; @@ -53,6 +57,7 @@ import org.pitest.mutationtest.config.ReportOptions; import org.pitest.mutationtest.config.SettingsFactory; import org.pitest.mutationtest.engine.Mutater; +import org.pitest.mutationtest.engine.MutationDetails; import org.pitest.mutationtest.engine.MutationDetailsMother; import org.pitest.mutationtest.engine.MutationEngine; import org.pitest.mutationtest.engine.gregor.config.GregorEngineFactory; @@ -149,6 +154,8 @@ public void shouldRecordClassPath() { new ClassIdentifier(0, clazz), "0"); final ClassInfo foo = ClassInfoMother.make(fooId.getId()); + when(this.mutater.findMutations(ClassName.fromClass(Foo.class))).thenReturn(aMutantIn(Foo.class)); + when(this.code.getCodeUnderTestNames()).thenReturn( Collections.singleton(clazz)); when(this.code.getClassInfo(anyCollection())).thenReturn( @@ -173,18 +180,40 @@ public void shouldReportNoMutationsFoundWhenNoneDetected() { } @Test - @Ignore("is triggering filter with fake classes") + public void shouldNotRunCoverageWhenNoMutationsFound() { + this.data.setFailWhenNoMutations(false); + createAndRunTestee(); + verify(coverage, never()).calculateCoverage(); + } + + @Test + public void shouldNotInitializeHistoryWhenNoMutationsFound() { + this.data.setFailWhenNoMutations(false); + createAndRunTestee(); + verify(history, never()).initialize(); + } + + @Test public void shouldReportMutationsFoundWhenSomeDetected() { this.data.setFailWhenNoMutations(false); final ClassName foo = ClassName.fromClass(Foo.class); - when(this.mutater.findMutations(foo)).thenReturn( - MutationDetailsMother.aMutationDetail().build(1)); + when(this.mutater.findMutations(foo)).thenReturn(aMutantIn(Foo.class)); when(this.code.getCodeUnderTestNames()).thenReturn( Collections.singleton(foo)); final CombinedStatistics actual = createAndRunTestee(); assertEquals(1, actual.getMutationStatistics().getTotalMutations()); } + + private List aMutantIn(Class clazz) { + return MutationDetailsMother.aMutationDetail() + .withId(aMutationId().withLocation(aLocation() + .withClass(ClassName.fromClass(clazz)) + .withMethod("method") + .withMethodDescription("()I"))) + .build(1); + } + private CombinedStatistics createAndRunTestee() { final MutationStrategies strategies = new MutationStrategies( new GregorEngineFactory(), this.history, this.coverage, @@ -205,4 +234,7 @@ private CombinedStatistics createAndRunTestee() { class Foo { + int method() { + return 1; + } }