Add test strength statistic

pitest-ant/src/main/java/org/pitest/ant/PitestTask.java

@@ -236,6 +236,10 @@ public void setMutationThreshold(final String value) {
     this.setOption(ConfigOption.MUTATION_THRESHOLD, value);
   }
 
+  public void setTestStrengthThreshold(final String value) {
+    this.setOption(ConfigOption.TEST_STRENGTH_THRESHOLD, value);
+  }
+
   public void setMaxSurviving(final String value) {
     this.setOption(ConfigOption.MAX_SURVIVING, value);
   }

pitest-ant/src/test/java/org/pitest/ant/PitestTaskTest.java

@@ -440,6 +440,13 @@ public void shouldPassMutationThresholdToJavaTask() {
     verify(this.arg).setValue("--mutationThreshold=42");
   }
 
+  @Test
+  public void shouldPassTestStrengthThresholdToJavaTask() {
+    this.pitestTask.setTestStrengthThreshold("42");
+    this.pitestTask.execute(this.java);
+    verify(this.arg).setValue("--testStrengthThreshold=42");
+  }
+
   @Test
   public void shouldPassMaxSurvivorsToJavaTask() {
     this.pitestTask.setMaxSurviving("42");

pitest-command-line/src/main/java/org/pitest/mutationtest/commandline/MutationCoverageReport.java

@@ -46,6 +46,8 @@ public static void main(final String[] args) {
 
       throwErrorIfScoreBelowCoverageThreshold(stats.getCoverageSummary(),
           data.getCoverageThreshold());
+      throwErrorIfScoreBelowTestStrengthThreshold(stats.getMutationStatistics(),
+              data.getTestStrengthThreshold());
       throwErrorIfScoreBelowMutationThreshold(stats.getMutationStatistics(),
           data.getMutationThreshold());
       throwErrorIfMoreThanMaxSuvivingMutants(stats.getMutationStatistics(), data.getMaximumAllowedSurvivors());
@@ -70,6 +72,15 @@ private static void throwErrorIfScoreBelowMutationThreshold(
     }
   }
 
+  private static void throwErrorIfScoreBelowTestStrengthThreshold(
+          final MutationStatistics stats, final int threshold) {
+    if ((threshold != 0) && (stats.getTestStrength() < threshold)) {
+      throw new RuntimeException("Test strength score of "
+              + stats.getTestStrength() + " is below threshold of "
+              + threshold);
+    }
+  }
+
   private static void throwErrorIfMoreThanMaxSuvivingMutants(
       final MutationStatistics stats, final long threshold) {
     if ((threshold > 0)

pitest-command-line/src/main/java/org/pitest/mutationtest/commandline/OptionsParser.java

@@ -14,6 +14,33 @@
  */
 package org.pitest.mutationtest.commandline;
 
+import joptsimple.ArgumentAcceptingOptionSpec;
+import joptsimple.OptionException;
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import joptsimple.OptionSpecBuilder;
+import joptsimple.util.KeyValuePair;
+import org.pitest.classpath.ClassPath;
+import org.pitest.functional.FCollection;
+import org.pitest.mutationtest.config.ConfigOption;
+import org.pitest.mutationtest.config.ReportOptions;
+import org.pitest.testapi.TestGroupConfig;
+import org.pitest.util.Glob;
+import org.pitest.util.Log;
+import org.pitest.util.Unchecked;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+
 import static org.pitest.mutationtest.config.ConfigOption.AVOID_CALLS;
 import static org.pitest.mutationtest.config.ConfigOption.CHILD_JVM;
 import static org.pitest.mutationtest.config.ConfigOption.CLASSPATH;
@@ -49,6 +76,7 @@
 import static org.pitest.mutationtest.config.ConfigOption.TARGET_CLASSES;
 import static org.pitest.mutationtest.config.ConfigOption.TEST_FILTER;
 import static org.pitest.mutationtest.config.ConfigOption.TEST_PLUGIN;
+import static org.pitest.mutationtest.config.ConfigOption.TEST_STRENGTH_THRESHOLD;
 import static org.pitest.mutationtest.config.ConfigOption.THREADS;
 import static org.pitest.mutationtest.config.ConfigOption.TIMEOUT_CONST;
 import static org.pitest.mutationtest.config.ConfigOption.TIMEOUT_FACTOR;
@@ -57,34 +85,6 @@
 import static org.pitest.mutationtest.config.ConfigOption.USE_INLINED_CODE_DETECTION;
 import static org.pitest.mutationtest.config.ConfigOption.VERBOSE;
 
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-import java.util.function.Predicate;
-import java.util.logging.Logger;
-
-import org.pitest.classpath.ClassPath;
-import org.pitest.functional.FCollection;
-import org.pitest.mutationtest.config.ConfigOption;
-import org.pitest.mutationtest.config.ReportOptions;
-import org.pitest.testapi.TestGroupConfig;
-import org.pitest.util.Glob;
-import org.pitest.util.Log;
-import org.pitest.util.Unchecked;
-
-import joptsimple.ArgumentAcceptingOptionSpec;
-import joptsimple.OptionException;
-import joptsimple.OptionParser;
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-import joptsimple.OptionSpecBuilder;
-import joptsimple.util.KeyValuePair;
-
 public class OptionsParser {
 
   private final Predicate<String>                    dependencyFilter;
@@ -124,6 +124,7 @@ public class OptionsParser {
   private final ArgumentAcceptingOptionSpec<Boolean> timestampedReportsSpec;
   private final ArgumentAcceptingOptionSpec<Boolean> detectInlinedCode;
   private final ArgumentAcceptingOptionSpec<Integer> mutationThreshHoldSpec;
+  private final ArgumentAcceptingOptionSpec<Integer> testStrengthThreshHoldSpec;
   private final ArgumentAcceptingOptionSpec<Integer> coverageThreshHoldSpec;
   private final ArgumentAcceptingOptionSpec<Integer> maxSurvivingSpec;
   private final OptionSpec<String>                   mutationEngine;
@@ -332,6 +333,11 @@ public OptionsParser(Predicate<String> dependencyFilter) {
         .describedAs("Mutation score below which to throw an error")
         .defaultsTo(MUTATION_THRESHOLD.getDefault(Integer.class));
 
+    this.testStrengthThreshHoldSpec = parserAccepts(TEST_STRENGTH_THRESHOLD)
+            .withRequiredArg().ofType(Integer.class)
+            .describedAs("Test strength score below which to throw an error")
+            .defaultsTo(TEST_STRENGTH_THRESHOLD.getDefault(Integer.class));
+
     this.maxSurvivingSpec = parserAccepts(MAX_SURVIVING)
         .withRequiredArg().ofType(Integer.class)
         .describedAs("Maximum number of surviving mutants to allow without throwing an error")
@@ -425,6 +431,7 @@ private ParseResult parseCommandLine(final ReportOptions data,
     data.setHistoryInputLocation(this.historyInputSpec.value(userArgs));
     data.setHistoryOutputLocation(this.historyOutputSpec.value(userArgs));
     data.setMutationThreshold(this.mutationThreshHoldSpec.value(userArgs));
+    data.setTestStrengthThreshold(this.testStrengthThreshHoldSpec.value(userArgs));
     data.setMaximumAllowedSurvivors(this.maxSurvivingSpec.value(userArgs));
     data.setCoverageThreshold(this.coverageThreshHoldSpec.value(userArgs));
     data.setMutationEngine(this.mutationEngine.value(userArgs));

pitest-command-line/src/test/java/org/pitest/mutationtest/commandline/OptionsParserTest.java

@@ -398,6 +398,13 @@ public void shouldParseMutationThreshold() {
     assertEquals(42, actual.getMutationThreshold());
   }
 
+  @Test
+  public void shouldParseTestStrengthThreshold() {
+    final ReportOptions actual = parseAddingRequiredArgs("--testStrengthThreshold",
+            "50");
+    assertEquals(50, actual.getTestStrengthThreshold());
+  }
+
   @Test
   public void shouldParseMaximumAllowedSurvivingMutants() {
     final ReportOptions actual = parseAddingRequiredArgs("--maxSurviving",

pitest-entry/src/main/java/org/pitest/mutationtest/config/ConfigOption.java

@@ -14,10 +14,10 @@
  */
 package org.pitest.mutationtest.config;
 
-import java.io.Serializable;
-
 import org.pitest.mutationtest.build.PercentAndConstantTimeoutStrategy;
 
+import java.io.Serializable;
+
 public enum ConfigOption {
 
   /**
@@ -179,6 +179,11 @@ public enum ConfigOption {
    */
   MUTATION_THRESHOLD("mutationThreshold", 0),
 
+  /**
+   * Test strength score below which to throw an error
+   */
+  TEST_STRENGTH_THRESHOLD("testStrengthThreshold", 0),
+
   /**
    * Number of surviving mutants at which to throw an error
    */

pitest-entry/src/main/java/org/pitest/mutationtest/config/ReportOptions.java

@@ -122,6 +122,7 @@ public class ReportOptions {
   private boolean                        exportLineCoverage             = false;
   private int                            mutationThreshold;
   private int                            coverageThreshold;
+  private int                            testStrengthThreshold;
 
   private String                         mutationEngine                 = "gregor";
 
@@ -547,6 +548,15 @@ public void setCoverageThreshold(final int coverageThreshold) {
     this.coverageThreshold = coverageThreshold;
   }
 
+  public int getTestStrengthThreshold() {
+    return this.testStrengthThreshold;
+  }
+
+  public void setTestStrengthThreshold(final int testStrengthThreshold) {
+    this.testStrengthThreshold = testStrengthThreshold;
+  }
+
+
   public String getJavaExecutable() {
     return this.javaExecutable;
   }
@@ -639,7 +649,7 @@ public String toString() {
         + groupConfig + ", fullMutationMatrix=" + fullMutationMatrix + ", mutationUnitSize=" + mutationUnitSize
         + ", shouldCreateTimestampedReports=" + shouldCreateTimestampedReports
         + ", detectInlinedCode=" + detectInlinedCode + ", exportLineCoverage="
-        + exportLineCoverage + ", mutationThreshold=" + mutationThreshold
+        + exportLineCoverage + ", mutationThreshold=" + mutationThreshold + ", testStrengthThreshold=" + testStrengthThreshold
         + ", coverageThreshold=" + coverageThreshold + ", mutationEngine="
         + mutationEngine + ", javaExecutable=" + javaExecutable
         + ", includeLaunchClasspath=" + includeLaunchClasspath + ", properties="
@@ -649,7 +659,5 @@ public String toString() {
         + ", skipFailingTests=" + skipFailingTests + "]";
   }
 
-
-
 
 }

pitest-entry/src/main/java/org/pitest/mutationtest/statistics/MutationStatistics.java

@@ -24,13 +24,15 @@ public final class MutationStatistics {
   private final long totalMutations;
   private final long numberOfTestsRun;
   private final long totalDetected;
+  private final long totalWithCoverage;
 
   public MutationStatistics(Iterable<Score> scores, long totalMutations,
-      long totalDetected, long numberOfTestsRun) {
+                            long totalDetected, long totalWithCoverage, long numberOfTestsRun) {
     this.scores = scores;
     this.totalMutations = totalMutations;
     this.totalDetected = totalDetected;
     this.numberOfTestsRun = numberOfTestsRun;
+    this.totalWithCoverage = totalWithCoverage;
   }
 
   public Iterable<Score> getScores() {
@@ -45,6 +47,14 @@ public long getTotalDetectedMutations() {
     return this.totalDetected;
   }
 
+  public long getTotalMutationsWithCoverage() {
+    return this.totalWithCoverage;
+  }
+
+  private long getTotalMutationsWithoutCoverage() {
+    return this.totalMutations - this.totalWithCoverage;
+  }
+
   public long getTotalSurvivingMutations() {
     return getTotalMutations() - getTotalDetectedMutations();
   }
@@ -66,6 +76,8 @@ public void report(final PrintStream out) {
     out.println(">> Generated " + this.getTotalMutations()
         + " mutations Killed " + this.getTotalDetectedMutations() + " ("
         + this.getPercentageDetected() + "%)");
+    out.println(">> Mutations with no coverage " + this.getTotalMutationsWithoutCoverage()
+            + ". Test strength " + this.getTestStrength() + "%");
     out.println(">> Ran " + this.numberOfTestsRun + " tests ("
         + getTestsPerMutation() + " tests per mutation)");
 
@@ -82,4 +94,16 @@ private String getTestsPerMutation() {
         .format(testsPerMutation);
   }
 
+  public long getTestStrength() {
+    if (getTotalMutations() == 0) {
+      return 100;
+    }
+
+    if (getTotalMutationsWithCoverage() == 0) {
+      return 0;
+    }
+
+    return Math.round((100f / getTotalMutationsWithCoverage())
+            * getTotalDetectedMutations());
+  }
 }

pitest-entry/src/main/java/org/pitest/mutationtest/statistics/MutationStatisticsPrecursor.java

@@ -38,7 +38,8 @@ public MutationStatistics toStatistics() {
     final long totalMutations = FCollection.fold(addTotals(), 0L, scores);
     final long totalDetected = FCollection
         .fold(addDetectedTotals(), 0L, scores);
-    return new MutationStatistics(scores, totalMutations, totalDetected,
+    final long totalWithCoverage = FCollection.fold(addCoveredTotals(), 0L, scores);
+    return new MutationStatistics(scores, totalMutations, totalDetected, totalWithCoverage,
         this.numberOfTestsRun);
   }
 
@@ -57,4 +58,8 @@ private static BiFunction<Long, Score, Long> addTotals() {
   private static BiFunction<Long, Score, Long> addDetectedTotals() {
     return (a, b) -> a + b.getTotalDetectedMutations();
   }
+
+  private static BiFunction<Long, Score, Long> addCoveredTotals() {
+    return (a, b) -> a + b.getTotalWithCoverage();
+  }
 }

pitest-entry/src/main/java/org/pitest/mutationtest/statistics/Score.java

@@ -22,13 +22,15 @@ public final class Score {
   private final Iterable<StatusCount> counts;
   private final long                  totalMutations;
   private final long                  totalDetected;
+  private final long totalWithCoverage;
 
   public Score(final String name, Iterable<StatusCount> counts,
-      long totalMutations, long totalDetected) {
+               long totalMutations, long totalDetected, long totalWithCoverage) {
     this.mutatorName = name;
     this.counts = counts;
     this.totalMutations = totalMutations;
     this.totalDetected = totalDetected;
+    this.totalWithCoverage = totalWithCoverage;
   }
 
   public void report(final PrintStream out) {
@@ -60,6 +62,10 @@ public long getTotalDetectedMutations() {
     return this.totalDetected;
   }
 
+  public long getTotalWithCoverage() {
+    return this.totalWithCoverage;
+  }
+
   public int getPercentageDetected() {
     if (getTotalMutations() == 0) {
       return 100;

pitest-entry/src/main/java/org/pitest/mutationtest/statistics/ScorePrecursor.java

@@ -36,10 +36,19 @@ private long getTotalDetectedMutations() {
         FCollection.filter(this.counts.values(), isDetected()));
   }
 
+  private long getTotalMutationsWithCoverage() {
+    return FCollection.fold(addTotals(), 0L,
+            FCollection.filter(this.counts.values(), hasCoverage()));
+  }
+
   private static Predicate<StatusCount> isDetected() {
     return a -> a.getStatus().isDetected();
   }
 
+  private static Predicate<StatusCount> hasCoverage() {
+    return a -> a.getStatus().hasCoverage();
+  }
+
   private BiFunction<Long, StatusCount, Long> addTotals() {
     return (a, b) -> a + b.getCount();
   }
@@ -54,7 +63,7 @@ private static Map<DetectionStatus, StatusCount> createMap() {
 
   Score toScore() {
     return new Score(this.mutatorName, this.getCounts(), this.getTotalMutations(),
-        this.getTotalDetectedMutations());
+        this.getTotalDetectedMutations(), this.getTotalMutationsWithCoverage());
   }
 }