chore(test) Add pitest for mutation testing

- Add profiles to run mutation tests - Generate mutation reports for each module - TODO: mutation tests for Spring Boot modules fail - TODO: report aggregation doesn't work for now (see hcoles/pitest#622 and hcoles/pitest#621) - Add coverage report in codecov - TODO: run mutation tests in Travis or other CI tool for develop and master branches
groupe-sii · Jun 15, 2019 · 14f0133 · 14f0133
1 parent 96d2f43
commit 14f0133
Show file tree

Hide file tree

Showing 7 changed files with 273 additions and 25 deletions.
diff --git a/.externalToolBuilders/asciidoctor.launch b/.externalToolBuilders/asciidoctor.launch
@@ -1,10 +1,11 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <launchConfiguration type="org.eclipse.ui.externaltools.ProgramBuilderLaunchConfigurationType">
 <booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="true"/>
-<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/pom.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/docs&quot; type=&quot;2&quot;/&gt;&#10;&lt;/resources&gt;}"/>
+<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_BUILDER_ENABLED" value="true"/>
+<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/src/docs&quot; type=&quot;2&quot;/&gt;&#10;&lt;/resources&gt;}"/>
 <stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/ogham/mvnw}"/>
 <stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="auto,"/>
-<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="-Dmaven.site.skip&#10;post-site&#10;-N"/>
+<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="post-site&#10;-N&#10;-Dmaven.site.skip&#10;-Dsass.skip=true"/>
 <booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
 <stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${workspace_loc:/ogham}"/>
 </launchConfiguration>
diff --git a/.externalToolBuilders/compile sass.launch b/.externalToolBuilders/compile sass.launch
@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <launchConfiguration type="org.eclipse.ui.externaltools.ProgramBuilderLaunchConfigurationType">
 <booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="true"/>
+<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_BUILDER_ENABLED" value="true"/>
 <stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/src/site/scss&quot; type=&quot;2&quot;/&gt;&#10;&lt;/resources&gt;}"/>
 <stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/ogham/mvnw}"/>
 <stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="auto,"/>

diff --git a/.externalToolBuilders/site [target].launch → ...lBuilders/site [target][no-report].launch b/.externalToolBuilders/site [target].launch → ...lBuilders/site [target][no-report].launch
@@ -1,10 +1,10 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <launchConfiguration type="org.eclipse.ui.externaltools.ProgramBuilderLaunchConfigurationType">
 <booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="true"/>
-<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/pom.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/docs&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/asciidoc&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/resources&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/site.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;/resources&gt;}"/>
+<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/pom.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/asciidoc&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/resources&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/site.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;/resources&gt;}"/>
 <stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/ogham/mvnw}"/>
 <stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="auto,"/>
-<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="clean post-site -N&#10;-DgenerateProjectInfo=false&#10;-DgenerateReports=false&#10;-DgenerateSitemap=false&#10;-Dmaven.javadoc.skip=true"/>
+<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="post-site -N&#10;-DgenerateProjectInfo=false&#10;-DgenerateReports=false&#10;-DgenerateSitemap=false&#10;-Dmaven.javadoc.skip=true&#10;-Dmpir.skip=true&#10;-Dasciidoctor.skip=true&#10;-Dsass.skip=true"/>
 <booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
 <stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${workspace_loc:/ogham}"/>
 </launchConfiguration>
diff --git a/.travis.yml b/.travis.yml
@@ -43,23 +43,25 @@ before_install:
     spring-initializr/./mvnw clean spring-boot:run -f spring-initializr > "$HOME/spring-initializr.log" &
     echo "$!" > "$HOME/spring-initializr.pid"
     tail -f "$HOME/spring-initializr.log" &
-  - grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr.log")
-  - cat "$HOME/spring-initializr.log"
+    grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr.log")
+    #cat "$HOME/spring-initializr.log"
   - |
-    [ "$JAVA_RELEASE_VERSION" != "8" ] && exit 0
-    touch "$HOME/spring-initializr-older-versions.log"
-    spring-initializr/./mvnw clean spring-boot:run -f spring-initializr -Dspring-boot.version=1.5.21.RELEASE -Dspring-initializr.version=0.4.0.RELEASE -Drun.jvmArguments="-Dspring.profiles.active=older-versions" > "$HOME/spring-initializr-older-versions.log" &
-    echo "$!" > "$HOME/spring-initializr-older-versions.pid"
-    tail -f "$HOME/spring-initializr-older-versions.log" &
-  - grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr-older-versions.log")
-  - cat "$HOME/spring-initializr-older-versions.log"
+    if [ "$JAVA_RELEASE_VERSION" == "8" ]; then
+      touch "$HOME/spring-initializr-older-versions.log"
+      spring-initializr/./mvnw clean spring-boot:run -f spring-initializr -Dspring-boot.version=1.5.21.RELEASE -Dspring-initializr.version=0.4.0.RELEASE -Drun.jvmArguments="-Dspring.profiles.active=older-versions" > "$HOME/spring-initializr-older-versions.log" &
+      echo "$!" > "$HOME/spring-initializr-older-versions.pid"
+      tail -f "$HOME/spring-initializr-older-versions.log" &
+      grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr-older-versions.log")
+      #cat "$HOME/spring-initializr-older-versions.log"
+    fi
   # generate projects used to test the classpath on the current Ogham version
   - OGHAM_VERSION=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:1.3.1:exec)
   # TODO: it seems that parallel execution sometimes fails on Travis (seems to be spring-initializr that fails, why ?)
   - ./mvnw spring-boot:run -pl :ogham-test-classpath -Dspring-boot.run.arguments="$HOME/classpath-tests" -Dspring-boot.run.jvmArguments="-Drunner.parallel=false -Dogham-version=$OGHAM_VERSION -Dspring.initializer.url=http://localhost:8050/starter.zip"
   - |
-    [ "$JAVA_RELEASE_VERSION" != "8" ] && exit 0
-    ./mvnw spring-boot:run -pl :ogham-test-classpath -Dspring-boot.run.arguments="$HOME/classpath-tests" -Dspring-boot.run.jvmArguments="-Drunner.parallel=false -Dogham-version=$OGHAM_VERSION -Dspring.profiles.active=older-versions -Dspring.initializer.url=http://localhost:8052/starter.zip"
+    if [ "$JAVA_RELEASE_VERSION" == "8" ]; then
+      ./mvnw spring-boot:run -pl :ogham-test-classpath -Dspring-boot.run.arguments="$HOME/classpath-tests" -Dspring-boot.run.jvmArguments="-Drunner.parallel=false -Dogham-version=$OGHAM_VERSION -Dspring.profiles.active=older-versions -Dspring.initializer.url=http://localhost:8052/starter.zip"
+    fi
   - ls -l "$HOME"
   - ls -l "$HOME/classpath-tests"
   # stop spring initializr
@@ -76,7 +78,23 @@ script:
   - $HOME/classpath-tests/$TEST_FOLDER/./mvnw test -T 2.5C -f "$HOME/classpath-tests/$TEST_FOLDER/pom.xml" -Dmaven.test.redirectTestOutputToFile=true -fae -q -B #| grep -Ei '(error|exception|Tests run:)'
 after_success:
   - ./mvnw sonar:sonar -B | grep -Ei '(warn|error|ANALYSIS SUCCESSFUL)'
+  - bash <(curl -s https://codecov.io/bash)
   - ./mvnw deploy --settings .travis-settings.xml -DskipTests=true -B | grep -Ei '(Uploading|Downloading|Uploaded|Downloaded)'
+  # generate site and reports
+  - |
+    #./mvnw post-site -Pmutation-testing
+    ./mvnw post-site
+# upload reports
+before_deploy:
+  - mkdir $HOME/$TRAVIS_BRANCH
+  - cp -r target/site $HOME/$TRAVIS_BRANCH/site
+deploy:
+  provider: surge
+  project: $HOME/$TRAVIS_BRANCH/site
+  domain: ogham-documentation.surge.sh
+  on:
+    all_branches: true
+
 # TODO: after error -> grab logs and send it
 after_failure:
   - cat "$HOME/spring-initializr.log"

diff --git a/pom.xml b/pom.xml
@@ -73,12 +73,15 @@
 		<skip.integration.tests>false</skip.integration.tests>
 		<skip.unit.tests>false</skip.unit.tests>
 
+		<!-- Mutation testing -->
+		<pit.report.path>${project.basedir}/../target/site/pit-reports</pit.report.path>
+
 		<!-- Code coverage -->
 		<jacoco.it.execution.data.file>${project.basedir}/../target/coverage-reports/jacoco-it.exec</jacoco.it.execution.data.file>
 		<jacoco.ut.execution.data.file>${project.basedir}/../target/coverage-reports/jacoco-ut.exec</jacoco.ut.execution.data.file>
 		<jacoco.all.execution.data.file>${project.basedir}/../target/coverage-reports/jacoco-all.exec</jacoco.all.execution.data.file>
 		<jacoco.report.path>${project.basedir}/../target/site/jacoco-aggregate</jacoco.report.path>
-		
+
 		<!-- Sonar properties -->
 		<sonar.junit.reportPaths>${project.build.directory}/surefire-reports,${project.build.directory}/failsafe-reports</sonar.junit.reportPaths>
 		<sonar.jacoco.reportPaths>${jacoco.ut.execution.data.file},${jacoco.it.execution.data.file}</sonar.jacoco.reportPaths>
@@ -100,6 +103,8 @@
 		<site.staging.dir>${project.build.outputDirectory}/staging</site.staging.dir>
 		<site.output.dir>${site.staging.dir}/v${docProjectVersion}</site.output.dir>
 		<doc-theme>cerulean</doc-theme>
+		<asciidoctor.skip>${maven.site.skip}</asciidoctor.skip>
+		<sass.skip>${maven.site.skip}</sass.skip>
 
 		<!-- Plugins -->
 		<maven-eclipse-plugin.version>2.10</maven-eclipse-plugin.version>
@@ -109,6 +114,9 @@
 		<jacoco-maven-plugin.version>0.8.4</jacoco-maven-plugin.version>
 		<maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version>
 		<maven-failsafe-plugin.version>2.22.2</maven-failsafe-plugin.version>
+		<!-- Plugins - mutation testing -->
+		<pitest-maven.version>1.4.7</pitest-maven.version>
+		<pitmp-maven-plugin.version>1.3.7</pitmp-maven-plugin.version>
 		<!-- Plugins - documentation -->
 		<maven-site-plugin.version>3.7</maven-site-plugin.version>
 		<maven-project-info-reports-plugin.version>3.0.0</maven-project-info-reports-plugin.version>
@@ -583,6 +591,12 @@
 							<propertyName>surefireArgLine</propertyName>
 						</configuration>
 					</execution>
+					<execution>
+						<id>generate-xml-report</id>
+						<goals>
+							<goal>report</goal>
+						</goals>
+					</execution>
 					<execution>
 						<id>pre-integration-test</id>
 						<goals>
@@ -592,6 +606,12 @@
 							<propertyName>failsafeArgLine</propertyName>
 						</configuration>
 					</execution>
+					<execution>
+						<id>generate-xml-integration-report</id>
+						<goals>
+							<goal>report-integration</goal>
+						</goals>
+					</execution>
 				</executions>
 			</plugin>
 			<!-- Used for unit tests -->
@@ -709,6 +729,7 @@
 							<goal>copy-resources</goal>
 						</goals>
 						<configuration>
+							<skip>${asciidoctor.skip}</skip>
 							<resources>
 								<resource>
 									<directory>src/docs/resources</directory>
@@ -724,6 +745,7 @@
 							<goal>copy-resources</goal>
 						</goals>
 						<configuration>
+							<skip>${asciidoctor.skip}</skip>
 							<resources>
 								<resource>
 									<directory>target/generated-docs</directory>
@@ -807,6 +829,7 @@
 							<goal>unpack</goal>
 						</goals>
 						<configuration>
+							<skip>${maven.site.skip}</skip>
 							<artifactItems>
 								<artifactItem>
 									<groupId>io.github.devacfr.maven.skins</groupId>
@@ -875,6 +898,7 @@
 							<goal>run</goal>
 						</goals>
 						<configuration>
+							<skip>${maven.site.skip}</skip>
 							<target>
 								<copy todir="target/generated-scss/asciidoctor">
 									<fileset dir="target/unpacked/gems">
@@ -915,6 +939,7 @@
 					</execution>
 				</executions>
 				<configuration>
+					<skip>${sass.skip}</skip>
 					<!-- Add resources in order to import files like it was in same source 
 						folder -->
 					<resources>
@@ -1100,6 +1125,41 @@
 						</lifecycleMappingMetadata>
 					</configuration>
 				</plugin>
+				<!-- Mutation testing -->
+				<plugin>
+					<groupId>eu.stamp-project</groupId>
+					<artifactId>pitmp-maven-plugin</artifactId>
+					<version>${pitmp-maven-plugin.version}</version>
+					<executions>
+						<execution>
+							<id>run-mutation-testing</id>
+							<phase>site</phase>
+							<goals>
+								<goal>run</goal>
+							</goals>
+							<configuration>
+								<mutationEngine>descartes</mutationEngine>
+								<skippedModules>
+									<param>ogham-spring-boot-autoconfigure</param>
+									<param>ogham-spring-boot-starter-all</param>
+									<param>ogham-spring-boot-starter-email</param>
+									<param>ogham-spring-boot-starter-sms</param>
+									<param>ogham-spring-boot-common-autoconfigure</param>
+								</skippedModules>
+								<outputFormats>
+									<outputFormat>HTML</outputFormat>
+									<outputFormat>XML</outputFormat>
+								</outputFormats>
+								<exportLineCoverage>true</exportLineCoverage>
+							</configuration>
+						</execution>
+					</executions>
+				</plugin>
+				<plugin>
+					<groupId>org.pitest</groupId>
+					<artifactId>pitest-maven</artifactId>
+					<version>${pitest-maven.version}</version>
+				</plugin>
 			</plugins>
 		</pluginManagement>
 	</build>
@@ -1140,6 +1200,34 @@
 							<report>report-integration</report>
 						</reports>
 					</reportSet>
+					<reportSet>
+						<reports>
+							<report>report-aggregate</report>
+						</reports>
+						<inherited>false</inherited>
+					</reportSet>
+				</reportSets>
+			</plugin>
+			<!-- Project info -->
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-project-info-reports-plugin</artifactId>
+				<reportSets>
+					<reportSet>
+						<reports>
+							<report>summary</report>
+							<report>licenses</report>
+							<report>modules</report>
+							<report>team</report>
+							<report>scm</report>
+							<report>issue-management</report>
+							<report>dependencies</report>
+							<report>dependency-info</report>
+							<report>dependency-convergence</report>
+							<report>dependency-management</report>
+							<report>plugins</report>
+						</reports>
+					</reportSet>
 				</reportSets>
 			</plugin>
 		</plugins>
@@ -1220,6 +1308,91 @@
 				<maven.compiler.release>8</maven.compiler.release>
 			</properties>
 		</profile>
+		<profile>
+			<id>mutation-testing</id>
+			<build>
+				<plugins>
+					<!-- Run mutation testing -->
+					<plugin>
+						<groupId>eu.stamp-project</groupId>
+						<artifactId>pitmp-maven-plugin</artifactId>
+					</plugin>
+				</plugins>
+			</build>
+			<!-- Add Maven report -->
+			<reporting>
+				<plugins>
+					<!-- <plugin> -->
+					<!-- <groupId>org.pitest</groupId> -->
+					<!-- <artifactId>pitest-maven</artifactId> -->
+					<!-- <version>${pitest-maven.version}</version> -->
+					<!-- <inherited>false</inherited> -->
+					<!-- <reportSets> -->
+					<!-- <reportSet> -->
+					<!-- <reports> -->
+					<!-- <report>report-aggregate-module</report> -->
+					<!-- </reports> -->
+					<!-- </reportSet> -->
+					<!-- </reportSets> -->
+					<!-- </plugin> -->
+				</plugins>
+			</reporting>
+		</profile>
+		<profile>
+			<id>mutation-testing-only</id>
+			<properties>
+				<skip.integration.tests>true</skip.integration.tests>
+				<skip.unit.tests>true</skip.unit.tests>
+				<sass.skip>true</sass.skip>
+				<asciidoctor.skip>true</asciidoctor.skip>
+				<generateProjectInfo>false</generateProjectInfo>
+				<generateReports>false</generateReports>
+				<generateSitemap>false</generateSitemap>
+				<maven.javadoc.skip>true</maven.javadoc.skip>
+				<mpir.skip>true</mpir.skip>
+			</properties>
+			<build>
+				<plugins>
+					<!-- Run mutation testing -->
+					<plugin>
+						<groupId>eu.stamp-project</groupId>
+						<artifactId>pitmp-maven-plugin</artifactId>
+					</plugin>
+				</plugins>
+			</build>
+			<!-- Add Maven report -->
+			<reporting>
+				<plugins>
+					<!-- <plugin> -->
+					<!-- <groupId>org.pitest</groupId> -->
+					<!-- <artifactId>pitest-maven</artifactId> -->
+					<!-- <version>${pitest-maven.version}</version> -->
+					<!-- <inherited>false</inherited> -->
+					<!-- <reportSets> -->
+					<!-- <reportSet> -->
+					<!-- <reports> -->
+					<!-- <report>report-aggregate-module</report> -->
+					<!-- </reports> -->
+					<!-- </reportSet> -->
+					<!-- </reportSets> -->
+					<!-- </plugin> -->
+				</plugins>
+			</reporting>
+		</profile>
+		<profile>
+			<id>mutation-testing-aggregate-report-only</id>
+			<properties>
+				<skip.integration.tests>true</skip.integration.tests>
+				<skip.unit.tests>true</skip.unit.tests>
+				<sass.skip>true</sass.skip>
+				<asciidoctor.skip>true</asciidoctor.skip>
+				<generateProjectInfo>false</generateProjectInfo>
+				<generateReports>false</generateReports>
+				<generateSitemap>false</generateSitemap>
+				<maven.javadoc.skip>true</maven.javadoc.skip>
+				<mpir.skip>true</mpir.skip>
+			</properties>
+		</profile>
 	</profiles>
 
 	<licenses>