From 14f013385f4ffeb1b6258c1f9a21c29ab8f7e500 Mon Sep 17 00:00:00 2001 From: abaudet Date: Fri, 14 Jun 2019 16:50:25 +0400 Subject: [PATCH] chore(test) Add pitest for mutation testing - Add profiles to run mutation tests - Generate mutation reports for each module - TODO: mutation tests for Spring Boot modules fail - TODO: report aggregation doesn't work for now (see https://github.com/hcoles/pitest/issues/622 and https://github.com/hcoles/pitest/issues/621) - Add coverage report in codecov - TODO: run mutation tests in Travis or other CI tool for develop and master branches --- .externalToolBuilders/asciidoctor.launch | 5 +- .externalToolBuilders/compile sass.launch | 1 + ...launch => site [target][no-report].launch} | 4 +- .travis.yml | 40 ++-- pom.xml | 175 +++++++++++++++++- reporting/pom.xml | 65 ++++++- src/site/site.xml | 8 +- 7 files changed, 273 insertions(+), 25 deletions(-) rename .externalToolBuilders/{site [target].launch => site [target][no-report].launch} (67%) diff --git a/.externalToolBuilders/asciidoctor.launch b/.externalToolBuilders/asciidoctor.launch index 57fb8f3242..44a1f8be7a 100644 --- a/.externalToolBuilders/asciidoctor.launch +++ b/.externalToolBuilders/asciidoctor.launch @@ -1,10 +1,11 @@ - + + - + diff --git a/.externalToolBuilders/compile sass.launch b/.externalToolBuilders/compile sass.launch index 09167ca136..b5eb54bf6b 100644 --- a/.externalToolBuilders/compile sass.launch +++ b/.externalToolBuilders/compile sass.launch @@ -1,6 +1,7 @@ + diff --git a/.externalToolBuilders/site [target].launch b/.externalToolBuilders/site [target][no-report].launch similarity index 67% rename from .externalToolBuilders/site [target].launch rename to .externalToolBuilders/site [target][no-report].launch index 217252139f..b5324402ff 100644 --- a/.externalToolBuilders/site [target].launch +++ b/.externalToolBuilders/site [target][no-report].launch @@ -1,10 +1,10 @@ - + - + diff --git a/.travis.yml b/.travis.yml index e4c46cc5c4..25c33fd02f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,23 +43,25 @@ before_install: spring-initializr/./mvnw clean spring-boot:run -f spring-initializr > "$HOME/spring-initializr.log" & echo "$!" > "$HOME/spring-initializr.pid" tail -f "$HOME/spring-initializr.log" & - - grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr.log") - - cat "$HOME/spring-initializr.log" + grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr.log") + #cat "$HOME/spring-initializr.log" - | - [ "$JAVA_RELEASE_VERSION" != "8" ] && exit 0 - touch "$HOME/spring-initializr-older-versions.log" - spring-initializr/./mvnw clean spring-boot:run -f spring-initializr -Dspring-boot.version=1.5.21.RELEASE -Dspring-initializr.version=0.4.0.RELEASE -Drun.jvmArguments="-Dspring.profiles.active=older-versions" > "$HOME/spring-initializr-older-versions.log" & - echo "$!" > "$HOME/spring-initializr-older-versions.pid" - tail -f "$HOME/spring-initializr-older-versions.log" & - - grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr-older-versions.log") - - cat "$HOME/spring-initializr-older-versions.log" + if [ "$JAVA_RELEASE_VERSION" == "8" ]; then + touch "$HOME/spring-initializr-older-versions.log" + spring-initializr/./mvnw clean spring-boot:run -f spring-initializr -Dspring-boot.version=1.5.21.RELEASE -Dspring-initializr.version=0.4.0.RELEASE -Drun.jvmArguments="-Dspring.profiles.active=older-versions" > "$HOME/spring-initializr-older-versions.log" & + echo "$!" > "$HOME/spring-initializr-older-versions.pid" + tail -f "$HOME/spring-initializr-older-versions.log" & + grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr-older-versions.log") + #cat "$HOME/spring-initializr-older-versions.log" + fi # generate projects used to test the classpath on the current Ogham version - OGHAM_VERSION=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:1.3.1:exec) # TODO: it seems that parallel execution sometimes fails on Travis (seems to be spring-initializr that fails, why ?) - ./mvnw spring-boot:run -pl :ogham-test-classpath -Dspring-boot.run.arguments="$HOME/classpath-tests" -Dspring-boot.run.jvmArguments="-Drunner.parallel=false -Dogham-version=$OGHAM_VERSION -Dspring.initializer.url=http://localhost:8050/starter.zip" - | - [ "$JAVA_RELEASE_VERSION" != "8" ] && exit 0 - ./mvnw spring-boot:run -pl :ogham-test-classpath -Dspring-boot.run.arguments="$HOME/classpath-tests" -Dspring-boot.run.jvmArguments="-Drunner.parallel=false -Dogham-version=$OGHAM_VERSION -Dspring.profiles.active=older-versions -Dspring.initializer.url=http://localhost:8052/starter.zip" + if [ "$JAVA_RELEASE_VERSION" == "8" ]; then + ./mvnw spring-boot:run -pl :ogham-test-classpath -Dspring-boot.run.arguments="$HOME/classpath-tests" -Dspring-boot.run.jvmArguments="-Drunner.parallel=false -Dogham-version=$OGHAM_VERSION -Dspring.profiles.active=older-versions -Dspring.initializer.url=http://localhost:8052/starter.zip" + fi - ls -l "$HOME" - ls -l "$HOME/classpath-tests" # stop spring initializr @@ -76,7 +78,23 @@ script: - $HOME/classpath-tests/$TEST_FOLDER/./mvnw test -T 2.5C -f "$HOME/classpath-tests/$TEST_FOLDER/pom.xml" -Dmaven.test.redirectTestOutputToFile=true -fae -q -B #| grep -Ei '(error|exception|Tests run:)' after_success: - ./mvnw sonar:sonar -B | grep -Ei '(warn|error|ANALYSIS SUCCESSFUL)' + - bash <(curl -s https://codecov.io/bash) - ./mvnw deploy --settings .travis-settings.xml -DskipTests=true -B | grep -Ei '(Uploading|Downloading|Uploaded|Downloaded)' + # generate site and reports + - | + #./mvnw post-site -Pmutation-testing + ./mvnw post-site +# upload reports +before_deploy: + - mkdir $HOME/$TRAVIS_BRANCH + - cp -r target/site $HOME/$TRAVIS_BRANCH/site +deploy: + provider: surge + project: $HOME/$TRAVIS_BRANCH/site + domain: ogham-documentation.surge.sh + on: + all_branches: true + # TODO: after error -> grab logs and send it after_failure: - cat "$HOME/spring-initializr.log" diff --git a/pom.xml b/pom.xml index 383e1d50ff..0bc73e419b 100644 --- a/pom.xml +++ b/pom.xml @@ -73,12 +73,15 @@ false false + + ${project.basedir}/../target/site/pit-reports + ${project.basedir}/../target/coverage-reports/jacoco-it.exec ${project.basedir}/../target/coverage-reports/jacoco-ut.exec ${project.basedir}/../target/coverage-reports/jacoco-all.exec ${project.basedir}/../target/site/jacoco-aggregate - + ${project.build.directory}/surefire-reports,${project.build.directory}/failsafe-reports ${jacoco.ut.execution.data.file},${jacoco.it.execution.data.file} @@ -100,6 +103,8 @@ ${project.build.outputDirectory}/staging ${site.staging.dir}/v${docProjectVersion} cerulean + ${maven.site.skip} + ${maven.site.skip} 2.10 @@ -109,6 +114,9 @@ 0.8.4 2.22.2 2.22.2 + + 1.4.7 + 1.3.7 3.7 3.0.0 @@ -583,6 +591,12 @@ surefireArgLine + + generate-xml-report + + report + + pre-integration-test @@ -592,6 +606,12 @@ failsafeArgLine + + generate-xml-integration-report + + report-integration + + @@ -709,6 +729,7 @@ copy-resources + ${asciidoctor.skip} src/docs/resources @@ -724,6 +745,7 @@ copy-resources + ${asciidoctor.skip} target/generated-docs @@ -807,6 +829,7 @@ unpack + ${maven.site.skip} io.github.devacfr.maven.skins @@ -875,6 +898,7 @@ run + ${maven.site.skip} @@ -915,6 +939,7 @@ + ${sass.skip} @@ -1100,6 +1125,41 @@ + + + eu.stamp-project + pitmp-maven-plugin + ${pitmp-maven-plugin.version} + + + run-mutation-testing + site + + run + + + descartes + + ogham-spring-boot-autoconfigure + ogham-spring-boot-starter-all + ogham-spring-boot-starter-email + ogham-spring-boot-starter-sms + ogham-spring-boot-common-autoconfigure + + + HTML + XML + + true + + + + + + org.pitest + pitest-maven + ${pitest-maven.version} + @@ -1140,6 +1200,34 @@ report-integration + + + report-aggregate + + false + + + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + + + + summary + licenses + modules + team + scm + issue-management + dependencies + dependency-info + dependency-convergence + dependency-management + plugins + + @@ -1220,6 +1308,91 @@ 8 + + mutation-testing + + + + + eu.stamp-project + pitmp-maven-plugin + + + + + + + + + + + + + + + + + + + + + + + + mutation-testing-only + + true + true + true + true + false + false + false + true + true + + + + + + eu.stamp-project + pitmp-maven-plugin + + + + + + + + + + + + + + + + + + + + + + + + mutation-testing-aggregate-report-only + + true + true + true + true + false + false + false + true + true + + diff --git a/reporting/pom.xml b/reporting/pom.xml index 86ecb4a2e1..9fb613c4dd 100644 --- a/reporting/pom.xml +++ b/reporting/pom.xml @@ -171,19 +171,14 @@ report - site + verify report-aggregate - - fr/sii/ogham/test/classpath/** - *mvnwrapper-for-projects/** - - ${jacoco.report.path} @@ -191,5 +186,63 @@ + + + + + org.pitest + pitest-maven + + + merge-pitest-reports + site + + report-aggregate + + + ${pit.report.path} + + + + + + + + + + mutation-testing + + + + org.pitest + pitest-maven + + + + + + mutation-testing-only + + + + org.pitest + pitest-maven + + + + + + mutation-testing-aggregate-report-only + + + + org.pitest + pitest-maven + + + + + + \ No newline at end of file diff --git a/src/site/site.xml b/src/site/site.xml index 1dae1a5d65..4328d3ff10 100644 --- a/src/site/site.xml +++ b/src/site/site.xml @@ -44,7 +44,7 @@ Main Download|Contribute Documentation - reports + Reports|Maven Reports - @@ -203,6 +202,9 @@ - + + + + \ No newline at end of file