Skip to content

Commit

Permalink
chore(test) Add pitest for mutation testing
Browse files Browse the repository at this point in the history
- Add profiles to run mutation tests
- Generate mutation reports for each module
- TODO: mutation tests for Spring Boot modules fail
- TODO: report aggregation doesn't work for now (see hcoles/pitest#622 and hcoles/pitest#621)
- Add coverage report in codecov
- TODO: run mutation tests in Travis or other CI tool for develop and master branches
  • Loading branch information
aurelien-baudet committed Jun 17, 2019
1 parent 96d2f43 commit 2e3f007
Show file tree
Hide file tree
Showing 16 changed files with 556 additions and 80 deletions.
5 changes: 3 additions & 2 deletions .externalToolBuilders/asciidoctor.launch
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.ui.externaltools.ProgramBuilderLaunchConfigurationType">
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/pom.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/docs&quot; type=&quot;2&quot;/&gt;&#10;&lt;/resources&gt;}"/>
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_BUILDER_ENABLED" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/src/docs&quot; type=&quot;2&quot;/&gt;&#10;&lt;/resources&gt;}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/ogham/mvnw}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="auto,"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="-Dmaven.site.skip&#10;post-site&#10;-N"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="post-site&#10;-N&#10;-Dmaven.site.skip&#10;-Dsass.skip=true"/>
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${workspace_loc:/ogham}"/>
</launchConfiguration>
1 change: 1 addition & 0 deletions .externalToolBuilders/compile sass.launch
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.ui.externaltools.ProgramBuilderLaunchConfigurationType">
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="true"/>
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_BUILDER_ENABLED" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/src/site/scss&quot; type=&quot;2&quot;/&gt;&#10;&lt;/resources&gt;}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/ogham/mvnw}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="auto,"/>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.ui.externaltools.ProgramBuilderLaunchConfigurationType">
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/pom.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/docs&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/asciidoc&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/resources&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/site.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;/resources&gt;}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/pom.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/asciidoc&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/resources&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/site.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;/resources&gt;}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/ogham/mvnw}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="auto,"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="clean post-site -N&#10;-DgenerateProjectInfo=false&#10;-DgenerateReports=false&#10;-DgenerateSitemap=false&#10;-Dmaven.javadoc.skip=true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="post-site -N&#10;-DgenerateProjectInfo=false&#10;-DgenerateReports=false&#10;-DgenerateSitemap=false&#10;-Dmaven.javadoc.skip=true&#10;-Dmpir.skip=true&#10;-Dasciidoctor.skip=true&#10;-Dsass.skip=true"/>
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${workspace_loc:/ogham}"/>
</launchConfiguration>
318 changes: 252 additions & 66 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
language: java
dist: trusty
jdk:
- oraclejdk9
- openjdk9
- oraclejdk8
- openjdk8
- openjdk10
- oraclejdk11
- openjdk11
#- oraclejdk12
- openjdk12
#jdk:
# - oraclejdk9
# - openjdk9
# - oraclejdk8
# - openjdk8
# - openjdk10
# - oraclejdk11
# - openjdk11
# #- oraclejdk12
# - openjdk12
sudo: required
addons:
apt:
Expand All @@ -25,75 +25,261 @@ addons:
branches:
except:
- gh-.*



before_install:
- echo "MAVEN_OPTS='-Xms512m -Xmx1024m -XX:PermSize=256m -XX:MaxPermSize=512m'" > ~/.mavenrc # see https://github.com/travis-ci/travis-ci/issues/4613
#- echo "MAVEN_OPTS='-Xms512m -Xmx1024m -XX:PermSize=256m -XX:MaxPermSize=512m'" > ~/.mavenrc # see https://github.com/travis-ci/travis-ci/issues/4613
- set -o pipefail
# detect current Java version in order to execute classpath tests only for this version
- |
java_version=$(java -version 2>&1 | grep -i version | cut -d'"' -f2)
major=$(echo $java_version | cut -d'.' -f1)
[ "1" = "$major" ] \
&& echo "$java_version" | cut -d'.' -f2 > "$HOME/java-version.info" \
|| echo "$java_version" | cut -d'.' -f1 > "$HOME/java-version.info"
- JAVA_RELEASE_VERSION=$(cat "$HOME/java-version.info")
# start spring initializr for Ogham in background
- |
touch "$HOME/spring-initializr.log"
spring-initializr/./mvnw clean spring-boot:run -f spring-initializr > "$HOME/spring-initializr.log" &
echo "$!" > "$HOME/spring-initializr.pid"
tail -f "$HOME/spring-initializr.log" &
- grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr.log")
- cat "$HOME/spring-initializr.log"
- |
[ "$JAVA_RELEASE_VERSION" != "8" ] && exit 0
touch "$HOME/spring-initializr-older-versions.log"
spring-initializr/./mvnw clean spring-boot:run -f spring-initializr -Dspring-boot.version=1.5.21.RELEASE -Dspring-initializr.version=0.4.0.RELEASE -Drun.jvmArguments="-Dspring.profiles.active=older-versions" > "$HOME/spring-initializr-older-versions.log" &
echo "$!" > "$HOME/spring-initializr-older-versions.pid"
tail -f "$HOME/spring-initializr-older-versions.log" &
- grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr-older-versions.log")
- cat "$HOME/spring-initializr-older-versions.log"
# generate projects used to test the classpath on the current Ogham version
- OGHAM_VERSION=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:1.3.1:exec)
# TODO: it seems that parallel execution sometimes fails on Travis (seems to be spring-initializr that fails, why ?)
- ./mvnw spring-boot:run -pl :ogham-test-classpath -Dspring-boot.run.arguments="$HOME/classpath-tests" -Dspring-boot.run.jvmArguments="-Drunner.parallel=false -Dogham-version=$OGHAM_VERSION -Dspring.initializer.url=http://localhost:8050/starter.zip"
- |
[ "$JAVA_RELEASE_VERSION" != "8" ] && exit 0
./mvnw spring-boot:run -pl :ogham-test-classpath -Dspring-boot.run.arguments="$HOME/classpath-tests" -Dspring-boot.run.jvmArguments="-Drunner.parallel=false -Dogham-version=$OGHAM_VERSION -Dspring.profiles.active=older-versions -Dspring.initializer.url=http://localhost:8052/starter.zip"
- ls -l "$HOME"
- ls -l "$HOME/classpath-tests"
# stop spring initializr
- kill `cat "$HOME/spring-initializr.pid"` || echo ""
- kill `cat "$HOME/spring-initializr-older-versions.pid"` || echo ""
- TEST_FOLDER=$(echo "JAVA_$JAVA_RELEASE_VERSION")
- echo $TEST_FOLDER
- ls -l "$HOME/classpath-tests/$TEST_FOLDER"
script:
- ./mvnw clean install -DskipTests=true -Dmaven.javadoc.skip=true -Dskip.integration.tests=true -Dskip.unit.tests=true -B -q
- ./mvnw test verify -Dmaven.test.redirectTestOutputToFile=true -fae -B -q
- ./mvnw javadoc:javadoc -fae -B | grep -Ei '(error|warn)'
# execute generated classpath tests for the right java version
- $HOME/classpath-tests/$TEST_FOLDER/./mvnw test -T 2.5C -f "$HOME/classpath-tests/$TEST_FOLDER/pom.xml" -Dmaven.test.redirectTestOutputToFile=true -fae -q -B #| grep -Ei '(error|exception|Tests run:)'
after_success:
- ./mvnw sonar:sonar -B | grep -Ei '(warn|error|ANALYSIS SUCCESSFUL)'
- ./mvnw deploy --settings .travis-settings.xml -DskipTests=true -B | grep -Ei '(Uploading|Downloading|Uploaded|Downloaded)'
# TODO: after error -> grab logs and send it
after_failure:
- cat "$HOME/spring-initializr.log"
- cat "$HOME/spring-initializr-older-versions.log"
#install: true
#script: true

jobs:
include:
- stage: "Build"
jdk: oraclejdk8
script:
- build/./build.sh
- stage: "Build"
jdk: openjdk8
script:
- build/./build.sh
- stage: "Build"
jdk: oraclejdk9
script:
- build/./build.sh
- stage: "Build"
jdk: openjdk9
script:
- build/./build.sh
- stage: "Build"
jdk: openjdk10
script:
- build/./build.sh
- stage: "Build"
jdk: oraclejdk11
script:
- build/./build.sh
- stage: "Build"
jdk: openjdk11
script:
- build/./build.sh
- stage: "Build"
jdk: openjdk12
script:
- build/./build.sh

- stage: test
name: "Unit Tests (oraclejdk8)"
jdk: oraclejdk8
script: build/./run-unit-tests.sh
- stage: test
name: "Unit Tests (openjdk8)"
jdk: openjdk8
script: build/./run-unit-tests.sh
- stage: test
name: "Unit Tests (oraclejdk9)"
jdk: oraclejdk9
script: build/./run-unit-tests.sh
- stage: test
name: "Unit Tests (openjdk9)"
jdk: openjdk9
script: build/./run-unit-tests.sh
- stage: test
name: "Unit Tests (openjdk10)"
jdk: openjdk10
script: build/./run-unit-tests.sh
- stage: test
name: "Unit Tests (oraclejdk11)"
jdk: oraclejdk11
script: build/./run-unit-tests.sh
- stage: test
name: "Unit Tests (openjdk11)"
jdk: openjdk11
script: build/./run-unit-tests.sh
- stage: test
name: "Unit Tests (openjdk12)"
jdk: openjdk12
script: build/./run-unit-tests.sh

- stage: test
name: "Integration Tests (oraclejdk8)"
jdk: oraclejdk8
script: build/./run-integration-tests.sh
- stage: test
name: "Integration Tests (openjdk8)"
jdk: openjdk8
script: build/./run-integration-tests.sh
- stage: test
name: "Integration Tests (oraclejdk9)"
jdk: oraclejdk9
script: build/./run-integration-tests.sh
- stage: test
name: "Integration Tests (openjdk9)"
jdk: openjdk9
script: build/./run-integration-tests.sh
- stage: test
name: "Integration Tests (openjdk10)"
jdk: openjdk10
script: build/./run-integration-tests.sh
- stage: test
name: "Integration Tests (oraclejdk11)"
jdk: oraclejdk11
script: build/./run-integration-tests.sh
- stage: test
name: "Integration Tests (openjdk11)"
jdk: openjdk11
script: build/./run-integration-tests.sh
- stage: test
name: "Integration Tests (openjdk12)"
jdk: openjdk12
script: build/./run-integration-tests.sh

- stage: test
name: "Classpath Tests (oraclejdk8)"
jdk: oraclejdk8
script:
- TEST_FOLDER=JAVA_8 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 build/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_8 INITIALIZER_NAME=spring-initializr-older-versions INITIALIZER_PORT=8052 INITIALIZER_ARGS='-Dspring-boot.version=1.5.21.RELEASE -Dspring-initializr.version=0.4.0.RELEASE -Drun.jvmArguments="-Dspring.profiles.active=older-versions"' GENERATION_ARGS="-Dspring.profiles.active=older-versions" build/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_8 build/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (openjdk8)"
jdk: openjdk8
script:
- TEST_FOLDER=JAVA_8 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 build/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_8 INITIALIZER_NAME=spring-initializr-older-versions INITIALIZER_PORT=8052 INITIALIZER_ARGS='-Dspring-boot.version=1.5.21.RELEASE -Dspring-initializr.version=0.4.0.RELEASE -Drun.jvmArguments="-Dspring.profiles.active=older-versions"' GENERATION_ARGS="-Dspring.profiles.active=older-versions" build/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_8 build/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (oraclejdk9)"
jdk: oraclejdk9
script:
- TEST_FOLDER=JAVA_9 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 build/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_9 build/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (openjdk9)"
jdk: openjdk9
script:
- TEST_FOLDER=JAVA_9 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 build/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_9 build/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (openjdk10)"
jdk: openjdk10
script:
- TEST_FOLDER=JAVA_10 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 build/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_10 build/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (oraclejdk11)"
jdk: oraclejdk11
script:
- TEST_FOLDER=JAVA_11 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 build/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_11 build/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (openjdk11)"
jdk: openjdk11
script:
- TEST_FOLDER=JAVA_11 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 build/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_11 build/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (openjdk12)"
jdk: openjdk12
script:
- TEST_FOLDER=JAVA_12 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 build/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_12 build/./run-classpath-tests.sh

- stage: "Analyze"
jdk: oraclejdk8
script:
- build/./run-unit-tests.sh
- build/./run-integration-tests.sh
- build/./analyze.sh

- stage: "Javadoc"
jdk: oraclejdk8
script:
- build/./generate-javadoc.sh
- stage: "Javadoc"
jdk: openjdk8
script:
- build/./generate-javadoc.sh
- stage: "Javadoc"
jdk: oraclejdk9
script:
- build/./generate-javadoc.sh
- stage: "Javadoc"
jdk: openjdk9
script:
- build/./generate-javadoc.sh
- stage: "Javadoc"
jdk: openjdk10
script:
- build/./generate-javadoc.sh
- stage: "Javadoc"
jdk: oraclejdk11
script:
- build/./generate-javadoc.sh
- stage: "Javadoc"
jdk: openjdk11
script:
- build/./generate-javadoc.sh
- stage: "Javadoc"
jdk: openjdk12
script:
- build/./generate-javadoc.sh

- stage: "Documentation"
jdk: oraclejdk8
script:
- build/./build.sh # need to rebuild and run tests again as files are not shared
- build/./generate-documentation.sh
deploy:
skip_cleanup: true
provider: surge
project: $HOME/docs/
domain: ogham-documentation.surge.sh
on:
all_branches: true

- stage: "Deploy"
jdk: oraclejdk8
script:
- build/./build.sh # need to rebuild and run tests again as files are not shared
- ./mvnw deploy --settings .travis-settings.xml -DskipTests=true -B | grep -Ei '(Uploading|Downloading|Uploaded|Downloaded)'


stages:
- "Build"
- test
- "Javadoc"
- "Analyze"
- "Documentation"
- "Deploy"

notifications:
slack:
rooms:
- secure: NdaSwFJ16HC93H1iimYB07/0rGwQfVCto2/XLrVJNeMcvdRsHbuhBnKwwUkNCTLeLLtB1DVmoNeCBdo2pmtAvoyTnl9OXhG3GWjoB++Al+/66x/TzC1g891y59JI/F7btVqZRKMi5URdcXt7VqSTUyu3Qjf6rxxeRMfiCgg4eLP2ABWQTvNpi7AgV9ArkVqat5Fo1p98iuxpUL0kByXmyuoN15f0azVm0Mhwv2LtMyHAf5Lq1gKOt7TfKccMEJaZXPZMfFWkXblZUWggx6EYXB2ylXJ7e45UeunsroqPYuXJtepwOjcT00NSm7tR+wDaVZl3hXDfexwb35wP+g/tcIiQYTWgQQvr5b6Z0tUX3WgZt7aBMH9gaJPRolC7FXwukXkdBwjvyDLsra147M2pm7fc5zj73TV0gaEdqDewd+UgFA12Vm1gwC0eWH/HVhOZ0HGOQdYmtYkZJ67KTwes9B+SKDkrSF4Nq7T3svOWvGxzDCaw4CCOBY5U1xRc3VVOo/xtFa8dAKfxpo0IFy/qaHS4vIb2BDHJ0LU7+WarcjcmADxJXvhvOvvByS5k0sqkUXF4uB+QpxQqQEGNMzLC2iMog0UnBIIEZEe6pvrzchoQIpKyom4GvnAi+Xco8aJaMgrLZY9JwmlLP8r1YFyMawH4kmJ8XpPZ5+fIN2HSkD4=
template:
- "[%{result}] %{branch}"
- "*[%{result}]* %{branch}"
- "Build: <%{build_url}|#%{build_number}> (%{duration})"
- "Commit: <%{compare_url}|%{commit}> by %{author}"
- "%{commit_message}"
- ""
- "---"
- ""
- "Documentation and reports:
- " - <http://ogham-documentation.surge.sh/%{branch}/index.html|Generated site>"
- " - <http://ogham-documentation.surge.sh/%{branch}/jacoco-aggregate/index.html|JaCoCo>"
- " - <https://codecov.io/gh/groupe-sii/ogham/branch/${branch}|Codecov>"
- " - <http://ogham-documentation.surge.sh/%{branch}/pit-reports/index.html|Mutation testing>"
cache:
directories:
# - $HOME/classpath-tests
- $HOME/.m2
# - $HOME/cache
before_cache:
- rm -rf $HOME/.m2/repository/fr/sii
- rm -rf $HOME/.m2/repository/fr/sii
# - find . -type d -name target -exec cp -rf --parents {} $HOME/cache \;
#before_script:
# - ls -l $HOME/cache/ || true
# - cp -rf $HOME/cache/* .
7 changes: 7 additions & 0 deletions build/analyze.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/bin/bash

# Sonar
./mvnw sonar:sonar -B | grep -Ei '(warn|error|ANALYSIS SUCCESSFUL)'

# Codecov
bash <(curl -s https://codecov.io/bash)
3 changes: 3 additions & 0 deletions build/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/sh

./mvnw clean install -DskipTests=true -Dmaven.javadoc.skip=true -Dskip.integration.tests=true -Dskip.unit.tests=true -B -q
Loading

0 comments on commit 2e3f007

Please sign in to comment.