Skip to content

Commit

Permalink
chore(test) Add pitest for mutation testing
Browse files Browse the repository at this point in the history
- Add profiles to run mutation tests
- Generate mutation reports for each module
- TODO: mutation tests for Spring Boot modules fail
- report aggregation doesn't work for now (see hcoles/pitest#622 and hcoles/pitest#621)
  => using a temporary fix
- Add coverage report in codecov
- Run mutation tests in Travis
  • Loading branch information
aurelien-baudet committed Jun 21, 2019
1 parent 96d2f43 commit 1bd9577
Show file tree
Hide file tree
Showing 17 changed files with 560 additions and 87 deletions.
16 changes: 16 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
version: 2
jobs:
build:
docker:
- image: circleci/openjdk:8-jdk
steps:
- checkout
- run: ci/./generate-documentation.sh
# TODO: deploy on surge
# TODO: notification with URLs
- slack/notify:
color: '#42e2f4'
message: This is a custom message notification
webhook: webhook
orbs:
slack: circleci/slack@3.2.0
5 changes: 3 additions & 2 deletions .externalToolBuilders/asciidoctor.launch
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.ui.externaltools.ProgramBuilderLaunchConfigurationType">
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/pom.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/docs&quot; type=&quot;2&quot;/&gt;&#10;&lt;/resources&gt;}"/>
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_BUILDER_ENABLED" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/src/docs&quot; type=&quot;2&quot;/&gt;&#10;&lt;/resources&gt;}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/ogham/mvnw}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="auto,"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="-Dmaven.site.skip&#10;post-site&#10;-N"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="post-site&#10;-N&#10;-Dmaven.site.skip&#10;-Dsass.skip=true"/>
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${workspace_loc:/ogham}"/>
</launchConfiguration>
1 change: 1 addition & 0 deletions .externalToolBuilders/compile sass.launch
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.ui.externaltools.ProgramBuilderLaunchConfigurationType">
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="true"/>
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_BUILDER_ENABLED" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/src/site/scss&quot; type=&quot;2&quot;/&gt;&#10;&lt;/resources&gt;}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/ogham/mvnw}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="auto,"/>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.ui.externaltools.ProgramBuilderLaunchConfigurationType">
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/pom.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/docs&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/asciidoc&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/resources&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/site.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;/resources&gt;}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE" value="${working_set:&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;resources&gt;&#10;&lt;item path=&quot;/ogham/pom.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/asciidoc&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/resources&quot; type=&quot;2&quot;/&gt;&#10;&lt;item path=&quot;/ogham/src/site/site.xml&quot; type=&quot;1&quot;/&gt;&#10;&lt;/resources&gt;}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/ogham/mvnw}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="auto,"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="clean post-site -N&#10;-DgenerateProjectInfo=false&#10;-DgenerateReports=false&#10;-DgenerateSitemap=false&#10;-Dmaven.javadoc.skip=true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="post-site -N&#10;-DgenerateProjectInfo=false&#10;-DgenerateReports=false&#10;-DgenerateSitemap=false&#10;-Dmaven.javadoc.skip=true&#10;-Dmpir.skip=true&#10;-Dasciidoctor.skip=true&#10;-Dsass.skip=true"/>
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${workspace_loc:/ogham}"/>
</launchConfiguration>
302 changes: 230 additions & 72 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,99 +1,257 @@
language: java
dist: trusty
jdk:
- oraclejdk9
- openjdk9
- oraclejdk8
- openjdk8
- openjdk10
- oraclejdk11
- openjdk11
#- oraclejdk12
- openjdk12
sudo: required
addons:
apt:
packages:
- graphviz
sonarcloud:
organization: aurelien-baudet-github
branches:
- master
- refactoring/.*
- features/.*
- test/.*
branches:
except:
- gh-.*



before_install:
- echo "MAVEN_OPTS='-Xms512m -Xmx1024m -XX:PermSize=256m -XX:MaxPermSize=512m'" > ~/.mavenrc # see https://github.com/travis-ci/travis-ci/issues/4613
- set -o pipefail
# detect current Java version in order to execute classpath tests only for this version
- |
java_version=$(java -version 2>&1 | grep -i version | cut -d'"' -f2)
major=$(echo $java_version | cut -d'.' -f1)
[ "1" = "$major" ] \
&& echo "$java_version" | cut -d'.' -f2 > "$HOME/java-version.info" \
|| echo "$java_version" | cut -d'.' -f1 > "$HOME/java-version.info"
- JAVA_RELEASE_VERSION=$(cat "$HOME/java-version.info")
# start spring initializr for Ogham in background
- |
touch "$HOME/spring-initializr.log"
spring-initializr/./mvnw clean spring-boot:run -f spring-initializr > "$HOME/spring-initializr.log" &
echo "$!" > "$HOME/spring-initializr.pid"
tail -f "$HOME/spring-initializr.log" &
- grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr.log")
- cat "$HOME/spring-initializr.log"
- |
[ "$JAVA_RELEASE_VERSION" != "8" ] && exit 0
touch "$HOME/spring-initializr-older-versions.log"
spring-initializr/./mvnw clean spring-boot:run -f spring-initializr -Dspring-boot.version=1.5.21.RELEASE -Dspring-initializr.version=0.4.0.RELEASE -Drun.jvmArguments="-Dspring.profiles.active=older-versions" > "$HOME/spring-initializr-older-versions.log" &
echo "$!" > "$HOME/spring-initializr-older-versions.pid"
tail -f "$HOME/spring-initializr-older-versions.log" &
- grep -q 'Started SpringInitializrApplication' <(tail -f "$HOME/spring-initializr-older-versions.log")
- cat "$HOME/spring-initializr-older-versions.log"
# generate projects used to test the classpath on the current Ogham version
- OGHAM_VERSION=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:1.3.1:exec)
# TODO: it seems that parallel execution sometimes fails on Travis (seems to be spring-initializr that fails, why ?)
- ./mvnw spring-boot:run -pl :ogham-test-classpath -Dspring-boot.run.arguments="$HOME/classpath-tests" -Dspring-boot.run.jvmArguments="-Drunner.parallel=false -Dogham-version=$OGHAM_VERSION -Dspring.initializer.url=http://localhost:8050/starter.zip"
- |
[ "$JAVA_RELEASE_VERSION" != "8" ] && exit 0
./mvnw spring-boot:run -pl :ogham-test-classpath -Dspring-boot.run.arguments="$HOME/classpath-tests" -Dspring-boot.run.jvmArguments="-Drunner.parallel=false -Dogham-version=$OGHAM_VERSION -Dspring.profiles.active=older-versions -Dspring.initializer.url=http://localhost:8052/starter.zip"
- ls -l "$HOME"
- ls -l "$HOME/classpath-tests"
# stop spring initializr
- kill `cat "$HOME/spring-initializr.pid"` || echo ""
- kill `cat "$HOME/spring-initializr-older-versions.pid"` || echo ""
- TEST_FOLDER=$(echo "JAVA_$JAVA_RELEASE_VERSION")
- echo $TEST_FOLDER
- ls -l "$HOME/classpath-tests/$TEST_FOLDER"
script:
- ./mvnw clean install -DskipTests=true -Dmaven.javadoc.skip=true -Dskip.integration.tests=true -Dskip.unit.tests=true -B -q
- ./mvnw test verify -Dmaven.test.redirectTestOutputToFile=true -fae -B -q
- ./mvnw javadoc:javadoc -fae -B | grep -Ei '(error|warn)'
# execute generated classpath tests for the right java version
- $HOME/classpath-tests/$TEST_FOLDER/./mvnw test -T 2.5C -f "$HOME/classpath-tests/$TEST_FOLDER/pom.xml" -Dmaven.test.redirectTestOutputToFile=true -fae -q -B #| grep -Ei '(error|exception|Tests run:)'
after_success:
- ./mvnw sonar:sonar -B | grep -Ei '(warn|error|ANALYSIS SUCCESSFUL)'
- ./mvnw deploy --settings .travis-settings.xml -DskipTests=true -B | grep -Ei '(Uploading|Downloading|Uploaded|Downloaded)'
# TODO: after error -> grab logs and send it
after_failure:
- cat "$HOME/spring-initializr.log"
- cat "$HOME/spring-initializr-older-versions.log"

jobs:
include:
- stage: "Build"
jdk: oraclejdk8
script:
- ci/./build.sh
- stage: "Build"
jdk: openjdk8
script:
- ci/./build.sh
- stage: "Build"
jdk: oraclejdk9
script:
- ci/./build.sh
- stage: "Build"
jdk: openjdk9
script:
- ci/./build.sh
- stage: "Build"
jdk: openjdk10
script:
- ci/./build.sh
- stage: "Build"
jdk: oraclejdk11
script:
- ci/./build.sh
- stage: "Build"
jdk: openjdk11
script:
- ci/./build.sh
- stage: "Build"
jdk: openjdk12
script:
- ci/./build.sh

- stage: test
name: "Unit Tests (oraclejdk8)"
jdk: oraclejdk8
script: ci/./run-unit-tests.sh
- stage: test
name: "Unit Tests (openjdk8)"
jdk: openjdk8
script: ci/./run-unit-tests.sh
- stage: test
name: "Unit Tests (oraclejdk9)"
jdk: oraclejdk9
script: ci/./run-unit-tests.sh
- stage: test
name: "Unit Tests (openjdk9)"
jdk: openjdk9
script: ci/./run-unit-tests.sh
- stage: test
name: "Unit Tests (openjdk10)"
jdk: openjdk10
script: ci/./run-unit-tests.sh
- stage: test
name: "Unit Tests (oraclejdk11)"
jdk: oraclejdk11
script: ci/./run-unit-tests.sh
- stage: test
name: "Unit Tests (openjdk11)"
jdk: openjdk11
script: ci/./run-unit-tests.sh
- stage: test
name: "Unit Tests (openjdk12)"
jdk: openjdk12
script: ci/./run-unit-tests.sh

- stage: test
name: "Integration Tests (oraclejdk8)"
jdk: oraclejdk8
script: ci/./run-integration-tests.sh
- stage: test
name: "Integration Tests (openjdk8)"
jdk: openjdk8
script: ci/./run-integration-tests.sh
- stage: test
name: "Integration Tests (oraclejdk9)"
jdk: oraclejdk9
script: ci/./run-integration-tests.sh
- stage: test
name: "Integration Tests (openjdk9)"
jdk: openjdk9
script: ci/./run-integration-tests.sh
- stage: test
name: "Integration Tests (openjdk10)"
jdk: openjdk10
script: ci/./run-integration-tests.sh
- stage: test
name: "Integration Tests (oraclejdk11)"
jdk: oraclejdk11
script: ci/./run-integration-tests.sh
- stage: test
name: "Integration Tests (openjdk11)"
jdk: openjdk11
script: ci/./run-integration-tests.sh
- stage: test
name: "Integration Tests (openjdk12)"
jdk: openjdk12
script: ci/./run-integration-tests.sh

- stage: test
name: "Classpath Tests (oraclejdk8)"
jdk: oraclejdk8
script:
- TEST_FOLDER=JAVA_8 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 ci/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_8 INITIALIZER_NAME=spring-initializr-older-versions INITIALIZER_PORT=8052 INITIALIZER_ARGS='-Dspring-boot.version=1.5.21.RELEASE -Dspring-initializr.version=0.4.0.RELEASE -Drun.jvmArguments="-Dspring.profiles.active=older-versions"' GENERATION_ARGS="-Dspring.profiles.active=older-versions" ci/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_8 ci/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (openjdk8)"
jdk: openjdk8
script:
- TEST_FOLDER=JAVA_8 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 ci/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_8 INITIALIZER_NAME=spring-initializr-older-versions INITIALIZER_PORT=8052 INITIALIZER_ARGS='-Dspring-boot.version=1.5.21.RELEASE -Dspring-initializr.version=0.4.0.RELEASE -Drun.jvmArguments="-Dspring.profiles.active=older-versions"' GENERATION_ARGS="-Dspring.profiles.active=older-versions" ci/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_8 ci/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (oraclejdk9)"
jdk: oraclejdk9
script:
- TEST_FOLDER=JAVA_9 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 ci/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_9 ci/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (openjdk9)"
jdk: openjdk9
script:
- TEST_FOLDER=JAVA_9 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 ci/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_9 ci/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (openjdk10)"
jdk: openjdk10
script:
- TEST_FOLDER=JAVA_10 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 ci/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_10 ci/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (oraclejdk11)"
jdk: oraclejdk11
script:
- TEST_FOLDER=JAVA_11 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 ci/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_11 ci/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (openjdk11)"
jdk: openjdk11
script:
- TEST_FOLDER=JAVA_11 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 ci/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_11 ci/./run-classpath-tests.sh
- stage: test
name: "Classpath Tests (openjdk12)"
jdk: openjdk12
script:
- TEST_FOLDER=JAVA_12 INITIALIZER_NAME=spring-initializr INITIALIZER_PORT=8050 ci/./generate-classpath-tests.sh
- TEST_FOLDER=JAVA_12 ci/./run-classpath-tests.sh

- stage: "Analyze"
jdk: oraclejdk8
script:
- ci/./analyze.sh

- stage: "Javadoc"
jdk: oraclejdk8
script:
- ci/./generate-javadoc.sh
- stage: "Javadoc"
jdk: openjdk8
script:
- ci/./generate-javadoc.sh
- stage: "Javadoc"
jdk: oraclejdk9
script:
- ci/./generate-javadoc.sh
- stage: "Javadoc"
jdk: openjdk9
script:
- ci/./generate-javadoc.sh
- stage: "Javadoc"
jdk: openjdk10
script:
- ci/./generate-javadoc.sh
- stage: "Javadoc"
jdk: oraclejdk11
script:
- ci/./generate-javadoc.sh
- stage: "Javadoc"
jdk: openjdk11
script:
- ci/./generate-javadoc.sh
- stage: "Javadoc"
jdk: openjdk12
script:
- ci/./generate-javadoc.sh

# - stage: "Documentation"
# jdk: oraclejdk8
# script:
# - travis_wait 30 ci/./generate-documentation.sh
# deploy:
# skip_cleanup: true
# provider: surge
# project: target/surge
# domain: ogham-documentation.surge.sh
# on:
# all_branches: true

- stage: "Deploy"
jdk: oraclejdk8
script:
- ./mvnw clean install deploy --settings .travis-settings.xml -DskipTests=true -B | grep -Ei '(Uploading|Downloading|Uploaded|Downloaded)'
#

stages:
- "Build"
- test
- "Javadoc"
- "Analyze"
- "Documentation"
- "Deploy"

notifications:
slack:
rooms:
- secure: NdaSwFJ16HC93H1iimYB07/0rGwQfVCto2/XLrVJNeMcvdRsHbuhBnKwwUkNCTLeLLtB1DVmoNeCBdo2pmtAvoyTnl9OXhG3GWjoB++Al+/66x/TzC1g891y59JI/F7btVqZRKMi5URdcXt7VqSTUyu3Qjf6rxxeRMfiCgg4eLP2ABWQTvNpi7AgV9ArkVqat5Fo1p98iuxpUL0kByXmyuoN15f0azVm0Mhwv2LtMyHAf5Lq1gKOt7TfKccMEJaZXPZMfFWkXblZUWggx6EYXB2ylXJ7e45UeunsroqPYuXJtepwOjcT00NSm7tR+wDaVZl3hXDfexwb35wP+g/tcIiQYTWgQQvr5b6Z0tUX3WgZt7aBMH9gaJPRolC7FXwukXkdBwjvyDLsra147M2pm7fc5zj73TV0gaEdqDewd+UgFA12Vm1gwC0eWH/HVhOZ0HGOQdYmtYkZJ67KTwes9B+SKDkrSF4Nq7T3svOWvGxzDCaw4CCOBY5U1xRc3VVOo/xtFa8dAKfxpo0IFy/qaHS4vIb2BDHJ0LU7+WarcjcmADxJXvhvOvvByS5k0sqkUXF4uB+QpxQqQEGNMzLC2iMog0UnBIIEZEe6pvrzchoQIpKyom4GvnAi+Xco8aJaMgrLZY9JwmlLP8r1YFyMawH4kmJ8XpPZ5+fIN2HSkD4=
template:
- "[%{result}] %{branch}"
- "*[%{result}]* %{branch}"
- "Build: <%{build_url}|#%{build_number}> (%{duration})"
- "Commit: <%{compare_url}|%{commit}> by %{author}"
- "%{commit_message}"
- ""
- "---"
- ""
- "Documentation and reports:"
- " * <http://ogham-documentation.surge.sh/%{branch}/index.html|Generated site>"
- " * <http://ogham-documentation.surge.sh/%{branch}/jacoco-aggregate/index.html|JaCoCo>"
- " * <https://codecov.io/gh/groupe-sii/ogham/branch/%{branch}|Codecov>"
- " * <http://ogham-documentation.surge.sh/%{branch}/pit-reports/index.html|Mutation testing>"
cache:
directories:
# - $HOME/classpath-tests
- $HOME/.m2
before_cache:
- rm -rf $HOME/.m2/repository/fr/sii
- rm -rf $HOME/.m2/repository/fr/sii
7 changes: 7 additions & 0 deletions ci/analyze.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/bin/bash

# Sonar
./mvnw clean install sonar:sonar -Dmaven.test.redirectTestOutputToFile=true -B | grep -Ei '(warn|error|ANALYSIS SUCCESSFUL)'

# Codecov
bash <(curl -s https://codecov.io/bash)
3 changes: 3 additions & 0 deletions ci/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/sh

./mvnw clean install -DskipTests=true -Dmaven.javadoc.skip=true -Dskip.integration.tests=true -Dskip.unit.tests=true -B -q
Loading

0 comments on commit 1bd9577

Please sign in to comment.