From 0e9c92e7d75569ee316246da3b2714add4a87b24 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Fri, 22 Apr 2016 15:30:35 -0700 Subject: [PATCH 01/27] Add docker-based ROS prerelease test. --- .travis.yml | 10 ++--- README.rst | 3 ++ ros_pre-release.sh | 93 ++++++++++++++++++++++++++++++++++++++++++++++ travis.sh | 6 +++ 4 files changed, 106 insertions(+), 6 deletions(-) create mode 100755 ros_pre-release.sh diff --git a/.travis.yml b/.travis.yml index 89752a5d7..1135f3a50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,20 +12,18 @@ compiler: # Whoever receives success and/or failure via email env: matrix: -# - ROS_DISTRO=hydro AFTER_INSTALL="" -# - ROS_DISTRO=hydro USE_CATKIN_MAKE=true BEFORE_SCRIPT="pwd; git clone http://github.com/jsk-ros-pkg/jsk_common_msgs" NOT_TEST_INSTALL=true -# - ROS_DISTRO=hydro AFTER_INSTALL="" TEST_PKGS=" " -# - ROS_DISTRO=hydro AFTER_INSTALL="" ROS_REPOSITORY_PATH=http://packages.ros.org/ros/ubuntu -# - ROS_DISTRO=hydro AFTER_INSTALL="mongodb-clients" -# - ROS_DISTRO=hydro BEFORE_SCRIPT="pwd; git clone https://github.com/ros-industrial/industrial_core" - ROS_DISTRO=indigo BUILDER='not-compile' - ROS_DISTRO=indigo BUILDER='not-compile' TRAVIS_REPO_SLUG=ros-industrial/industrial_ci - ROS_DISTRO=indigo BUILDER='not-compile' TRAVIS_REPO_SLUG=ros-industrial/industrial_ci TRAVIS_BRANCH=master TRAVIS_COMMIT='' TRAVIS_PULL_REQUEST='' - ROS_DISTRO=indigo NOT_TEST_BUILD='true' - ROS_DISTRO=indigo NOT_TEST_INSTALL='true' - ROS_DISTRO=indigo CATKIN_PARALLEL_JOBS='-p1' ROS_PARALLEL_JOBS='-j1' # Intend build on low-power platform + - ROS_DISTRO=indigo PRERELEASE=true + - ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 matrix: allow_failures: + - env: ROS_DISTRO=indigo PRERELEASE=true + - env: ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 - env: ROS_DISTRO=jade BUILDER='not-compile' - env: ROS_DISTRO=jade BUILDER='not-compile' TRAVIS_REPO_SLUG=ros-industrial/industrial_ci - env: ROS_DISTRO=jade BUILDER='not-compile' TRAVIS_REPO_SLUG=ros-industrial/industrial_ci TRAVIS_BRANCH=master TRAVIS_COMMIT='' TRAVIS_PULL_REQUEST='' diff --git a/README.rst b/README.rst index ccbbfb883..787817ece 100644 --- a/README.rst +++ b/README.rst @@ -118,6 +118,9 @@ Note that some of these currently tied only to a single option, but we still lea * `CI_PARENT_DIR` (default: .ci_config): (NOT recommended to specify) This is the folder name that is used in downstream repositories in order to point to this repo. * `NOT_TEST_BUILD` (default: not set): If true, tests in build space won't be run. * `NOT_TEST_INSTALL` (default: not set): If true, tests in `install` space won't be run. +* `PRERELEASE` (default: false): If `true`, run `prerelease test on docker that emulates ROS buildfarm `_. Since prerelease tests are usually necessary only when you are about to make a new version release, you're advised to put the tests that run prerelease into "allow_failures" section (or if your package is not planned to be released at all, no need to use it). See the usage sample in `.travis in indusrial_ci repository`_. +* `PRERELEASE_DOWNSTREAM_DEPTH` (0 to 4, default: 1): Number of the levels of the packages that depend on the prerelease test target package. Upper bound is what is defined by ROS buildfarm prerelease test website (``_ etc.) +* `PRERELEASE_REPONAME` (default: not set): The target of prerelease test (that you select at ``_, ``_ etc.) If not set then it tests the package of the repository's name. You can specify this by your ROS package name format (with underscore e.g. `industrial_core`), not Debian package name format. * `PKGS_DOWNSTREAM` (default: explained): Packages in downstream to be tested. By default, `TARGET_PKGS` is used if set, if not then `BUILD_PKGS` is used. * `ROS_PARALLEL_JOBS` (default: -j8): Maximum number of packages to be built in parallel by the underlining build tool. As of Jan 2016, this is only enabled with `catkin_tools` (with `make` as an underlining builder). * `ROS_PARALLEL_TEST_JOBS` (default: -j8): Maximum number of packages which could be examined in parallel during the test run by the underlining build tool. If not set it's filled by `ROS_PARALLEL_JOBS`. As of Jan 2016, this is only enabled with `catkin_tools` (with `make` as an underlining builder). diff --git a/ros_pre-release.sh b/ros_pre-release.sh new file mode 100755 index 000000000..c20eb7a3d --- /dev/null +++ b/ros_pre-release.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# Software License Agreement (BSD License) +# +# Copyright (c) 2016, Isaac I. Y. Saito +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the Isaac I. Y. Saito, nor the names +# of its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +## Author: Isaac I. Y. Saito + +set -x + +function travis_time_start { + set +x + TRAVIS_START_TIME=$(date +%s%N) + TRAVIS_TIME_ID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1) + TRAVIS_FOLD_NAME=$1 + echo -e "\e[0Ktravis_fold:start:$TRAVIS_FOLD_NAME" + echo -e "\e[0Ktravis_time:start:$TRAVIS_TIME_ID\e[34m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\e[0m" + set -x +} + +function travis_time_end { + set +x + _COLOR=${1:-32} + TRAVIS_END_TIME=$(date +%s%N) + TIME_ELAPSED_SECONDS=$(( ($TRAVIS_END_TIME - $TRAVIS_START_TIME)/1000000000 )) + echo -e "travis_time:end:$TRAVIS_TIME_ID:start=$TRAVIS_START_TIME,finish=$TRAVIS_END_TIME,duration=$(($TRAVIS_END_TIME - $TRAVIS_START_TIME))\e[0K" + echo -e "travis_fold:end:$TRAVIS_FOLD_NAME\e[${_COLOR}m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\e[0m" + echo -e "\e[0K\e[${_COLOR}mFunction $TRAVIS_FOLD_NAME took $(( $TIME_ELAPSED_SECONDS / 60 )) min $(( $TIME_ELAPSED_SECONDS % 60 )) sec\e[0m" + set -x +} + +function error { + travis_time_end 31 + trap - ERR + exit 1 +} + +if [ ! "$PRERELEASE_DOWNSTREAM_DEPTH" ]; then export PRERELEASE_DOWNSTREAM_DEPTH="1"; fi +if [ ! "$PRERELEASE_REPONAME" ]; then NAME_REPO=$(echo $TRAVIS_REPO_SLUG | cut -d'/' -f 1); fi +echo "PRERELEASE_REPONAME = ${PRERELEASE_REPONAME}" + +travis_time_start setup_docker + +sudo usermod -aG docker ubuntu +# ROS Buildfarm for prerelease http://wiki.ros.org/regression_tests#How_do_I_setup_my_system_to_run_a_prerelease.3F +sudo -E sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' +sudo -E apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 0xB01FA116 +# Buildfarm workaround for Python3 http://wiki.ros.org/regression_tests#How_do_I_setup_my_system_to_run_a_prerelease.3F +sudo -E apt-get update && sudo -E apt-get install python3 python3-pip python-ros-buildfarm +sudo python3 -m pip install -U EmPy + +travis_time_end # setup_docker + +travis_time_start setup_prerelease_scripts + +mkdir -p /tmp/prerelease_job; cd /tmp/prerelease_job; generate_prerelease_script.py https://raw.githubusercontent.com/ros-infrastructure/ros_buildfarm_config/production/index.yaml $ROS_DISTRO default ubuntu trusty amd64 ${PRERELEASE_REPONAME} --level $PRERELEASE_DOWNSTREAM_DEPTH --output-dir ./ + +travis_time_end # setup_prerelease_scripts + +travis_time_start run_prerelease + +./prerelease.sh; + +travis_time_end # run_prerelease + +cd $TRAVIS_BUILD_DIR # cd back to the repository's home directory with travis +pwd + diff --git a/travis.sh b/travis.sh index 5fb45964e..2655f7061 100755 --- a/travis.sh +++ b/travis.sh @@ -64,6 +64,12 @@ function error { exit 1 } +# Start prerelease, and once it finishs then finish this script too. +if [ "$PRERELEASE" == true ]; then + ./ros_pre-release.sh; + exit; +fi + BUILDER=catkin ROSWS=wstool CI_PARENT_DIR=.ci_config # This is the folder name that is used in downstream repositories in order to point to this repo. From 2b99f1c8559166706e7957c089a55cee1ddabbf4 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Thu, 28 Apr 2016 14:57:51 +0900 Subject: [PATCH 02/27] Update README to elaborate the usage and tips for Prerelease Test feature. --- README.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/README.rst b/README.rst index 787817ece..ec253c5eb 100644 --- a/README.rst +++ b/README.rst @@ -196,6 +196,52 @@ Maintainers of client repos are responsible for applying the changes that happen 2. Don't forget to commit the changes the command above makes. +(Optional) Run ROS Prerelease Test +------------------------------------------------------------------------------------- + +Running `docker-based ROS Prerelease Test `_ is strongly recommended when you make a release. To do so, add a single line to your Travis config (eg. `.travis.yml`): + +:: + + ROS_DISTRO=indigo PRERELEASE=true + +Or with more configuration: + +:: + + ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 + +The following is some tips to be shared for running Prerelease Test on Travis CI using `industrial_ci`. + +(Workaround) Prerelease Test check on Travis always passes +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +As of April 2016, because this check on Travis CI always passes regardless the result of Prerelease Test (due to the limitaiton discussed in `a pull request `_), you should not use this Prerelease Test-based check as a criteria for Travis CI check. + +Recommended way is to put the line in `allow_failures` matrix. E.g.: + +:: + + : + matrix: + allow_failures: + - env: ROS_DISTRO=indigo PRERELEASE=true + : + +(Workaround) Don't want to always run Prerelease Test ++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Running Prerelease Test may usually take longer than the tests commonly defined, which can result in longer time for the Travis check to finish. This is usually okay, as developers who are concerned with PRs might not wait for the Travis result that eagerly (besides that, Travis CI limits the maximum run time as 50 minutes so there can't be very long run). If you're concerned, however, then you may want to separately run the Prerelease Test. An example way to do this is to create a branch specifically for Prerelease Test where `.travis.yml` only defines a check entry with `PRERELEASE` turned on. E.g.: + +:: + + : + env: + matrix: + - ROS_DISTRO=indigo PRERELEASE=true + : + +Then open a pull request using this branch against the branch that the change is subject to be merged. You do not want to actually merge this branch no matter what the Travis result is. This branch is solely for Prerelease Test purpose. For maintainers of industrial_ci repository ================================================ From 1c36f7256b2cf091f42cade545a6ae394c7c24ec Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Thu, 28 Apr 2016 21:16:25 +0900 Subject: [PATCH 03/27] Fix wrong prerelease script invocation for client repositories. --- travis.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis.sh b/travis.sh index 2655f7061..117c534fb 100755 --- a/travis.sh +++ b/travis.sh @@ -65,8 +65,8 @@ function error { } # Start prerelease, and once it finishs then finish this script too. -if [ "$PRERELEASE" == true ]; then - ./ros_pre-release.sh; +if [ "$PRERELEASE" == true && -e ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh ]; then + ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh exit; fi From 5d030ce3f731fef17bb50cb4bab1bf67cd08fb04 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Thu, 28 Apr 2016 22:20:23 +0900 Subject: [PATCH 04/27] [README] Some clarifications. --- README.rst | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index ec253c5eb..522094add 100644 --- a/README.rst +++ b/README.rst @@ -101,8 +101,8 @@ You can configure the behavior in `.travis.yml` in your client repository. Required environment variables: -* `ROS_REPOSITORY_PATH`: Location of ROS' binary repositories where depended packages get installed from (typically both standard repo and `"Shadow-Fixed" repository `_) -* `ROS_DISTRO`: Version of ROS (Indigo, Jade etc.). +* `ROS_REPOSITORY_PATH`: Location of ROS' binary repositories where depended packages get installed from (typically both standard repo (`http://packages.ros.org/ros/ubuntu`) and `"Shadow-Fixed" repository `_ (`http://packages.ros.org/ros-shadow-fixed/ubuntu`)) +* `ROS_DISTRO`: Version of ROS in all lower case. E.g.: `indigo` / `jade` Optional environment variables ++++++++++++++++++++++++++++++++ @@ -118,8 +118,8 @@ Note that some of these currently tied only to a single option, but we still lea * `CI_PARENT_DIR` (default: .ci_config): (NOT recommended to specify) This is the folder name that is used in downstream repositories in order to point to this repo. * `NOT_TEST_BUILD` (default: not set): If true, tests in build space won't be run. * `NOT_TEST_INSTALL` (default: not set): If true, tests in `install` space won't be run. -* `PRERELEASE` (default: false): If `true`, run `prerelease test on docker that emulates ROS buildfarm `_. Since prerelease tests are usually necessary only when you are about to make a new version release, you're advised to put the tests that run prerelease into "allow_failures" section (or if your package is not planned to be released at all, no need to use it). See the usage sample in `.travis in indusrial_ci repository`_. -* `PRERELEASE_DOWNSTREAM_DEPTH` (0 to 4, default: 1): Number of the levels of the packages that depend on the prerelease test target package. Upper bound is what is defined by ROS buildfarm prerelease test website (``_ etc.) +* `PRERELEASE` (default: false): If `true`, run `prerelease test on docker that emulates ROS buildfarm `_. The usage of Prerelease Test feature is `explained more in this section `_. +* `PRERELEASE_DOWNSTREAM_DEPTH` (0 to 4, default: 1): Number of the levels of the packages dependecies the Prerelease Test targets at. Range of the level is defined by ROS buildfarm (``_). Note that you might as well use `0` for the packages that are depended on many packages, because the job simply won't finish within Travis CI's time limit (50 minutes). * `PRERELEASE_REPONAME` (default: not set): The target of prerelease test (that you select at ``_, ``_ etc.) If not set then it tests the package of the repository's name. You can specify this by your ROS package name format (with underscore e.g. `industrial_core`), not Debian package name format. * `PKGS_DOWNSTREAM` (default: explained): Packages in downstream to be tested. By default, `TARGET_PKGS` is used if set, if not then `BUILD_PKGS` is used. * `ROS_PARALLEL_JOBS` (default: -j8): Maximum number of packages to be built in parallel by the underlining build tool. As of Jan 2016, this is only enabled with `catkin_tools` (with `make` as an underlining builder). @@ -199,7 +199,9 @@ Maintainers of client repos are responsible for applying the changes that happen (Optional) Run ROS Prerelease Test ------------------------------------------------------------------------------------- -Running `docker-based ROS Prerelease Test `_ is strongly recommended when you make a release. To do so, add a single line to your Travis config (eg. `.travis.yml`): +Running `docker-based ROS Prerelease Test `_ is strongly recommended when you make a release. There are, however, some inconvenience (requires host computer setup, runs on your local host, etc. Detail discussed in `a ticket `_). `industrial_ci` provides a way to run it on your `Travis CI` test. + +To do so, add a single line to your Travis config (eg. `.travis.yml`): :: @@ -211,12 +213,14 @@ Or with more configuration: ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 +See the usage sample in `.travis in indusrial_ci repository `_. + The following is some tips to be shared for running Prerelease Test on Travis CI using `industrial_ci`. -(Workaround) Prerelease Test check on Travis always passes +(Workaround) Prerelease Test job on Travis always passes ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -As of April 2016, because this check on Travis CI always passes regardless the result of Prerelease Test (due to the limitaiton discussed in `a pull request `_), you should not use this Prerelease Test-based check as a criteria for Travis CI check. +As of April 2016, because the Prerelease Test job on Travis CI always passes regardless the test result (due to the limitaiton discussed in `a pull request `_), you should not use this Prerelease Test-based job as a criteria for Travis CI check. Recommended way is to put the line in `allow_failures` matrix. E.g.: From 1d14a288e1566579e3d29cfd583a916a291000d2 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Thu, 28 Apr 2016 22:26:43 +0900 Subject: [PATCH 05/27] [README] Clarify relaitonship b/w checks in travis.sh and Prerelease Test. --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 522094add..9582a3b3d 100644 --- a/README.rst +++ b/README.rst @@ -213,6 +213,8 @@ Or with more configuration: ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 +NOTE: A job that runs Prerelease Test does not run the checks that are defined in `travis.sh `_. To run both, use `matrix` in Travis config. + See the usage sample in `.travis in indusrial_ci repository `_. The following is some tips to be shared for running Prerelease Test on Travis CI using `industrial_ci`. From 32f45cea0a399e0acba34bd3741ac55a5f9b7bdb Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Thu, 28 Apr 2016 22:42:51 +0900 Subject: [PATCH 06/27] [Prerelease Test] Capture error from Prerelease Test. --- .travis.yml | 1 + travis.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1135f3a50..ce6edcd15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ matrix: allow_failures: - env: ROS_DISTRO=indigo PRERELEASE=true - env: ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 + - env: ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=moveit_msgs PRERELEASE_DOWNSTREAM_DEPTH=1 - env: ROS_DISTRO=jade BUILDER='not-compile' - env: ROS_DISTRO=jade BUILDER='not-compile' TRAVIS_REPO_SLUG=ros-industrial/industrial_ci - env: ROS_DISTRO=jade BUILDER='not-compile' TRAVIS_REPO_SLUG=ros-industrial/industrial_ci TRAVIS_BRANCH=master TRAVIS_COMMIT='' TRAVIS_PULL_REQUEST='' diff --git a/travis.sh b/travis.sh index 117c534fb..66285c15c 100755 --- a/travis.sh +++ b/travis.sh @@ -67,7 +67,7 @@ function error { # Start prerelease, and once it finishs then finish this script too. if [ "$PRERELEASE" == true && -e ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh ]; then ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh - exit; + catkin_test_results build || error fi BUILDER=catkin From 552fb8891bbdcd03eb4b0ce760de05c2f84864cd Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Thu, 28 Apr 2016 22:51:48 +0900 Subject: [PATCH 07/27] Common functions to a separate script file. --- ros_pre-release.sh | 28 +--------------------- travis.sh | 27 +-------------------- util.sh | 60 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 53 deletions(-) create mode 100644 util.sh diff --git a/ros_pre-release.sh b/ros_pre-release.sh index c20eb7a3d..a148a1563 100755 --- a/ros_pre-release.sh +++ b/ros_pre-release.sh @@ -33,32 +33,7 @@ set -x -function travis_time_start { - set +x - TRAVIS_START_TIME=$(date +%s%N) - TRAVIS_TIME_ID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1) - TRAVIS_FOLD_NAME=$1 - echo -e "\e[0Ktravis_fold:start:$TRAVIS_FOLD_NAME" - echo -e "\e[0Ktravis_time:start:$TRAVIS_TIME_ID\e[34m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\e[0m" - set -x -} - -function travis_time_end { - set +x - _COLOR=${1:-32} - TRAVIS_END_TIME=$(date +%s%N) - TIME_ELAPSED_SECONDS=$(( ($TRAVIS_END_TIME - $TRAVIS_START_TIME)/1000000000 )) - echo -e "travis_time:end:$TRAVIS_TIME_ID:start=$TRAVIS_START_TIME,finish=$TRAVIS_END_TIME,duration=$(($TRAVIS_END_TIME - $TRAVIS_START_TIME))\e[0K" - echo -e "travis_fold:end:$TRAVIS_FOLD_NAME\e[${_COLOR}m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\e[0m" - echo -e "\e[0K\e[${_COLOR}mFunction $TRAVIS_FOLD_NAME took $(( $TIME_ELAPSED_SECONDS / 60 )) min $(( $TIME_ELAPSED_SECONDS % 60 )) sec\e[0m" - set -x -} - -function error { - travis_time_end 31 - trap - ERR - exit 1 -} +source ./util.sh if [ ! "$PRERELEASE_DOWNSTREAM_DEPTH" ]; then export PRERELEASE_DOWNSTREAM_DEPTH="1"; fi if [ ! "$PRERELEASE_REPONAME" ]; then NAME_REPO=$(echo $TRAVIS_REPO_SLUG | cut -d'/' -f 1); fi @@ -90,4 +65,3 @@ travis_time_end # run_prerelease cd $TRAVIS_BUILD_DIR # cd back to the repository's home directory with travis pwd - diff --git a/travis.sh b/travis.sh index 66285c15c..9c93f25f5 100755 --- a/travis.sh +++ b/travis.sh @@ -37,32 +37,7 @@ set -x -function travis_time_start { - set +x - TRAVIS_START_TIME=$(date +%s%N) - TRAVIS_TIME_ID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1) - TRAVIS_FOLD_NAME=$1 - echo -e "\e[0Ktravis_fold:start:$TRAVIS_FOLD_NAME" - echo -e "\e[0Ktravis_time:start:$TRAVIS_TIME_ID\e[34m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\e[0m" - set -x -} - -function travis_time_end { - set +x - _COLOR=${1:-32} - TRAVIS_END_TIME=$(date +%s%N) - TIME_ELAPSED_SECONDS=$(( ($TRAVIS_END_TIME - $TRAVIS_START_TIME)/1000000000 )) - echo -e "travis_time:end:$TRAVIS_TIME_ID:start=$TRAVIS_START_TIME,finish=$TRAVIS_END_TIME,duration=$(($TRAVIS_END_TIME - $TRAVIS_START_TIME))\e[0K" - echo -e "travis_fold:end:$TRAVIS_FOLD_NAME\e[${_COLOR}m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\e[0m" - echo -e "\e[0K\e[${_COLOR}mFunction $TRAVIS_FOLD_NAME took $(( $TIME_ELAPSED_SECONDS / 60 )) min $(( $TIME_ELAPSED_SECONDS % 60 )) sec\e[0m" - set -x -} - -function error { - travis_time_end 31 - trap - ERR - exit 1 -} +source ./util.sh # Start prerelease, and once it finishs then finish this script too. if [ "$PRERELEASE" == true && -e ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh ]; then diff --git a/util.sh b/util.sh new file mode 100644 index 000000000..659704129 --- /dev/null +++ b/util.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Software License Agreement (BSD License) +# +# Copyright (c) 2016, Isaac I. Y. Saito +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the Isaac I. Y. Saito, nor the names +# of its contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +## util.sh +## This is a script where the functions commonly used within the industrial_ci repo are defined. + +function travis_time_start { + set +x + TRAVIS_START_TIME=$(date +%s%N) + TRAVIS_TIME_ID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1) + TRAVIS_FOLD_NAME=$1 + echo -e "\e[0Ktravis_fold:start:$TRAVIS_FOLD_NAME" + echo -e "\e[0Ktravis_time:start:$TRAVIS_TIME_ID\e[34m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\e[0m" + set -x +} + +function travis_time_end { + set +x + _COLOR=${1:-32} + TRAVIS_END_TIME=$(date +%s%N) + TIME_ELAPSED_SECONDS=$(( ($TRAVIS_END_TIME - $TRAVIS_START_TIME)/1000000000 )) + echo -e "travis_time:end:$TRAVIS_TIME_ID:start=$TRAVIS_START_TIME,finish=$TRAVIS_END_TIME,duration=$(($TRAVIS_END_TIME - $TRAVIS_START_TIME))\e[0K" + echo -e "travis_fold:end:$TRAVIS_FOLD_NAME\e[${_COLOR}m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\e[0m" + echo -e "\e[0K\e[${_COLOR}mFunction $TRAVIS_FOLD_NAME took $(( $TIME_ELAPSED_SECONDS / 60 )) min $(( $TIME_ELAPSED_SECONDS % 60 )) sec\e[0m" + set -x +} + +function error { + travis_time_end 31 + trap - ERR + exit 1 +} From da2f4922a079bb8f06cbbe2cc18748c0d72dc90e Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Thu, 28 Apr 2016 22:54:25 +0900 Subject: [PATCH 08/27] [Prerelease Test] jade jobs matrix has not been set properly. --- .travis.yml | 6 ++---- travis.sh | 9 +++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index ce6edcd15..0c6144ce4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,14 +20,12 @@ env: - ROS_DISTRO=indigo CATKIN_PARALLEL_JOBS='-p1' ROS_PARALLEL_JOBS='-j1' # Intend build on low-power platform - ROS_DISTRO=indigo PRERELEASE=true - ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 + - ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=warehouse_ros PRERELEASE_DOWNSTREAM_DEPTH=1 matrix: allow_failures: - env: ROS_DISTRO=indigo PRERELEASE=true - env: ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 - - env: ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=moveit_msgs PRERELEASE_DOWNSTREAM_DEPTH=1 - - env: ROS_DISTRO=jade BUILDER='not-compile' - - env: ROS_DISTRO=jade BUILDER='not-compile' TRAVIS_REPO_SLUG=ros-industrial/industrial_ci - - env: ROS_DISTRO=jade BUILDER='not-compile' TRAVIS_REPO_SLUG=ros-industrial/industrial_ci TRAVIS_BRANCH=master TRAVIS_COMMIT='' TRAVIS_PULL_REQUEST='' + - env: ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=warehouse_ros PRERELEASE_DOWNSTREAM_DEPTH=1 before_script: - CI_DIR=.ci_config - mkdir $CI_DIR; cp * $CI_DIR # Need to copy, since directory starting from dot is ignoreed by catkin build. Also this folder name is used for testing both this repo itself and the "client" repositories diff --git a/travis.sh b/travis.sh index 9c93f25f5..9351275a1 100755 --- a/travis.sh +++ b/travis.sh @@ -39,15 +39,17 @@ set -x source ./util.sh +export CI_SOURCE_PATH=$(pwd) +CI_PARENT_DIR=.ci_config # This is the folder name that is used in downstream repositories in order to point to this repo. + # Start prerelease, and once it finishs then finish this script too. -if [ "$PRERELEASE" == true && -e ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh ]; then +if [ "$PRERELEASE" == true ] && [ -e ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh ]; then ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh - catkin_test_results build || error + catkin_test_results build && (echo 'ROS Prerelease Test went successful.'; exit 0) || error fi BUILDER=catkin ROSWS=wstool -CI_PARENT_DIR=.ci_config # This is the folder name that is used in downstream repositories in order to point to this repo. trap error ERR @@ -57,7 +59,6 @@ if [ "`git diff origin/master FETCH_HEAD $CI_PARENT_DIR`" != "" ] ; then DIFF=`g travis_time_start setup_ros # Define some config vars -export CI_SOURCE_PATH=$(pwd) export DOWNSTREAM_REPO_NAME=${PWD##*/} if [ ! "$CATKIN_PARALLEL_JOBS" ]; then export CATKIN_PARALLEL_JOBS="-p4"; fi if [ ! "$CATKIN_PARALLEL_TEST_JOBS" ]; then export CATKIN_PARALLEL_TEST_JOBS="$CATKIN_PARALLEL_JOBS"; fi From a1ab678b9d6eec4db409f273bfcfed71e98d3965 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Fri, 29 Apr 2016 01:46:54 +0900 Subject: [PATCH 09/27] [Prerelease Test] Fix wrong variable name. --- ros_pre-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_pre-release.sh b/ros_pre-release.sh index a148a1563..5e622e010 100755 --- a/ros_pre-release.sh +++ b/ros_pre-release.sh @@ -36,7 +36,7 @@ set -x source ./util.sh if [ ! "$PRERELEASE_DOWNSTREAM_DEPTH" ]; then export PRERELEASE_DOWNSTREAM_DEPTH="1"; fi -if [ ! "$PRERELEASE_REPONAME" ]; then NAME_REPO=$(echo $TRAVIS_REPO_SLUG | cut -d'/' -f 1); fi +if [ ! "$PRERELEASE_REPONAME" ]; then PRERELEASE_REPONAME=$(echo $TRAVIS_REPO_SLUG | cut -d'/' -f 1); fi echo "PRERELEASE_REPONAME = ${PRERELEASE_REPONAME}" travis_time_start setup_docker From cd7b6f8c6f81e66dc96d19538f869905ce05f65f Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Fri, 29 Apr 2016 02:04:31 +0900 Subject: [PATCH 10/27] [Prerelease Test] Fix cutting slug directory. --- ros_pre-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_pre-release.sh b/ros_pre-release.sh index 5e622e010..0935530b2 100755 --- a/ros_pre-release.sh +++ b/ros_pre-release.sh @@ -36,7 +36,7 @@ set -x source ./util.sh if [ ! "$PRERELEASE_DOWNSTREAM_DEPTH" ]; then export PRERELEASE_DOWNSTREAM_DEPTH="1"; fi -if [ ! "$PRERELEASE_REPONAME" ]; then PRERELEASE_REPONAME=$(echo $TRAVIS_REPO_SLUG | cut -d'/' -f 1); fi +if [ ! "$PRERELEASE_REPONAME" ]; then PRERELEASE_REPONAME=$(echo $TRAVIS_REPO_SLUG | cut -d'/' -f 2); fi echo "PRERELEASE_REPONAME = ${PRERELEASE_REPONAME}" travis_time_start setup_docker From 4619388f4b4a2da666543d16d80245dbd9f946f7 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Fri, 29 Apr 2016 02:37:30 +0900 Subject: [PATCH 11/27] [Prerelease Test] Add exception handling to util functions. --- travis.sh | 2 ++ util.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/travis.sh b/travis.sh index 9351275a1..25300a658 100755 --- a/travis.sh +++ b/travis.sh @@ -42,11 +42,13 @@ source ./util.sh export CI_SOURCE_PATH=$(pwd) CI_PARENT_DIR=.ci_config # This is the folder name that is used in downstream repositories in order to point to this repo. +travis_time_start prerelease_from_travis_sh # Start prerelease, and once it finishs then finish this script too. if [ "$PRERELEASE" == true ] && [ -e ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh ]; then ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh catkin_test_results build && (echo 'ROS Prerelease Test went successful.'; exit 0) || error fi +travis_time_end # prerelease_from_travis_sh BUILDER=catkin ROSWS=wstool diff --git a/util.sh b/util.sh index 659704129..25a38ee62 100644 --- a/util.sh +++ b/util.sh @@ -44,6 +44,7 @@ function travis_time_start { function travis_time_end { set +x + if [ -z $TRAVIS_START_TIME ]; then echo '[travis_time_end] var TRAVIS_START_TIME is not set. You need to call `travis_time_start` in advance. Rerutning.'; return; fi _COLOR=${1:-32} TRAVIS_END_TIME=$(date +%s%N) TIME_ELAPSED_SECONDS=$(( ($TRAVIS_END_TIME - $TRAVIS_START_TIME)/1000000000 )) From 8b4758485360ac05cc84783704bebbc0ceed0a5c Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Fri, 29 Apr 2016 11:35:57 +0900 Subject: [PATCH 12/27] [Prerelease Test] Wait until after rosdep ran. --- travis.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/travis.sh b/travis.sh index 25300a658..bde3658d1 100755 --- a/travis.sh +++ b/travis.sh @@ -41,15 +41,6 @@ source ./util.sh export CI_SOURCE_PATH=$(pwd) CI_PARENT_DIR=.ci_config # This is the folder name that is used in downstream repositories in order to point to this repo. - -travis_time_start prerelease_from_travis_sh -# Start prerelease, and once it finishs then finish this script too. -if [ "$PRERELEASE" == true ] && [ -e ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh ]; then - ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh - catkin_test_results build && (echo 'ROS Prerelease Test went successful.'; exit 0) || error -fi -travis_time_end # prerelease_from_travis_sh - BUILDER=catkin ROSWS=wstool @@ -168,8 +159,16 @@ fi travis_time_end # rosdep_install -travis_time_start wstool_info +# Start prerelease, and once it finishs then finish this script too. +# This block needs to be here because catkin_test_results isn't available until up to this point. +travis_time_start prerelease_from_travis_sh +if [ "$PRERELEASE" == true ] && [ -e ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh ]; then + ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh + catkin_test_results build && (echo 'ROS Prerelease Test went successful.'; exit 0) || error +fi +travis_time_end # prerelease_from_travis_sh +travis_time_start wstool_info $ROSWS --version $ROSWS info -t . cd ../ From cd99d3306e4bf69ea928c4e839e2c0a1496ca93d Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Fri, 29 Apr 2016 14:37:43 +0900 Subject: [PATCH 13/27] [Prerelease Test] Move catkin_test_results to prerelease script. --- ros_pre-release.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ros_pre-release.sh b/ros_pre-release.sh index 0935530b2..c0a3b8dc1 100755 --- a/ros_pre-release.sh +++ b/ros_pre-release.sh @@ -61,6 +61,8 @@ travis_time_start run_prerelease ./prerelease.sh; +catkin_test_results build && (echo 'ROS Prerelease Test went successful.'; exit 0) || error + travis_time_end # run_prerelease cd $TRAVIS_BUILD_DIR # cd back to the repository's home directory with travis From 624ccda1e4a7bbbac5293a711be6e316c8ac41f0 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Sun, 1 May 2016 07:49:51 +0900 Subject: [PATCH 14/27] [Prerelease Test] downstream depth to 0 --- README.rst | 2 +- ros_pre-release.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 9582a3b3d..15b76b204 100644 --- a/README.rst +++ b/README.rst @@ -119,7 +119,7 @@ Note that some of these currently tied only to a single option, but we still lea * `NOT_TEST_BUILD` (default: not set): If true, tests in build space won't be run. * `NOT_TEST_INSTALL` (default: not set): If true, tests in `install` space won't be run. * `PRERELEASE` (default: false): If `true`, run `prerelease test on docker that emulates ROS buildfarm `_. The usage of Prerelease Test feature is `explained more in this section `_. -* `PRERELEASE_DOWNSTREAM_DEPTH` (0 to 4, default: 1): Number of the levels of the packages dependecies the Prerelease Test targets at. Range of the level is defined by ROS buildfarm (``_). Note that you might as well use `0` for the packages that are depended on many packages, because the job simply won't finish within Travis CI's time limit (50 minutes). +* `PRERELEASE_DOWNSTREAM_DEPTH` (0 to 4, default: 0): Number of the levels of the packages dependecies the Prerelease Test targets at. Range of the level is defined by ROS buildfarm (``_). Note that you might as well use `0` for the packages that are depended on many packages, because the job simply won't finish within Travis CI's time limit (50 minutes). * `PRERELEASE_REPONAME` (default: not set): The target of prerelease test (that you select at ``_, ``_ etc.) If not set then it tests the package of the repository's name. You can specify this by your ROS package name format (with underscore e.g. `industrial_core`), not Debian package name format. * `PKGS_DOWNSTREAM` (default: explained): Packages in downstream to be tested. By default, `TARGET_PKGS` is used if set, if not then `BUILD_PKGS` is used. * `ROS_PARALLEL_JOBS` (default: -j8): Maximum number of packages to be built in parallel by the underlining build tool. As of Jan 2016, this is only enabled with `catkin_tools` (with `make` as an underlining builder). diff --git a/ros_pre-release.sh b/ros_pre-release.sh index c0a3b8dc1..afbfd2f1d 100755 --- a/ros_pre-release.sh +++ b/ros_pre-release.sh @@ -35,7 +35,7 @@ set -x source ./util.sh -if [ ! "$PRERELEASE_DOWNSTREAM_DEPTH" ]; then export PRERELEASE_DOWNSTREAM_DEPTH="1"; fi +if [ ! "$PRERELEASE_DOWNSTREAM_DEPTH" ]; then export PRERELEASE_DOWNSTREAM_DEPTH="0"; fi if [ ! "$PRERELEASE_REPONAME" ]; then PRERELEASE_REPONAME=$(echo $TRAVIS_REPO_SLUG | cut -d'/' -f 2); fi echo "PRERELEASE_REPONAME = ${PRERELEASE_REPONAME}" From b817e02a4e9f2007063e537e0b8185067f492c3d Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Sun, 1 May 2016 08:06:46 +0900 Subject: [PATCH 15/27] [Prerelease Test] Install catkin for showing test result. --- ros_pre-release.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ros_pre-release.sh b/ros_pre-release.sh index afbfd2f1d..bb03337e3 100755 --- a/ros_pre-release.sh +++ b/ros_pre-release.sh @@ -39,6 +39,14 @@ if [ ! "$PRERELEASE_DOWNSTREAM_DEPTH" ]; then export PRERELEASE_DOWNSTREAM_DEPTH if [ ! "$PRERELEASE_REPONAME" ]; then PRERELEASE_REPONAME=$(echo $TRAVIS_REPO_SLUG | cut -d'/' -f 2); fi echo "PRERELEASE_REPONAME = ${PRERELEASE_REPONAME}" +travis_time_start install_for_display_testresult +# Set apt repo +sudo -E sh -c 'echo "deb $ROS_REPOSITORY_PATH `lsb_release -cs` main" > /etc/apt/sources.list.d/ros-latest.list' +# Common ROS install preparation +wget http://packages.ros.org/ros.key -O - | sudo apt-key add - +sudo apt-get -qq install -y ros-$ROS_DISTRO-catkin && source /opt/ros/$ROS_DISTRO/setup.bash || (echo 'ros-latest.list content: \n'; cat /etc/apt/sources.list.d/ros-latest.list; error;) +travis_time_end # install_for_display_testresult + travis_time_start setup_docker sudo usermod -aG docker ubuntu @@ -58,12 +66,13 @@ mkdir -p /tmp/prerelease_job; cd /tmp/prerelease_job; generate_prerelease_script travis_time_end # setup_prerelease_scripts travis_time_start run_prerelease - ./prerelease.sh; +travis_time_end # run_prerelease +travis_time_start show_testresult catkin_test_results build && (echo 'ROS Prerelease Test went successful.'; exit 0) || error +travis_time_end # show_testresult -travis_time_end # run_prerelease cd $TRAVIS_BUILD_DIR # cd back to the repository's home directory with travis pwd From 67019c814bdb6bbbb94a5bf467d5e3ce5db2edfd Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Sun, 1 May 2016 08:41:45 +0900 Subject: [PATCH 16/27] [Prerelease Test] Disable meaningless Travis job. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0c6144ce4..7c23ed093 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,10 +20,10 @@ env: - ROS_DISTRO=indigo CATKIN_PARALLEL_JOBS='-p1' ROS_PARALLEL_JOBS='-j1' # Intend build on low-power platform - ROS_DISTRO=indigo PRERELEASE=true - ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 - - ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=warehouse_ros PRERELEASE_DOWNSTREAM_DEPTH=1 + - ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=warehouse_ros PRERELEASE_DOWNSTREAM_DEPTH=1 # Intended to fail as of Apr 2016 matrix: allow_failures: - - env: ROS_DISTRO=indigo PRERELEASE=true + # - env: ROS_DISTRO=indigo PRERELEASE=true ## Comment out because this is meaningless for always failing without prerelease testable contents in industrial_ci. - env: ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 - env: ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=warehouse_ros PRERELEASE_DOWNSTREAM_DEPTH=1 before_script: From 2b87f8721dafb447d45a8150d83f2a8174765bd7 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Sun, 1 May 2016 08:45:01 +0900 Subject: [PATCH 17/27] Remove meaningless Travis jobs --- .travis.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7c23ed093..a8dff0ba0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,20 +12,16 @@ compiler: # Whoever receives success and/or failure via email env: matrix: - - ROS_DISTRO=indigo BUILDER='not-compile' - - ROS_DISTRO=indigo BUILDER='not-compile' TRAVIS_REPO_SLUG=ros-industrial/industrial_ci - - ROS_DISTRO=indigo BUILDER='not-compile' TRAVIS_REPO_SLUG=ros-industrial/industrial_ci TRAVIS_BRANCH=master TRAVIS_COMMIT='' TRAVIS_PULL_REQUEST='' - ROS_DISTRO=indigo NOT_TEST_BUILD='true' - ROS_DISTRO=indigo NOT_TEST_INSTALL='true' - ROS_DISTRO=indigo CATKIN_PARALLEL_JOBS='-p1' ROS_PARALLEL_JOBS='-j1' # Intend build on low-power platform - - ROS_DISTRO=indigo PRERELEASE=true + # - env: ROS_DISTRO=indigo PRERELEASE=true ## Comment out because this is meaningless for always failing without prerelease testable contents in industrial_ci. - ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 - - ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=warehouse_ros PRERELEASE_DOWNSTREAM_DEPTH=1 # Intended to fail as of Apr 2016 + - ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=warehouse_ros PRERELEASE_DOWNSTREAM_DEPTH=1 matrix: allow_failures: - # - env: ROS_DISTRO=indigo PRERELEASE=true ## Comment out because this is meaningless for always failing without prerelease testable contents in industrial_ci. - env: ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 - - env: ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=warehouse_ros PRERELEASE_DOWNSTREAM_DEPTH=1 + - env: ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=warehouse_ros PRERELEASE_DOWNSTREAM_DEPTH=1 # Intended to fail as of Apr 2016 before_script: - CI_DIR=.ci_config - mkdir $CI_DIR; cp * $CI_DIR # Need to copy, since directory starting from dot is ignoreed by catkin build. Also this folder name is used for testing both this repo itself and the "client" repositories From aa427a09ec930a2d9eb18f70c52dc8ecef297538 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Sun, 1 May 2016 08:55:40 +0900 Subject: [PATCH 18/27] Move Prerelease Test to after setting up ROS. --- travis.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/travis.sh b/travis.sh index bde3658d1..e78624968 100755 --- a/travis.sh +++ b/travis.sh @@ -77,6 +77,15 @@ if [ $HAVE_MONGO_DB == 0 ]; then fi travis_time_end # setup_ros + +# Start prerelease, and once it finishs then finish this script too. +# This block needs to be here because catkin_test_results isn't available until up to this point. +travis_time_start prerelease_from_travis_sh +if [ "$PRERELEASE" == true ] && [ -e ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh ]; then + ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh +fi +travis_time_end # prerelease_from_travis_sh + travis_time_start setup_rosdep # Setup rosdep From dcff27cb5cbb6bb9d4777d3e0b165eda514b9d5d Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Sun, 1 May 2016 08:58:36 +0900 Subject: [PATCH 19/27] Calling scripts via relative path not seem to work. --- ros_pre-release.sh | 2 +- travis.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ros_pre-release.sh b/ros_pre-release.sh index bb03337e3..e401d780f 100755 --- a/ros_pre-release.sh +++ b/ros_pre-release.sh @@ -33,7 +33,7 @@ set -x -source ./util.sh +source ${CI_SOURCE_PATH}/$CI_PARENT_DIR/util.sh if [ ! "$PRERELEASE_DOWNSTREAM_DEPTH" ]; then export PRERELEASE_DOWNSTREAM_DEPTH="0"; fi if [ ! "$PRERELEASE_REPONAME" ]; then PRERELEASE_REPONAME=$(echo $TRAVIS_REPO_SLUG | cut -d'/' -f 2); fi diff --git a/travis.sh b/travis.sh index e78624968..e0d489eef 100755 --- a/travis.sh +++ b/travis.sh @@ -37,13 +37,13 @@ set -x -source ./util.sh - export CI_SOURCE_PATH=$(pwd) CI_PARENT_DIR=.ci_config # This is the folder name that is used in downstream repositories in order to point to this repo. BUILDER=catkin ROSWS=wstool +source ${CI_SOURCE_PATH}/$CI_PARENT_DIR/util.sh + trap error ERR git branch --all From 94899b10a81f1bc9e72021655a9cb5942d475b70 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Sun, 1 May 2016 19:10:18 +0900 Subject: [PATCH 20/27] 'set -e' to exit upon errors. --- ros_pre-release.sh | 1 + travis.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/ros_pre-release.sh b/ros_pre-release.sh index e401d780f..61d5c76bf 100755 --- a/ros_pre-release.sh +++ b/ros_pre-release.sh @@ -31,6 +31,7 @@ # ## Author: Isaac I. Y. Saito +set -e set -x source ${CI_SOURCE_PATH}/$CI_PARENT_DIR/util.sh diff --git a/travis.sh b/travis.sh index e0d489eef..5bf0e2a36 100755 --- a/travis.sh +++ b/travis.sh @@ -35,6 +35,7 @@ ## This is a "common" script that can be run on travis CI at a downstream github repository. ## See ./README.rst for the detailed usage. +set -e set -x export CI_SOURCE_PATH=$(pwd) From a931f05abe9e471f424e6d1f93417e14fe2c561a Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Tue, 3 May 2016 08:28:33 +0900 Subject: [PATCH 21/27] Export a variable to subprocesses. --- travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis.sh b/travis.sh index 5bf0e2a36..3d1036c2b 100755 --- a/travis.sh +++ b/travis.sh @@ -39,7 +39,7 @@ set -e set -x export CI_SOURCE_PATH=$(pwd) -CI_PARENT_DIR=.ci_config # This is the folder name that is used in downstream repositories in order to point to this repo. +export CI_PARENT_DIR=.ci_config # This is the folder name that is used in downstream repositories in order to point to this repo. BUILDER=catkin ROSWS=wstool From 3b27f45c47a8f1450a7f6b835ec585f233c7500d Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Tue, 3 May 2016 16:05:26 +0900 Subject: [PATCH 22/27] [Prerelease Test] Use catkin_test_results build_isolated, in accordance with buildfarm. --- ros_pre-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_pre-release.sh b/ros_pre-release.sh index 61d5c76bf..ef1c8ad2f 100755 --- a/ros_pre-release.sh +++ b/ros_pre-release.sh @@ -71,7 +71,7 @@ travis_time_start run_prerelease travis_time_end # run_prerelease travis_time_start show_testresult -catkin_test_results build && (echo 'ROS Prerelease Test went successful.'; exit 0) || error +catkin_test_results build_isolated && (echo 'ROS Prerelease Test went successful.'; exit 0) || error travis_time_end # show_testresult From 5d07bf03a46d0f62b22ce3424c607ecc82e63e24 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Thu, 5 May 2016 03:10:08 +0900 Subject: [PATCH 23/27] [Prerelease Test] Use catkin_test_results w/o specifying test file path to workaround https://github.com/ros-infrastructure/ros_buildfarm/issues/283. Also use verbose option for better debug. --- ros_pre-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_pre-release.sh b/ros_pre-release.sh index ef1c8ad2f..878cdd96f 100755 --- a/ros_pre-release.sh +++ b/ros_pre-release.sh @@ -71,7 +71,7 @@ travis_time_start run_prerelease travis_time_end # run_prerelease travis_time_start show_testresult -catkin_test_results build_isolated && (echo 'ROS Prerelease Test went successful.'; exit 0) || error +catkin_test_results --verbose && (echo 'ROS Prerelease Test went successful.'; exit 0) || error travis_time_end # show_testresult From ead8d06c389399f92b6865aa2c981d0c19b7a017 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Thu, 5 May 2016 17:36:09 +0900 Subject: [PATCH 24/27] [Prerelease Test] Capture exit status to end the parent script (travis.sh). --- travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis.sh b/travis.sh index 3d1036c2b..6d04ab8ce 100755 --- a/travis.sh +++ b/travis.sh @@ -83,7 +83,7 @@ travis_time_end # setup_ros # This block needs to be here because catkin_test_results isn't available until up to this point. travis_time_start prerelease_from_travis_sh if [ "$PRERELEASE" == true ] && [ -e ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh ]; then - ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh + ${CI_SOURCE_PATH}/$CI_PARENT_DIR/ros_pre-release.sh && exit 0 || exit 1; fi travis_time_end # prerelease_from_travis_sh From 44b0926f64afcf2d36972f82459f728d2aebc109 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Fri, 6 May 2016 01:30:54 +0900 Subject: [PATCH 25/27] Clarify inspiring package. --- travis.sh | 2 +- util.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/travis.sh b/travis.sh index 6d04ab8ce..f868af9c3 100755 --- a/travis.sh +++ b/travis.sh @@ -29,7 +29,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # -## Greatly inspired by JSK travis with additional comments for maintainability: https://github.com/jsk-ros-pkg/jsk_travis +## Greatly inspired by JSK travis https://github.com/jsk-ros-pkg/jsk_travis ## Author: Isaac I. Y. Saito ## This is a "common" script that can be run on travis CI at a downstream github repository. diff --git a/util.sh b/util.sh index 25a38ee62..ee5f44d3d 100644 --- a/util.sh +++ b/util.sh @@ -1,5 +1,7 @@ #!/bin/bash +# Originally developed in JSK travis package https://github.com/jsk-ros-pkg/jsk_travis + # Software License Agreement (BSD License) # # Copyright (c) 2016, Isaac I. Y. Saito From e66d9356df88e1824f9fb576f7a2336235e5de60 Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Fri, 6 May 2016 06:06:48 +0900 Subject: [PATCH 26/27] Clarify purpose of failure --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a8dff0ba0..70d4f3da7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ env: matrix: allow_failures: - env: ROS_DISTRO=indigo PRERELEASE=true PRERELEASE_REPONAME=industrial_core PRERELEASE_DOWNSTREAM_DEPTH=0 - - env: ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=warehouse_ros PRERELEASE_DOWNSTREAM_DEPTH=1 # Intended to fail as of Apr 2016 + - env: ROS_DISTRO=jade PRERELEASE=true PRERELEASE_REPONAME=warehouse_ros PRERELEASE_DOWNSTREAM_DEPTH=1 # Intended to fail (as of Apr 2016), to test the capability of capturing Prerelease Test failure. before_script: - CI_DIR=.ci_config - mkdir $CI_DIR; cp * $CI_DIR # Need to copy, since directory starting from dot is ignoreed by catkin build. Also this folder name is used for testing both this repo itself and the "client" repositories From d179d25a465cb63e8fd2b63b58a92d9e64eb04ec Mon Sep 17 00:00:00 2001 From: "Isaac I.Y. Saito" <130s@2000.jukuin.keio.ac.jp> Date: Fri, 6 May 2016 06:07:29 +0900 Subject: [PATCH 27/27] Add an author to a script --- ros_pre-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_pre-release.sh b/ros_pre-release.sh index 878cdd96f..d27acf1f3 100755 --- a/ros_pre-release.sh +++ b/ros_pre-release.sh @@ -29,7 +29,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # -## Author: Isaac I. Y. Saito +## Author: Isaac I. Y. Saito, Mathias L¸«ädtke set -e set -x