Skip to content

Commit

Permalink
add options to toggle to continue rosdep and catkin/colcon on failure
Browse files Browse the repository at this point in the history
  • Loading branch information
@lreiher
lreiher committed Jun 6, 2024
1 parent cab9bd1 commit bc1309f
Show file tree
Hide file tree
Showing 7 changed files with 72 additions and 36 deletions.
68 changes: 35 additions & 33 deletions .gitlab-ci/docker-ros.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,39 +9,41 @@ workflow:


variables:
TARGET: run # Target stage of Dockerfile (comma-separated list) [dev|run]
PLATFORM: amd64 # Target platform architecture (comma-separated list) [amd64|arm64|...]
BASE_IMAGE: '' # Base image name:tag (required)
COMMAND: '' # Launch command of run image (required if target=run)
IMAGE_NAME: ${CI_REGISTRY_IMAGE} # Image name of run image
IMAGE_TAG: latest # Image tag of run image
DEV_IMAGE_NAME: ${IMAGE_NAME} # Image name of dev image
DEV_IMAGE_TAG: ${IMAGE_TAG}-dev # Image tag of dev image
BUILD_CONTEXT: . # Build context of Docker build process
REGISTRY: ${CI_REGISTRY} # Docker registry to push images to
REGISTRY_USER: ${CI_REGISTRY_USER} # Docker registry username
REGISTRY_PASSWORD: ${CI_REGISTRY_PASSWORD} # Docker registry password
ENABLE_INDUSTRIAL_CI: 'false' # Enable industrial_ci
ENABLE_SINGLEARCH_PUSH: 'false' # Enable push of single arch images with [-amd64|-arm64] postfix
ENABLE_PUSH_AS_LATEST: 'false' # Push images with tag `latest`/`latest-dev` in addition to the configured image names
RMW_IMPLEMENTATION: 'rmw_cyclonedds_cpp' # RMW implementation to use (only for ROS 2)
ROS_DISTRO: '' # ROS Distro (required if ROS is not installed in `base-image`)
GIT_HTTPS_SERVER: ${CI_SERVER_HOST} # Server URL (without protocol) for cloning private Git repositories via HTTPS
GIT_HTTPS_USER: gitlab-ci-token # Username for cloning private Git repositories via HTTPS
GIT_HTTPS_PASSWORD: ${CI_JOB_TOKEN} # Password for cloning private Git repositories via HTTPS
GIT_SSH_PRIVATE_KEY: '' # SSH private key for cloning private Git repositories via SSH
GIT_SSH_KNOWN_HOST_KEYS: '' # Known SSH host keys for cloning private Git repositories via SSH (may be obtained using `ssh-keyscan`)
ADDITIONAL_DEBS_FILE: docker/additional-debs.txt # Relative filepath to file containing additional apt deb packages to install
ENABLE_RECURSIVE_ADDITIONAL_DEBS: 'false' # Enable recursive discovery of files named `additional-debs-file`
ADDITIONAL_FILES_DIR: docker/additional-files # Relative path to directory containing additional files to copy into image"
ADDITIONAL_PIP_FILE: docker/additional-pip-requirements.txt # Relative filepath to file containing additional pip packages to install
ENABLE_RECURSIVE_ADDITIONAL_PIP: 'false' # Enable recursive discovery of files named `additional-pip-file`
BLACKLISTED_PACKAGES_FILE: docker/blacklisted-packages.txt # Relative filepath to file containing the blacklisted packages
ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: 'false' # Enable recursive discovery of files named `blacklisted-packages-file`
CUSTOM_SCRIPT_FILE: docker/custom.sh # Relative filepath to script containing custom installation commands
ENABLE_RECURSIVE_CUSTOM_SCRIPT: 'false' # Enable recursive discovery of files named `custom-script-file`
VCS_IMPORT_FILE: .repos # Relative filepath to file containing additional repos to install via vcstools (only relevant if ENABLE_RECURSIVE_VCS_IMPORT=false)
ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `*.repos`
TARGET: run # Target stage of Dockerfile (comma-separated list) [dev|run]
PLATFORM: amd64 # Target platform architecture (comma-separated list) [amd64|arm64|...]
BASE_IMAGE: '' # Base image name:tag (required)
COMMAND: '' # Launch command of run image (required if target=run)
IMAGE_NAME: ${CI_REGISTRY_IMAGE} # Image name of run image
IMAGE_TAG: latest # Image tag of run image
DEV_IMAGE_NAME: ${IMAGE_NAME} # Image name of dev image
DEV_IMAGE_TAG: ${IMAGE_TAG}-dev # Image tag of dev image
BUILD_CONTEXT: . # Build context of Docker build process
REGISTRY: ${CI_REGISTRY} # Docker registry to push images to
REGISTRY_USER: ${CI_REGISTRY_USER} # Docker registry username
REGISTRY_PASSWORD: ${CI_REGISTRY_PASSWORD} # Docker registry password
ENABLE_INDUSTRIAL_CI: 'false' # Enable industrial_ci
ENABLE_SINGLEARCH_PUSH: 'false' # Enable push of single arch images with [-amd64|-arm64] postfix
ENABLE_PUSH_AS_LATEST: 'false' # Push images with tag `latest`/`latest-dev` in addition to the configured image names
RMW_IMPLEMENTATION: 'rmw_cyclonedds_cpp' # RMW implementation to use (only for ROS 2)
ROS_DISTRO: '' # ROS Distro (required if ROS is not installed in `base-image`)
GIT_HTTPS_SERVER: ${CI_SERVER_HOST} # Server URL (without protocol) for cloning private Git repositories via HTTPS
GIT_HTTPS_USER: gitlab-ci-token # Username for cloning private Git repositories via HTTPS
GIT_HTTPS_PASSWORD: ${CI_JOB_TOKEN} # Password for cloning private Git repositories via HTTPS
GIT_SSH_PRIVATE_KEY: '' # SSH private key for cloning private Git repositories via SSH
GIT_SSH_KNOWN_HOST_KEYS: '' # Known SSH host keys for cloning private Git repositories via SSH (may be obtained using `ssh-keyscan`)
ADDITIONAL_DEBS_FILE: docker/additional-debs.txt # Relative filepath to file containing additional apt deb packages to install
ENABLE_RECURSIVE_ADDITIONAL_DEBS: 'false' # Enable recursive discovery of files named `additional-debs-file`
ADDITIONAL_FILES_DIR: docker/additional-files # Relative path to directory containing additional files to copy into image"
ADDITIONAL_PIP_FILE: docker/additional-pip-requirements.txt # Relative filepath to file containing additional pip packages to install
ENABLE_RECURSIVE_ADDITIONAL_PIP: 'false' # Enable recursive discovery of files named `additional-pip-file`
BLACKLISTED_PACKAGES_FILE: docker/blacklisted-packages.txt # Relative filepath to file containing the blacklisted packages
ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: 'false' # Enable recursive discovery of files named `blacklisted-packages-file`
CUSTOM_SCRIPT_FILE: docker/custom.sh # Relative filepath to script containing custom installation commands
ENABLE_RECURSIVE_CUSTOM_SCRIPT: 'false' # Enable recursive discovery of files named `custom-script-file`
VCS_IMPORT_FILE: .repos # Relative filepath to file containing additional repos to install via vcstools (only relevant if ENABLE_RECURSIVE_VCS_IMPORT=false)
ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `*.repos`
ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS: 'false' # Enable `rosdep install -r`
ENABLE_CONTINUE_BUILD_DESPITE_ERRORS 'false' # Enable `catkin build --continue-on-failure` / `colcon build --continue-on-error`
# -----
DOCKER_ROS_GIT_REF: main

Expand Down
8 changes: 8 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,8 @@ If your ROS-based repository (or any of your repository's upstream dependencies,
Create a file `blacklisted-packages.txt` in your `docker` folder (or configure a different `BLACKLISTED_PACKAGES_FILE`) and list any ROS package name to blacklist.
If you are having problems with (upstream) packages that you don't need, also have a look at `ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS` and `ENABLE_CONTINUE_BUILD_DESPITE_ERRORS`.

### Extra System Dependencies (*apt*)

If your ROS-based repository requires system dependencies that cannot be installed by specifying their [rosdep](https://docs.ros.org/en/independent/api/rosdep/html/) keys in a `package.xml`, you can use a special `additional-debs.txt` file.
Expand Down Expand Up @@ -395,6 +397,12 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe
- **`git-https-user` | `GIT_HTTPS_USER`**
Username for cloning private Git repositories via HTTPS
*default:* `${{ github.actor }}` | `gitlab-ci-token`
- **`enable-continue-build-despite-errors` | `ENABLE_CONTINUE_BUILD_DESPITE_ERRORS`**
Enable `catkin build --continue-on-failure` / `colcon build --continue-on-error`
*default:* `false`
- **`enable-continue-rosdep-install-despite-errors` | `ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS`**
Enable `rosdep install -r`
*default:* `false`
- **`enable-recursive-additional-debs` | `ENABLE_RECURSIVE_ADDITIONAL_DEBS`**
Enable recursive discovery of files named `additional-debs-file`
*default:* `false`
Expand Down
6 changes: 6 additions & 0 deletions action.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,8 @@ runs:
ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }}
ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }}
ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS: ${{ inputs.enable-continue-rosdep-install-despite-errors }}
ENABLE_CONTINUE_BUILD_DESPITE_ERRORS: ${{ inputs.enable-continue-build-despite-errors }}

- name: Set up industrial_ci
if: ${{ inputs.enable-industrial-ci == 'true' }}
Expand Down Expand Up @@ -274,6 +276,8 @@ runs:
ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }}
ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }}
ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS: ${{ inputs.enable-continue-rosdep-install-despite-errors }}
ENABLE_CONTINUE_BUILD_DESPITE_ERRORS: ${{ inputs.enable-continue-build-despite-errors }}
_ENABLE_IMAGE_PUSH: true
_IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }}

Expand Down Expand Up @@ -310,5 +314,7 @@ runs:
ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }}
VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }}
ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }}
ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS: ${{ inputs.enable-continue-rosdep-install-despite-errors }}
ENABLE_CONTINUE_BUILD_DESPITE_ERRORS: ${{ inputs.enable-continue-build-despite-errors }}
_ENABLE_IMAGE_PUSH: true
_IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }}
20 changes: 17 additions & 3 deletions docker/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ RUN if [[ -x "$(command -v colcon)" ]]; then \
$WORKSPACE/.remove-packages.sh 2> /dev/null | xargs rm -rfv
# create install script with list of rosdep dependencies
ARG ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS="false"
RUN echo "set -e" >> $WORKSPACE/.install-dependencies.sh && \
source /opt/ros/$ROS_DISTRO/setup.bash && \
apt-get update && \
Expand All @@ -103,7 +104,11 @@ RUN echo "set -e" >> $WORKSPACE/.install-dependencies.sh && \
export OS="ubuntu:$(lsb_release -c | awk '{print $2}')" && \
if [[ "$ROS_DISTRO" = "rolling" && "$OS" = "ubuntu:focal" ]]; then export OS="ubuntu:jammy"; fi && \
set -o pipefail && \
ROS_PACKAGE_PATH=$(pwd):$ROS_PACKAGE_PATH rosdep install --os $OS -y --simulate --from-paths src --ignore-src | tee -a $WORKSPACE/.install-dependencies.sh && \
if [[ $ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS == 'true' ]]; then \
ROS_PACKAGE_PATH=$(pwd):$ROS_PACKAGE_PATH rosdep install --os $OS -y --simulate --from-paths src --ignore-src -r | tee -a $WORKSPACE/.install-dependencies.sh ; \
else \
ROS_PACKAGE_PATH=$(pwd):$ROS_PACKAGE_PATH rosdep install --os $OS -y --simulate --from-paths src --ignore-src | tee -a $WORKSPACE/.install-dependencies.sh ; \
fi && \
chmod +x $WORKSPACE/.install-dependencies.sh && \
rm -rf /var/lib/apt/lists/*
Expand Down Expand Up @@ -250,12 +255,21 @@ CMD ["bash"]
FROM dev as build

# build ROS workspace
ARG ENABLE_CONTINUE_BUILD_DESPITE_ERRORS="false"
RUN if [[ -x "$(command -v colcon)" ]]; then \
source /opt/ros/${ROS_DISTRO}/setup.bash && \
colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release ; \
if [[ $ENABLE_CONTINUE_BUILD_DESPITE_ERRORS == 'true' ]]; then \
colcon build --continue-on-error --cmake-args -DCMAKE_BUILD_TYPE=Release ; \
else \
colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release ; \
fi ; \
elif [[ -x "$(command -v catkin)" ]]; then \
catkin config --install --extend /opt/ros/${ROS_DISTRO} && \
catkin build -DCMAKE_BUILD_TYPE=Release --force-color --no-status --summarize ; \
if [[ $ENABLE_CONTINUE_BUILD_DESPITE_ERRORS == 'true' ]]; then \
catkin build -DCMAKE_BUILD_TYPE=Release --force-color --no-status --summarize --continue-on-failure ; \
else \
catkin build -DCMAKE_BUILD_TYPE=Release --force-color --no-status --summarize ; \
fi ; \
fi

############ run ###############################################################
Expand Down
2 changes: 2 additions & 0 deletions scripts/build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ build_image() {
$(if [[ -n "${ENABLE_RECURSIVE_CUSTOM_SCRIPT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_CUSTOM_SCRIPT=${ENABLE_RECURSIVE_CUSTOM_SCRIPT}"; fi) \
$(if [[ -n "${VCS_IMPORT_FILE}" ]]; then echo "--build-arg VCS_IMPORT_FILE=${VCS_IMPORT_FILE}"; fi) \
$(if [[ -n "${ENABLE_RECURSIVE_VCS_IMPORT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_VCS_IMPORT=${ENABLE_RECURSIVE_VCS_IMPORT}"; fi) \
$(if [[ -n "${ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS}" ]]; then echo "--build-arg ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS=${ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS}"; fi) \
$(if [[ -n "${ENABLE_CONTINUE_BUILD_DESPITE_ERRORS}" ]]; then echo "--build-arg ENABLE_CONTINUE_BUILD_DESPITE_ERRORS=${ENABLE_CONTINUE_BUILD_DESPITE_ERRORS}"; fi) \
.
echo "Successfully built stage '${TARGET}' for platform '${PLATFORM}' as '${IMAGE}'"
}
Expand Down
2 changes: 2 additions & 0 deletions scripts/ci.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ CUSTOM_SCRIPT_FILE="${CUSTOM_SCRIPT_FILE:-}"
ENABLE_RECURSIVE_CUSTOM_SCRIPT="${ENABLE_RECURSIVE_CUSTOM_SCRIPT:-}"
VCS_IMPORT_FILE="${VCS_IMPORT_FILE:-}"
ENABLE_RECURSIVE_VCS_IMPORT="${ENABLE_RECURSIVE_VCS_IMPORT:-}"
ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS="${ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS:-}"
ENABLE_CONTINUE_BUILD_DESPITE_ERRORS="${ENABLE_CONTINUE_BUILD_DESPITE_ERRORS:-}"
_ENABLE_IMAGE_PUSH="${_ENABLE_IMAGE_PUSH:-false}"
_IMAGE_POSTFIX="${_IMAGE_POSTFIX:-""}"

Expand Down
2 changes: 2 additions & 0 deletions templates/docker-compose.template.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ x-build: &build
ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: $ENABLE_RECURSIVE_BLACKLISTED_PACKAGES
ENABLE_RECURSIVE_CUSTOM_SCRIPT: $ENABLE_RECURSIVE_CUSTOM_SCRIPT
ENABLE_RECURSIVE_VCS_IMPORT: $ENABLE_RECURSIVE_VCS_IMPORT
ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS: $ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS
ENABLE_CONTINUE_BUILD_DESPITE_ERRORS: $ENABLE_CONTINUE_BUILD_DESPITE_ERRORS
GIT_HTTPS_PASSWORD: $GIT_HTTPS_PASSWORD
GIT_HTTPS_SERVER: $GIT_HTTPS_SERVER
GIT_HTTPS_USER: $GIT_HTTPS_USER
Expand Down

0 comments on commit bc1309f

Please sign in to comment.