Skip to content

Commit

Permalink
Add Docker BuildX config (backport #266) (#270)
Browse files Browse the repository at this point in the history
* Add Docker BuildX config (#266)

* Allow pushing to apl-ocean-engineering

* Install cppzmq-dev for Gazebo

* Allow pushing to apl-ocean-engineering

* Reverse change in docker.yaml action.

* Preliminary version of docker-bake.hcl and updated Docker workflow.

* Updated comments in Dockerfile

* Attempt to allow manual triggering of builds

* Nope, that's not it.

* Add preliminary label to all Docker images.

* Removed redundant libcppzmq-dev

* Hack to lowercase repo name

* Re-activate the metadata-action, go back to stages in matrix.

* Add comment on lowercasing.

* Place files config at correct level.

* Re-activate the metadata-action, go back to stages in matrix.

* Place files config at correct level.

* Switch to registry cache for CI

* Removed stage from matrix

* Update repo URL, fix list formatting.

(cherry picked from commit a8681d0)

# Conflicts:
#	.github/workflows/docker.yaml

* Resolve conflicts while merging #266

---------

Co-authored-by: Aaron Marburg <amarburg@uw.edu>
  • Loading branch information
mergify[bot] and amarburg authored Aug 22, 2024
1 parent 0559e9c commit 388e8a6
Show file tree
Hide file tree
Showing 3 changed files with 170 additions and 130 deletions.
94 changes: 94 additions & 0 deletions .docker/docker-bake.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
#
# Override these variables with environment variables
# e.g.
#
# BLUE_ROS_DISTRO=iron docker buildx bake
#
# or
#
# export BLUE_ROS_DISTRO=iron
# docker buildx bake
#
variable "BLUE_ROS_DISTRO" { default = "rolling" }
variable "BLUE_GITHUB_REPO" { default = "robotic-decision-making-lab/blue" }

group "default" {
targets = ["ci", "robot", "desktop", "desktop-nvidia"]
}

# These are populated by the metadata-action Github action for each target
# when building in CI
#
target "docker-metadata-action-ci" {}
target "docker-metadata-action-robot" {}
target "docker-metadata-action-desktop" {}
target "docker-metadata-action-desktop-nvidia" {}


#
# All images can pull cache from the images published at Github
# or local storage (within the Buildkit image)
#
# ... and push cache to local storage
#
target "ci" {
inherits = ["docker-metadata-action-ci"]
dockerfile = ".docker/Dockerfile"
target = "ci"
context = ".."
args = {
ROS_DISTRO = "${BLUE_ROS_DISTRO}"
}
tags = [
"ghcr.io/${BLUE_GITHUB_REPO}:${BLUE_ROS_DISTRO}-ci"
]
labels = {
"org.opencontainers.image.source" = "https://github.com/${BLUE_GITHUB_REPO}"
}
cache_from =[
"ghcr.io/${BLUE_GITHUB_REPO}:cache-${BLUE_ROS_DISTRO}-ci",
"ghcr.io/${BLUE_GITHUB_REPO}:cache-${BLUE_ROS_DISTRO}-robot",
"ghcr.io/${BLUE_GITHUB_REPO}:cache-${BLUE_ROS_DISTRO}-desktop",
"ghcr.io/${BLUE_GITHUB_REPO}:cache-${BLUE_ROS_DISTRO}-desktop-nvidia",
"type=local,dest=.docker-cache"
]
cache_to = [
"type=local,dest=.docker-cache"
]
platforms = ["linux/amd64", "linux/arm64"]
}

target "robot" {
inherits = [ "ci", "docker-metadata-action-robot" ]
target = "robot"
tags = [
"ghcr.io/${BLUE_GITHUB_REPO}:${BLUE_ROS_DISTRO}-robot"
]
cache_to = [
"type=local,dest=.docker-cache"
]
}

target "desktop" {
inherits = [ "ci", "docker-metadata-action-desktop" ]
target = "desktop"
tags = [
"ghcr.io/${BLUE_GITHUB_REPO}:${BLUE_ROS_DISTRO}-desktop"
]
cache_to = [
"type=local,dest=.docker-cache"
]
# amd64 only builds for desktop and desktop-nvidia
platforms = ["linux/amd64"]
}

target "desktop-nvidia" {
inherits = [ "desktop", "docker-metadata-action-desktop-nvidia" ]
target = "desktop-nvidia"
tags = [
"ghcr.io/${BLUE_GITHUB_REPO}:${BLUE_ROS_DISTRO}-desktop-nvidia"
]
cache_to = [
"type=local,dest=.docker-cache"
]
}
203 changes: 73 additions & 130 deletions .github/workflows/docker.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
name: Docker

on:
schedule:
- cron: "0 17 * * 6"
# Disable schedule for preliminary testing
# schedule:
# - cron: "0 17 * * 6"
push:
branches:
- jazzy
Expand All @@ -18,7 +19,7 @@ env:
PUSH: ${{ (github.event_name != 'pull_request') && (github.repository == 'Robotic-Decision-Making-Lab/blue') }}

jobs:
ci:
docker_build:
strategy:
fail-fast: false
matrix:
Expand All @@ -28,164 +29,106 @@ jobs:
packages: write
contents: read
steps:
- name: Checkout repository
- name: Checkout
uses: actions/checkout@v4

- name: Log into registry
if: env.PUSH == 'true'
uses: docker/login-action@v3.3.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract Docker metadata
if: env.PUSH == 'true'
id: meta
uses: docker/metadata-action@v5.5.1
with:
images: ghcr.io/${{ github.repository }}
tags: |
type=raw,value=${{ matrix.ROS_DISTRO }}-${{ github.job }}
- name: Build and push Docker image
uses: docker/build-push-action@v6.7.0
with:
context: .
file: .docker/Dockerfile
build-args: ROS_DISTRO=${{ matrix.ROS_DISTRO }}
target: ${{ github.job }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
push: ${{ env.PUSH }}

robot:
strategy:
fail-fast: false
matrix:
ROS_DISTRO: [jazzy]
runs-on: ubuntu-latest
permissions:
packages: write
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Set up QEMU
uses: docker/setup-qemu-action@v3.2.0
- # Add support for more platforms with QEMU (optional)
# https://github.com/docker/setup-qemu-action
name: Set up QEMU
uses: docker/setup-qemu-action@v3

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Log into registry
if: env.PUSH == 'true'
- if: env.PUSH == 'true'
name: Log into registry
uses: docker/login-action@v3.3.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract Docker metadata
# buildx bake, unfortunately, requires lower-cased repository names
# This shell ... er, hack, creates a local variable containing
# a down-cased version of $GITHUB_REPOSITORY
#
- id: lowercase-repo
name: Repository to lowercase
run: |
echo "repository=${GITHUB_REPOSITORY@L}" >> $GITHUB_OUTPUT
# Set metadata for each stage-image separately
- name: Set Docker metadata for "ci"
if: env.PUSH == 'true'
id: meta
id: meta-ci
uses: docker/metadata-action@v5.5.1
with:
images: ghcr.io/${{ github.repository }}
bake-target: docker-metadata-action-ci
tags: |
type=raw,value=${{ matrix.ROS_DISTRO }}-${{ github.job }}
- name: Build and push Docker image
uses: docker/build-push-action@v6.7.0
with:
context: .
file: .docker/Dockerfile
build-args: ROS_DISTRO=${{ matrix.ROS_DISTRO }}
target: ${{ github.job }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
push: ${{ env.PUSH }}
platforms: linux/amd64
#platforms: linux/amd64,linux/arm64

desktop:
strategy:
fail-fast: false
matrix:
ROS_DISTRO: [jazzy]
runs-on: ubuntu-latest
permissions:
packages: write
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Log into registry
if: env.PUSH == 'true'
uses: docker/login-action@v3.3.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
type=raw,value=${{ matrix.ROS_DISTRO }}-ci
- name: Extract Docker metadata
- name: Set Docker metadata for "robot"
if: env.PUSH == 'true'
id: meta
id: meta-robot
uses: docker/metadata-action@v5.5.1
with:
images: ghcr.io/${{ github.repository }}
bake-target: docker-metadata-action-robot
tags: |
type=raw,value=${{ matrix.ROS_DISTRO }}-${{ github.job }}
- name: Build and push Docker image
uses: docker/build-push-action@v6.7.0
with:
context: .
file: .docker/Dockerfile
build-args: ROS_DISTRO=${{ matrix.ROS_DISTRO }}
target: ${{ github.job }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
push: ${{ env.PUSH }}

desktop-nvidia:
strategy:
fail-fast: false
matrix:
ROS_DISTRO: [jazzy]
runs-on: ubuntu-latest
permissions:
packages: write
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@v4
type=raw,value=${{ matrix.ROS_DISTRO }}-robot
- name: Log into registry
- name: Set Docker metadata for "desktop"
if: env.PUSH == 'true'
uses: docker/login-action@v3.3.0
id: meta-desktop
uses: docker/metadata-action@v5.5.1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
images: ghcr.io/${{ github.repository }}
bake-target: docker-metadata-action-desktop
tags: |
type=raw,value=${{ matrix.ROS_DISTRO }}-desktop
- name: Extract Docker metadata
- name: Set Docker metadata for "desktop-nvidia"
if: env.PUSH == 'true'
id: meta
id: meta-desktop-nvidia
uses: docker/metadata-action@v5.5.1
with:
images: ghcr.io/${{ github.repository }}
bake-target: docker-metadata-action-desktop-nvidia
tags: |
type=raw,value=${{ matrix.ROS_DISTRO }}-${{ github.job }}
- name: Build and push Docker image
uses: docker/build-push-action@v6.7.0
type=raw,value=${{ matrix.ROS_DISTRO }}-desktop-nvidia
- if: github.event_name == 'push'
name: Build and push (non PR)
uses: docker/bake-action@v5.5.0
env:
BLUE_ROS_DISTRO: ${{ matrix.ROS_DISTRO }}
BLUE_GITHUB_REPO: ${{ steps.lowercase-repo.outputs.repository }}
with:
context: .
file: .docker/Dockerfile
build-args: ROS_DISTRO=${{ matrix.ROS_DISTRO }}
target: ${{ github.job }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
workdir: .docker
files: |
./docker-bake.hcl
${{ steps.meta-ci.outputs.bake-file }}
${{ steps.meta-robot.outputs.bake-file }}
${{ steps.meta-desktop.outputs.bake-file }}
${{ steps.meta-desktop-nvidia.outputs.bake-file }}
push: ${{ env.PUSH }}
set: |
*.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:cache-${{ matrix.ROS_DISTRO }}
*.cache-to=type=registry,mode=max,ref=ghcr.io/${{ github.repository }}:cache-${{ matrix.ROS_DISTRO }}
# Pull request builds are not cached; and only built for AMD64
- if: github.event_name == 'pull_request'
name: Build and push (PR)
uses: docker/bake-action@v5.5.0
env:
BLUE_ROS_DISTRO: ${{ matrix.ROS_DISTRO }}
BLUE_GITHUB_REPO: ${{ steps.lowercase-repo.outputs.repository }}
with:
workdir: .docker
files: |
./docker-bake.hcl
set: |
*.platform=linux/amd64
*.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}:cache-${{ matrix.ROS_DISTRO }}
*.cache-to=
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,6 @@ mav.parm
mav.tlog
mav.tlog.raw
logs/

# Allow overrides in docker-bake
.docker/docker-bake.override.hcl

0 comments on commit 388e8a6

Please sign in to comment.