Switch from docker-compose build to docker buildx bake

.github/workflows/core.yml

@@ -43,47 +43,12 @@ jobs:
         with:
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build the tagged Docker image
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Build and push Docker images
         run: |
-          docker-compose -f compose/${{ matrix.r_version }}.yml build
-      - name: Push the tagged Docker image
-        run: |
-          docker-compose -f compose/${{ matrix.r_version }}.yml push
-      - name: Push ml multi-tagged image
-        run: |
-          docker image tag rocker/cuda:${{ matrix.r_version }} rocker/cuda:${{ matrix.r_version }}-cuda10.1
-          docker image tag rocker/cuda:${{ matrix.r_version }} rocker/r-ver:${{ matrix.r_version }}-cuda10.1
-          docker image tag rocker/ml:${{ matrix.r_version }} rocker/ml:${{ matrix.r_version }}-cuda10.1
-          docker image tag rocker/ml-verse:${{ matrix.r_version }} rocker/ml-verse:${{ matrix.r_version }}-cuda10.1
-          docker image push rocker/cuda:${{ matrix.r_version }}-cuda10.1
-          docker image push rocker/r-ver:${{ matrix.r_version }}-cuda10.1
-          docker image push rocker/ml:${{ matrix.r_version }}-cuda10.1
-          docker image push rocker/ml-verse:${{ matrix.r_version }}-cuda10.1
-      - name: Push latest tagged Docker image
-        if: matrix.r_latest == true
-        run: |
-          docker image tag rocker/r-ver:${{ matrix.r_version }} rocker/r-ver:latest
-          docker image tag rocker/rstudio:${{ matrix.r_version }} rocker/rstudio:latest
-          docker image tag rocker/tidyverse:${{ matrix.r_version }} rocker/tidyverse:latest
-          docker image tag rocker/verse:${{ matrix.r_version }} rocker/verse:latest
-          docker image tag rocker/shiny:${{ matrix.r_version }} rocker/shiny:latest
-          docker image tag rocker/shiny-verse:${{ matrix.r_version }} rocker/shiny-verse:latest
-          docker image tag rocker/geospatial:${{ matrix.r_version }} rocker/geospatial:latest
-          docker image tag rocker/binder:${{ matrix.r_version }} rocker/binder:latest
-          docker image tag rocker/cuda:${{ matrix.r_version }} rocker/cuda:latest
-          docker image tag rocker/ml:${{ matrix.r_version }} rocker/ml:latest
-          docker image tag rocker/ml-verse:${{ matrix.r_version }} rocker/ml-verse:latest
-          docker image push rocker/r-ver:latest
-          docker image push rocker/rstudio:latest
-          docker image push rocker/tidyverse:latest
-          docker image push rocker/verse:latest
-          docker image push rocker/shiny:latest
-          docker image push rocker/shiny-verse:latest
-          docker image push rocker/geospatial:latest
-          docker image push rocker/binder:latest
-          docker image push rocker/cuda:latest
-          docker image push rocker/ml:latest
-          docker image push rocker/ml-verse:latest
+          BAKE_JSON=bakefiles/${{ matrix.r_version }}.docker-bake.json BAKE_OPTION=--load\ --push make bake-json-all
       - name: Inspect built image
         run: |
           IMAGELIST_NAME=${{ matrix.r_version }}.tsv make inspect-image-all

Makefile

@@ -1,81 +1,45 @@
-LATEST_TAG=4.1.1
-
 SHELL=/bin/bash
-STACKFILES=$(wildcard stacks/*.json)
-STACKS=$(notdir $(basename $(STACKFILES)))
-COMPOSEFILES=$(addprefix compose/,$(addsuffix .yml,$(STACKS)))
-PUSHES=$(addsuffix .push,$(STACKS))
-
-.PHONY: clean build setup push latest
-.PHONY: $(STACKS) $(PUSHES)
-
-all: clean build push
-latest: clean setup $(LATEST_TAG)
 
+.PHONY: clean setup print-% pull-image% bake-json% inspect-image% report% wiki%
 
-## Alternate way of specifying stacks by group
-4.1.1: clean setup 4.1.1
-4.1.0: clean setup 4.1.0
-4.0.5: clean setup 4.0.5
-4.0.4: clean setup 4.0.4
-4.0.3: clean setup 4.0.3
-4.0.2: clean setup 4.0.2
-4.0.1: clean setup 4.0.1
-4.0.0: clean setup 4.0.0
+all:
 
-
-setup: ./build/make-dockerfiles.R ./build/write-compose.R ./build/make-bakejson.R $(STACKFILES)
+setup:
 	./build/make-dockerfiles.R
 	./build/write-compose.R
 	./build/make-bakejson.R
 
-
-## Builds all stacks
-build: $(STACKS)
-
-
-
-
-$(STACKS): %: compose/%.yml
-	docker-compose -f compose/$@.yml build
-	docker-compose -f compose/$@.yml push
-
-
-
-## Dependency order
-geospatial-ubuntugis: $(LATEST_TAG)
-geospatial-dev-osgeos: $(LATEST_TAG)
-geospatial-4.0.0-ubuntu18.04: core-4.0.0-ubuntu18.04
-
-
-## Assumes we are logged into the Docker Registry already
-push: $(PUSHES)
-
-$(PUSHES): %.push: %
-	docker-compose -f compose/$<.yml push; \
-	./tag.sh $< $(LATEST_TAG)
-
-
 IMAGE_SOURCE ?= https://github.com/rocker-org/rocker-versioned2
-IMAGE_FILTER ?= label=org.opencontainers.image.source=$(IMAGE_SOURCE)
+COMMIT_HASH := $(shell git rev-parse HEAD)
+IMAGE_VERSION ?= $(COMMIT_HASH)
 REPORT_SOURCE_ROOT ?= tmp/inspects
 IMAGELIST_DIR ?= tmp/imagelist
 IMAGELIST_NAME ?= imagelist.tsv
 REPORT_DIR ?= reports
 
-## Display the value. ex. print-REPORT_SOURCE_DIR
+# Display the value.
+# ex. $ make print-REPORT_SOURCE_DIR
+# ex. $ make print-IMAGE_VERSION
 print-%:
 	@echo $* = $($*)
 
-
-# Set docker-bake.json file path to BAKE_JSON before running `make pull-image-all`.
-# ex. $ BAKE_JSON=bakefiles/devel.docker-bake.json make pull-image-all
+# Set docker-bake.json file path to BAKE_JSON before running `make pull-image-all` or `make bake-json-all`.
+# ex. $ BAKE_JSONbakefiles/core-latest-daily.docker-bake.json make pull-image-all
 BAKE_JSON ?= ""
 pull-image/%:
 	-docker pull $(subst pull-image/, , $@)
 pull-image-all: $(foreach I, $(shell jq '.target[].tags[]' -r $(BAKE_JSON) | sed -e 's/:/\\:/g'), pull-image/$(I))
 
+# docker buildx bake options. When specifying multiple options, please escape spaces with "\".
+# ex. $ BAKE_JSON=bakefiles/core-latest-daily.docker-bake.json BAKE_OPTION=--load make bake-json-all
+# ex. $ BAKE_JSON=bakefiles/devel.docker-bake.json BAKE_OPTION=--print\ -f\ build/platforms.docker-bake.override.json make bake-json/r-ver
+BAKE_OPTION ?= --print
+bake-json/%:
+	docker buildx bake -f $(BAKE_JSON) --set=*.labels.org.opencontainers.image.revision=$(IMAGE_VERSION) $(BAKE_OPTION) $(@F)
+bake-json-all: $(foreach I, $(shell jq '.target | keys_unsorted | .[]' -r $(BAKE_JSON)), bake-json/$(I))
+
 
+IMAGE_FILTER ?= label=org.opencontainers.image.source=$(IMAGE_SOURCE)
 inspect-image/%:
 	mkdir -p $(REPORT_SOURCE_ROOT)/$(@F)
 	-docker image inspect $(@F) > $(REPORT_SOURCE_ROOT)/$(@F)/docker_inspect.json
@@ -86,16 +50,18 @@ inspect-image-all: $(foreach I, $(shell docker image ls -q -f "$(IMAGE_FILTER)")
 	mkdir -p $(IMAGELIST_DIR)
 	docker image ls -f "$(IMAGE_FILTER)" --format "{{.ID}}\t{{.Repository}}\t{{.Tag}}\t{{.CreatedAt}}" > $(IMAGELIST_DIR)/$(IMAGELIST_NAME)
 
+
 REPORT_SOURCE_DIR := $(wildcard $(REPORT_SOURCE_ROOT)/*)
 report/%:
 	mkdir -p $(REPORT_DIR)
 	-./build/knit-report.R -d ../../$(REPORT_SOURCE_ROOT)/$(@F) $(@F) $(REPORT_DIR)
 report-all: $(foreach I, $(REPORT_SOURCE_DIR), report/$(I))
 
+
 # Move image list to wiki and update Home.md
 wiki-home:
 	cp -r $(IMAGELIST_DIR) $(REPORT_DIR)
 	-Rscript -e 'rmarkdown::render(input = "build/reports/wiki_home.Rmd", output_dir = "$(REPORT_DIR)", output_file = "Home.md")'
 
 clean:
-	rm -f dockerfiles/Dockerfile_* compose/*.yml bakefiles/*.json
+	rm -f dockerfiles/Dockerfile_* compose/*.yml bakefiles/*.json tmp/*

build/README.md

@@ -4,12 +4,13 @@ This folder contains scripts and other files to create (or assist in creating) d
 
 ## Automatic update of container definition files
 
-The Dockerfiles for pre-build images will be generated by executing the scripts in the following order. It will also generate the `docker-compose.yml` files that can be used to build the container images.
+The Dockerfiles for pre-build images will be generated by executing the scripts in the following order. It will also generate the `docker-compose.yml` files and the `docker-bake.json` files that can be used to build the container images.
 
 1. `./build/sync-template-vars.R`
 2. `./build/make-stacks.R`
 3. `./build/make-dockerfiles.R`
 4. `./build/write-compose.R`
+5. `./build/make-bakejson.R`
 
 These scripts are run daily by GitHub Actions and automatically create a Pull Request if the run results in a variance from the default branch. For example, check the PR when R4.1.1 was released. <https://github.com/rocker-org/rocker-versioned2/pull/213>
 

build/platforms.docker-bake.override.json

@@ -0,0 +1,10 @@
+{
+  "target": {
+    "r-ver": {
+      "platforms": [
+        "linux/amd64",
+        "linux/arm64"
+      ]
+    }
+  }
+}