Build arm64 main image (#1487)

Co-authored-by: Guillaume Grossetie <ggrossetie@yuzutech.fr>

.github/workflows/main.yaml

@@ -11,12 +11,28 @@ on:
       - 'docs/**'
     branches:
       - '*'
+  workflow_call:
+    inputs:
+      build_multiarch:
+        default: 'false'
+        required: true
+        type: boolean
 
 jobs:
   test-containers:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v3
+
+    - name: Free up disk space
+      if: ${{ inputs.build_multiarch }}
+      uses: jlumbroso/free-disk-space@main
+      with:
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        swap-storage: true
 
     - name: Set up JDK 11
       uses: actions/setup-java@v3
@@ -31,11 +47,21 @@ jobs:
       run: make buildServer
 
     - name: Set up QEMU
+      if: ${{ inputs.build_multiarch }}
       uses: docker/setup-qemu-action@v2
 
     - name: Set up Docker Buildx
+      if: ${{ !inputs.build_multiarch }}
       uses: docker/setup-buildx-action@v2
 
+    - name: Set up Docker Buildx
+      if: ${{ inputs.build_multiarch }}
+      uses: docker/setup-buildx-action@v2
+      with:
+        config-inline: |
+          [worker.oci]
+            max-parallelism = 2
+
     - name: Cache Docker layers
       uses: actions/cache@v3
       with:
@@ -47,6 +73,7 @@ jobs:
     - name: Build container images
       run: make buildDockerImages
       env:
+        BUILD_MULTIARCH: ${{ inputs.build_multiarch }}
         CACHE_FROM: 'type=local,src=/tmp/.buildx-cache'
         CACHE_TO: 'type=local,dest=/tmp/.buildx-cache-new'
 

.github/workflows/nightly.yaml

@@ -0,0 +1,11 @@
+name: CI (nightly)
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  test-containers-multiarch:
+    uses: ./.github/workflows/main.yaml
+    with:
+      build_multiarch: true

Makefile

@@ -1,5 +1,3 @@
-MULTI_ARCH_AVAILABLE := $(shell docker buildx inspect | grep amd64 | grep arm64 > /dev/null 2>&1; echo $$?)
-
 TESTS_DIR=ci/tests
 COMPOSE_TIMEOUT=20
 SERVICES_TIMEOUT=15
@@ -19,20 +17,20 @@ endif
 	mvn versions:set -DnewVersion=$(RELEASE_VERSION)
 
 buildDockerImages:
-ifeq ($(MULTI_ARCH_AVAILABLE), 0)
-	docker buildx bake --set "*.cache-from=$(CACHE_FROM)" --set "*.cache-to=$(CACHE_TO)" --set "kroki-*.platform=linux/arm64,linux/amd64"
+ifdef BUILD_MULTIARCH
+	docker buildx bake kroki companion-images --set "*.cache-from=$(CACHE_FROM)" --set "*.cache-to=$(CACHE_TO)" --set "*.platform=linux/arm64,linux/amd64"
 else
-	docker buildx bake --set "*.cache-from=$(CACHE_FROM)" --set "*.cache-to=$(CACHE_TO)"
+	docker buildx bake kroki companion-images --set "*.cache-from=$(CACHE_FROM)" --set "*.cache-to=$(CACHE_TO)"
 endif
 
 publishDockerImages:
 ifndef RELEASE_VERSION
 	$(error RELEASE_VERSION is undefined)
 endif
-	docker buildx bake -f docker-bake.hcl -f docker-bake-release.hcl --push --set "kroki-*.platform=linux/arm64,linux/amd64"
+	docker buildx bake -f docker-bake.hcl -f docker-bake-release.hcl kroki companion-images --push --set "*.platform=linux/arm64,linux/amd64"
 
 smokeTests:
-	TAG=smoketests docker buildx bake --load --set "*.cache-from=$(CACHE_FROM)" --set "*.cache-to=$(CACHE_TO)"
+	TAG=smoketests docker buildx bake kroki companion-images --load --set "*.cache-from=$(CACHE_FROM)" --set "*.cache-to=$(CACHE_TO)"
 	@docker-compose --file "$(TESTS_DIR)/docker-compose.yaml" up --build --detach \
 	&& echo \
 	&& docker-compose --file "$(TESTS_DIR)/docker-compose.yaml" ps \

bpmn/Dockerfile

@@ -1,5 +1,5 @@
 # Package the Node.js project into a single binary
-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:16.18.1-alpine3.16 as builder
+FROM --platform=$BUILDPLATFORM node:16.18.1-alpine3.16 as builder
 
 # Workaround: https://github.com/nodejs/docker-node/issues/813#issuecomment-407339011
 # Error: could not get uid/gid
@@ -22,7 +22,7 @@ RUN npm run lint
 RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM}-$([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH") . -o app.bin
 
 # Create the image
-FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine:3.16.3
+FROM alpine:3.16.3
 
 RUN addgroup -g 1000 kroki && adduser -D -G kroki -u 1000 kroki
 

diagrams.net/Dockerfile

@@ -1,5 +1,5 @@
 # Package the Node.js project into a single binary
-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:16.18.1-alpine3.16 as builder
+FROM --platform=$BUILDPLATFORM node:16.18.1-alpine3.16 as builder
 
 # Workaround: https://github.com/nodejs/docker-node/issues/813#issuecomment-407339011
 # Error: could not get uid/gid
@@ -22,7 +22,7 @@ RUN npm run lint
 RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM}-$([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH") . -o app.bin
 
 # Create the image
-FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine:3.16.3
+FROM alpine:3.16.3
 
 RUN addgroup -g 1000 kroki && adduser -D -G kroki -u 1000 kroki
 

docker-bake.hcl

@@ -2,8 +2,8 @@ variable "TAG" {
   default = "latest"
 }
 
-group "default" {
-  targets = ["kroki", "kroki-blockdiag", "kroki-mermaid", "kroki-bpmn", "kroki-excalidraw", "kroki-diagramsnet", "kroki-wireviz"]
+group "companion-images" {
+  targets = ["kroki-blockdiag", "kroki-mermaid", "kroki-bpmn", "kroki-excalidraw", "kroki-diagramsnet", "kroki-wireviz"]
 }
 
 target "kroki" {

excalidraw/Dockerfile

@@ -1,5 +1,5 @@
 # Package the Node.js project into a single binary
-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:16.18.1-alpine3.16 as builder
+FROM --platform=$BUILDPLATFORM node:16.18.1-alpine3.16 as builder
 
 # Workaround: https://github.com/nodejs/docker-node/issues/813#issuecomment-407339011
 # Error: could not get uid/gid
@@ -24,7 +24,7 @@ RUN npm run prestart
 RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM}-$([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH") . -o app.bin
 
 # Create the image
-FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine:3.16.3
+FROM alpine:3.16.3
 
 RUN addgroup -g 1000 kroki && adduser -D -G kroki -u 1000 kroki
 

mermaid/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:16.18-alpine3.16
+FROM node:16.18-alpine3.16
 
 RUN addgroup -g 1001 kroki && adduser -D -G kroki -u 1001 kroki
 

server/ops/docker/.cargo/config

@@ -0,0 +1,5 @@
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"
+
+[target.x86_64-unknown-linux-gnu]
+linker = "x86_64-linux-gnu-gcc"

server/ops/docker/jdk11-jammy/Dockerfile

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1.4
 
 ## TikZ
-FROM ubuntu:jammy AS kroki-builder-dvisvgm
+FROM --platform=$BUILDPLATFORM ubuntu:jammy AS kroki-builder-dvisvgm
 
 RUN apt-get update && apt-get install --no-install-recommends --yes \
     build-essential \
@@ -17,40 +17,50 @@ RUN apt-get update && apt-get install --no-install-recommends --yes \
 
 ARG DVISVGM_VERSION=3.0.4
 
-RUN \
-    curl -LO https://github.com/mgieseki/dvisvgm/releases/download/$DVISVGM_VERSION/dvisvgm-$DVISVGM_VERSION.tar.gz \
-    && tar xf dvisvgm-$DVISVGM_VERSION.tar.gz \
-    && cd dvisvgm-$DVISVGM_VERSION \
-    && ./configure --enable-bundled-libs \
-    && make \
-    && make install
+ARG BUILDARCH
+ARG TARGETARCH
+RUN curl -LO https://github.com/mgieseki/dvisvgm/releases/download/$DVISVGM_VERSION/dvisvgm-$DVISVGM_VERSION.tar.gz && \
+    tar xf dvisvgm-$DVISVGM_VERSION.tar.gz && \
+    cd dvisvgm-$DVISVGM_VERSION && \
+    case $BUILDARCH in \
+        "amd64") MAKE_BUILD_ARCH=x86_64 ;; \
+        "arm64") MAKE_BUILD_ARCH=aarch64 ;; \
+        *) MAKE_BUILD_ARCH=$TARGETARCH ;; \
+    esac && \
+    case $TARGETARCH in \
+        "amd64") MAKE_TARGET_ARCH=x86_64 ;; \
+        "arm64") MAKE_TARGET_ARCH=aarch64 ;; \
+        *) MAKE_TARGET_ARCH=$TARGETARCH ;; \
+    esac && \
+    ./configure --enable-bundled-libs --build $MAKE_BUILD_ARCH-linux-gnu --host $MAKE_TARGET_ARCH-linux-gnu && \
+    make && \
+    make install
 
 ## D2
 FROM golang:1.19.3-bullseye AS kroki-builder-d2
 COPY ops/docker/go.mod .
 
-RUN D2_VERSION=`cat go.mod | grep "oss.terrastruct.com/d2" | cut -d' ' -f3` \
-  && rm go.mod \
-  && go install "oss.terrastruct.com/d2@${D2_VERSION}"
+RUN D2_VERSION=`cat go.mod | grep "oss.terrastruct.com/d2" | cut -d' ' -f3` && \
+    rm go.mod && \
+    go install "oss.terrastruct.com/d2@${D2_VERSION}"
 
 ## Nomnoml
-FROM node:16.18.1-bullseye-slim AS kroki-builder-nomnoml
+FROM --platform=$BUILDPLATFORM node:16.18.1-bullseye-slim AS kroki-builder-nomnoml
 
 RUN npm install -g pkg@5.6.0 pkg-fetch@3.3.0
 
 ENV NODE node16
 ENV PLATFORM linux
-RUN /usr/local/bin/pkg-fetch -n ${NODE} -p ${PLATFORM}
+RUN /usr/local/bin/pkg-fetch -n ${NODE} -p ${PLATFORM} -a $([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH")
 
 COPY --from=nomnoml index.js package.json package-lock.json /app/
 WORKDIR /app
 
-RUN npm i
-RUN npm run lint
-RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM} . -o app.bin
+RUN npm install && npm run lint
+RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM}-$([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH") . -o app.bin
 
 ## Vega
-FROM node:16.18.1-bullseye-slim AS kroki-builder-vega
+FROM --platform=$BUILDPLATFORM node:16.18.1-bullseye-slim AS kroki-builder-vega
 
 # System dependencies for "canvas" Node package
 # https://github.com/Automattic/node-canvas#compiling
@@ -67,65 +77,62 @@ RUN npm install -g pkg@5.6.0 pkg-fetch@3.3.0
 
 ENV NODE node16
 ENV PLATFORM linux
-RUN /usr/local/bin/pkg-fetch -n ${NODE} -p ${PLATFORM}
+RUN /usr/local/bin/pkg-fetch -n ${NODE} -p ${PLATFORM} -a $([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH")
 
 COPY --from=vega src /app/src
 COPY --from=vega tests /app/tests
 COPY --from=vega package.json package-lock.json /app/
 WORKDIR /app
 
-RUN npm i
-RUN npm run lint && npm t
-RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM} . -o app.bin
+RUN npm install 2 && npm run lint && npm t
+RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM}-$([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH") . -o app.bin
 
 ## DBML
-FROM node:16.18.1-bullseye-slim AS kroki-builder-dbml
+FROM --platform=$BUILDPLATFORM node:16.18.1-bullseye-slim AS kroki-builder-dbml
 
 RUN npm config set unsafe-perm true
 
 RUN npm install -g pkg@5.6.0 pkg-fetch@3.3.0
 
 ENV NODE node16
 ENV PLATFORM linux
-RUN /usr/local/bin/pkg-fetch -n ${NODE} -p ${PLATFORM}
+RUN /usr/local/bin/pkg-fetch -n ${NODE} -p ${PLATFORM} -a $([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH")
 
 COPY --from=dbml index.js package.json package-lock.json /app/
 WORKDIR /app
 
-RUN npm i
-RUN npm run lint
-RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM} . -o app.bin
+RUN npm install && npm run lint
+RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM}-$([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH") . -o app.bin
 
 ## Wavedrom
-FROM node:16.18.1-bullseye-slim AS kroki-builder-wavedrom
+FROM --platform=$BUILDPLATFORM node:16.18.1-bullseye-slim AS kroki-builder-wavedrom
 
 RUN npm install -g pkg@5.6.0 pkg-fetch@3.3.0
 
 ENV NODE node16
 ENV PLATFORM linux
-RUN /usr/local/bin/pkg-fetch -n ${NODE} -p ${PLATFORM}
+RUN /usr/local/bin/pkg-fetch -n ${NODE} -p ${PLATFORM} -a $([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH")
 
 COPY --from=wavedrom index.js package.json package-lock.json /app/
 WORKDIR /app
 
-RUN npm i
-RUN npm run lint
-RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM} . -o app.bin
+RUN npm install && npm run lint
+RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM}-$([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH") . -o app.bin
 
 ## Bytefield
-FROM node:16.18.1-bullseye-slim AS kroki-builder-bytefield
+FROM --platform=$BUILDPLATFORM node:16.18.1-bullseye-slim AS kroki-builder-bytefield
 
 RUN npm install -g pkg@5.6.0 pkg-fetch@3.3.0
 
 ENV NODE node16
 ENV PLATFORM linux
-RUN /usr/local/bin/pkg-fetch -n ${NODE} -p ${PLATFORM}
+RUN /usr/local/bin/pkg-fetch -n ${NODE} -p ${PLATFORM} -a $([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH")
 
 COPY --from=bytefield index.js package.json package-lock.json /app/
 WORKDIR /app
 
-RUN npm i
-RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM} . -o app.bin
+RUN npm install
+RUN /usr/local/bin/pkg --targets ${NODE}-${PLATFORM}-$([ "$TARGETARCH" == "amd64" ] && echo "x64" || echo "$TARGETARCH") . -o app.bin
 
 ## ERD
 # use a pre-built image to reduce build time
@@ -170,12 +177,29 @@ RUN wget -q "https://pikchr.org/home/raw/${PIKCHR_VERSION}" -O pikchr.c
 RUN gcc -O0 -g -static -Wall -Wextra -DPIKCHR_SHELL pikchr.c -o pikchr -lm
 
 ## SVGBob
-FROM rust:1.65-slim-bullseye AS kroki-builder-static-svgbob
-RUN rustup target add x86_64-unknown-linux-musl
-COPY ops/docker/Cargo.toml .
+FROM --platform=$BUILDPLATFORM rust:1.68-slim-bullseye AS kroki-builder-static-svgbob
+
+# Install cross-compilation tools
+RUN apt-get update && apt-get install --no-install-recommends --yes \
+    gcc-aarch64-linux-gnu \
+    gcc-x86-64-linux-gnu \
+    libc6-dev-amd64-cross \
+    libc6-dev-arm64-cross && \
+    apt-get clean && apt-get autoremove
 
-RUN SVGBOB_VERSION=`cat Cargo.toml | grep "svgbob_cli =" | sed -r 's/.*"([^"]+)"/\1/'` \
-  && cargo install --quiet --target x86_64-unknown-linux-musl --version $SVGBOB_VERSION svgbob_cli
+COPY ops/docker/Cargo.toml .
+COPY ops/docker/.cargo .cargo
+
+ARG TARGETARCH
+ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
+RUN SVGBOB_VERSION=`cat Cargo.toml | grep "svgbob_cli =" | sed -r 's/.*"([^"]+)"/\1/'` && \
+    case $TARGETARCH in \
+        "amd64") RUST_TARGET_ARCH=x86_64 ;; \
+        "arm64") RUST_TARGET_ARCH=aarch64 ;; \
+        *) RUST_TARGET_ARCH=$TARGETARCH ;; \
+    esac && \
+    rustup target add $RUST_TARGET_ARCH-unknown-linux-gnu && \
+    cargo install --quiet --target $RUST_TARGET_ARCH-unknown-linux-gnu --version $SVGBOB_VERSION svgbob_cli
 
 ## UMlet
 # use a pre-built image to reduce build time
@@ -226,7 +250,8 @@ RUN apt-get update && apt-get install --no-install-recommends --yes \
     texlive-latex-extra \
     texlive-pictures \
     texlive-pstricks \
-    texlive-science
+    texlive-science && \
+    apt-get clean && apt-get autoremove
 
 COPY --chown=kroki:kroki ops/docker/logback.xml /etc/kroki/logback.xml
 

server/src/test/java/io/kroki/server/service/VegaServiceTest.java

@@ -33,8 +33,8 @@ public void should_call_vega_with_correct_arguments() throws Throwable {
       assertThat(buffer.toString()).isEqualTo("<svg>vega</svg>");
       testContext.completeNow();
     })));
-    // wait at most 2000ms
-    assertThat(testContext.awaitCompletion(2, TimeUnit.SECONDS)).isTrue();
+    // wait at most 4000ms
+    assertThat(testContext.awaitCompletion(4, TimeUnit.SECONDS)).isTrue();
     if (testContext.failed()) {
       throw testContext.causeOfFailure();
     }
@@ -55,8 +55,8 @@ public void should_call_vega_lite_with_correct_arguments() throws Throwable {
       assertThat(buffer.toString()).isEqualTo("<svg>vega-lite</svg>");
       testContext.completeNow();
     })));
-    // wait at most 2000ms
-    assertThat(testContext.awaitCompletion(2, TimeUnit.SECONDS)).isTrue();
+    // wait at most 4000ms
+    assertThat(testContext.awaitCompletion(4, TimeUnit.SECONDS)).isTrue();
     if (testContext.failed()) {
       throw testContext.causeOfFailure();
     }