New build and CI system (#4)

Build with docker bake and apply with multiple tags. Idea borrowed from aiidalab images.
The tests are added to confirm the libraries are compiled into the container.

.github/workflows/ci.yml

@@ -0,0 +1,184 @@
+---
+name: Build images and run tests and publish
+
+on:
+  pull_request:
+  push:
+      branches:
+          - main
+      tags:
+          - "v*"
+  workflow_dispatch:  
+
+env:
+  BUILDKIT_PROGRESS: plain
+  FORCE_COLOR: 1
+
+# https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+    # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    outputs:
+      image: ${{ steps.bake_metadata.outputs.image }}
+
+    steps:
+      - name: Checkout Repo ⚡️
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        if: ${{ inputs.platforms != 'linux/amd64' }}
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry 🔑
+        uses: docker/login-action@v3
+        with:
+            registry: ghcr.io
+            username: ${{ github.actor }}
+            password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+
+      - uses: crazy-max/ghaction-github-runtime@v3
+      - name: Build and upload to ghcr.io 📤
+        id: build-upload
+        uses: docker/bake-action@v4
+        with:
+            push: true
+            # Using provenance to disable default attestation so it will build only desired images:
+            # https://github.com/orgs/community/discussions/45969
+            provenance: false
+            set: |
+                *.platform=linux/amd64
+                *.output=type=registry,name-canonical=true,push-by-digest=true
+                *.cache-from=type=gha
+                *.cache-to=type=gha,mode=max
+
+            files: |
+                docker-bake.hcl
+                build.json
+                .github/workflows/env.hcl
+
+      - name: Set output variables
+        id: bake_metadata
+        run: |
+            .github/workflows/extract-image-name.sh | tee -a "${GITHUB_OUTPUT}"
+        env:
+            BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }}
+
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: build
+
+    steps:
+
+      - name: Checkout Repo ⚡️
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry 🔑
+        uses: docker/login-action@v3
+        with:
+            registry: ghcr.io
+            username: ${{ github.actor }}
+            password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run container checking libraries exist in the container
+        run: |
+          docker run --rm ${{ needs.build.outputs.image }} /bin/bash -c "ls -l /usr/local" > /tmp/ls-l.txt
+          if grep -q libxc /tmp/ls-l.txt; then
+              echo "libxc found"
+          else
+              echo "libxc not found"
+              exit 1
+          fi
+
+          if grep -q lapack /tmp/ls-l.txt; then
+              echo "lapack found"
+          else
+              echo "lapack not found"
+              exit 1
+          fi
+
+  publish:
+      runs-on: ubuntu-latest
+      timeout-minutes: 30
+      needs: [build, test]
+      strategy:
+        matrix:
+          registry: ["docker.io", "ghcr.io"]
+      if: >-
+          github.repository == 'pspgen/build-machine'
+          && (github.ref_type == 'tag' || github.ref_name == 'main')
+
+      steps:
+        - uses: actions/checkout@v4
+
+        - name: Login to GitHub Container Registry 🔑
+          uses: docker/login-action@v3
+          with:
+              registry: ghcr.io
+              username: ${{ github.actor }}
+              password: ${{ secrets.GITHUB_TOKEN }}
+
+        - name: Login to DockerHub 🔑
+          uses: docker/login-action@v3
+          if: inputs.registry == 'docker.io'
+          with:
+              registry: docker.io
+              username: ${{ secrets.DOCKER_USERNAME }}
+              password: ${{ secrets.DOCKER_PASSWORD }}
+
+        - name: Read build variables
+          id: build_vars
+          run: |
+              vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries')
+              echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}"
+
+        - name: Docker meta
+          id: meta
+          uses: docker/metadata-action@v5
+          env: ${{ fromJSON(steps.build_vars.outputs.vars) }}
+          with:
+              images: ${{ matrix.registry }}/${{ github.repository_owner }}/build-machine
+              tags: |
+                  type=edge,enable={{is_default_branch}}
+                  type=raw,value={{tag}},enable=${{ github.ref_type == 'tag' && ! startsWith(github.ref_name, 'v') }}
+                  type=raw,value=gnu-compiler-${{ env.GNU_COMPILER_VERSION }},enable=${{ github.ref_type == 'tag'  && startsWith(github.ref_name, 'v') }}
+                  type=raw,value=libxc-${{ env.LIBXC_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }}
+                  type=raw,value=lapack-${{ env.LAPACK_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }}
+                  type=match,pattern=v(\d{4}\.\d{4}(-.+)?),group=1
+
+        - name: Determine source image
+          id: images
+          run: |
+              src=$(echo '${{ inputs.images }}'| jq -cr '.[("${{ matrix.target }}"|ascii_upcase|sub("-"; "_"; "g")) + "_IMAGE"]')
+              echo "src=$src" | tee -a "${GITHUB_OUTPUT}"
+
+        - name: Push image
+          uses: akhilerm/tag-push-action@v2.2.0
+          with:
+              src: ${{ needs.build.outputs.image }}
+              dst: ${{ steps.meta.outputs.tags }}
+
+        - name: Docker Hub Description
+          if: inputs.registry == 'docker.io'
+          uses: peter-evans/dockerhub-description@v4
+          with:
+              username: ${{ secrets.DOCKER_USERNAME }}
+              password: ${{ secrets.DOCKER_PASSWORD }}
+              repository: pspgen/build-machine
+              short-description: ${{ github.event.repository.description }}

.github/workflows/env.hcl

@@ -0,0 +1,2 @@
+# env.hcl
+REGISTRY = "ghcr.io"

.github/workflows/extract-image-name.sh

@@ -0,0 +1,34 @@
+# Extract image names together with their sha256 digests
+# from the docker/bake-action metadata output.
+# These together uniquely identify newly built images.
+
+# The input to this script is a JSON string passed via BAKE_METADATA env variable
+# Here's example input (trimmed to relevant bits):
+# BAKE_METADATA: {
+#    "base": {
+#      "containerimage.descriptor": {
+#        "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+#        "digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d",
+#        "size": 6170,
+#      },
+#      "containerimage.digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d",
+#      "image.name": "ghcr.io/pspgen/build-machine"
+#    }
+#  }
+#
+# Example output (real output is on one line):
+#
+# image="ghcr.io/pspgen/build-machine@sha256:79a0f984b9e03b733304fda809ad3e8eec8416992ff334052d75da00cadb8f12"
+# }
+#
+# This json output is later turned to environment variables using fromJson() GHA builtin
+# (e.g. BUILD_MACHINE_IMAGE=ghcr.io/pspgen/build-machine@sha256:8e57a52b...)
+# and these are in turn read in the docker-compose.<target>.yml files for tests.
+
+if [[ -z ${BAKE_METADATA-} ]];then
+    echo "ERROR: Environment variable BAKE_METADATA is not set!"
+    exit 1
+fi
+
+image=$(echo "${BAKE_METADATA}" | jq -c '. as $base | to_entries[] | [(.value."image.name"|split(",")[0]),(.value."containerimage.digest")]|join("@")')
+echo "image=$image"

Dockerfile

@@ -1,34 +1,42 @@
-FROM ubuntu:focal
+# syntax=docker/dockerfile:1
+FROM base-image
 
+# Tool chain for building
 RUN apt-get update && apt-get install -y \
     build-essential \
     automake \
     autoconf \
     libtool \
     wget \
-    gfortran-7
+    gfortran-7 && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
-RUN update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-7 7 && \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 7
+ARG GNU_COMPILER_VERSION
+
+RUN update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-${GNU_COMPILER_VERSION} 2 && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GNU_COMPILER_VERSION} 2 
 
 WORKDIR /build
 
-# compile lapack-3.10.1
-RUN wget -c -O lapack-3.10.1.tar.gz https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.10.1.tar.gz && \
-    tar xf lapack-3.10.1.tar.gz && \
-    cd lapack-3.10.1 && \
+ARG LAPACK_VERSION
+
+RUN wget -c -O lapack.tar.gz https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v${LAPACK_VERSION}.tar.gz && \
+    mkdir -p lapack && \
+    tar xf lapack.tar.gz -C lapack --strip-components=1 && \
+    cd lapack && \
     cp INSTALL/make.inc.gfortran make.inc && \
     make lapacklib blaslib && \
     mkdir -p /usr/local/lapack/lib && \
     cp *.a /usr/local/lapack/lib
 
-# Compile libxc-4.3.4
-RUN wget -c -O libxc-4.3.4.tar.gz http://www.tddft.org/programs/libxc/down.php?file=4.3.4/libxc-4.3.4.tar.gz && \
-    tar xf libxc-4.3.4.tar.gz && \
-    cd libxc-4.3.4 && \
+ARG LIBXC_VERSION
+RUN wget -c -O libxc.tar.gz https://gitlab.com/libxc/libxc/-/archive/4.3.4/libxc-4.3.4.tar.gz && \
+    mkdir -p libxc && \
+    tar xf libxc.tar.gz -C libxc --strip-components=1 && \
+    cd libxc && \
     autoreconf -i && \
     ./configure --prefix=/usr/local/libxc && \
     make && make install
 
-WORKDIR /
 RUN rm -rf /build
+WORKDIR /