docker.yml

name: Reusable workflow for Docker-based portability CI

on:
  workflow_call:
    inputs:
      targets_pre:
        default: build/make/Makefile
        type: string
      targets:
        default: build/make/Makefile
        type: string
      targets_optional:
        default: build/make/Makefile
        type: string
      tox_system_factors:
        description: 'Stringified JSON object listing tox system factors'
        type: string
        # 'tox -e update_docker_platforms' updates below
        default: >-
          [
           "ubuntu-xenial-toolchain-gcc_9",
           "ubuntu-bionic-gcc_8",
           "ubuntu-focal",
           "ubuntu-jammy",
           "ubuntu-lunar",
           "ubuntu-mantic",
           "ubuntu-noble",
           "debian-bullseye",
           "debian-bookworm",
           "debian-trixie",
           "debian-sid",
           "linuxmint-20.1",
           "linuxmint-20.2",
           "linuxmint-20.3",
           "linuxmint-21",
           "linuxmint-21.1",
           "linuxmint-21.2",
           "linuxmint-21.3",
           "fedora-30",
           "fedora-31",
           "fedora-32",
           "fedora-33",
           "fedora-34",
           "fedora-35",
           "fedora-36",
           "fedora-37",
           "fedora-38",
           "fedora-39",
           "fedora-40",
           "fedora-41",
           "centos-stream-9",
           "centos-stream-9-python3.12",
           "almalinux-8-python3.9",
           "almalinux-9-python3.11",
           "gentoo-python3.10",
           "gentoo-python3.11",
           "gentoo-python3.12",
           "archlinux-latest",
           "opensuse-15.5-gcc_11-python3.11",
           "opensuse-tumbleweed-python3.10",
           "opensuse-tumbleweed",
           "conda-forge-python3.11",
           "ubuntu-bionic-gcc_8-i386",
           "debian-bullseye-i386",
          ]
        # 'tox -e update_docker_platforms' updates above
      tox_packages_factors:
        description: 'Stringified JSON object listing tox packages factors'
        type: string
        default: >-
          ["minimal",
           "standard",
           ]
      extra_sage_packages:
        description: 'Extra Sage packages to install as system packages'
        type: string
        default: ""
      max_parallel:
        type: number
        default: 30
      free_disk_space:
        default: false
        type: boolean
      timeout:
        description: 'Elapsed time (seconds) at which to kill the build'
        default: 20000
        type: number
      logs_artifact:
        default: true
        type: boolean
      #
      # Publishing to GitHub Packages
      #
      docker_push_repository:
        required: false
        type: string
      #
      # Incremental builds
      #
      docker_targets:
        default: "with-system-packages configured with-targets-pre with-targets with-targets-optional"
        type: string
      incremental:
        default: false
        type: boolean
      from_docker_repository:
        required: false
        type: string
      from_docker_target:
        required: false
        type: string
      from_docker_tag:
        required: false
        default: "$BUILD_TAG"
        type: string
      #
      # For use in upstream CIs. sage_trac_* are now ignored and will be removed later.
      #
      upstream_artifact:
        required: false
        type: string
      sage_repo:
        required: false
        type: string
      sage_trac_git:
        required: false
        type: string
      sage_trac_ticket:
        required: false
        type: string
      sage_ref:
        required: false
        type: string

jobs:
  linux:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: ${{ inputs.max_parallel }}
      matrix:
        tox_system_factor:   ${{ fromJson(inputs.tox_system_factors) }}
        tox_packages_factor: ${{ fromJson(inputs.tox_packages_factors) }}
    env:
      TOX_ENV:            "docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}${{ inputs.incremental && '-incremental' || '' }}"
      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
      DOCKER_TARGETS:     ${{ inputs.docker_targets }}
      TARGETS_PRE:        ${{ inputs.targets_pre }}
      TARGETS:            ${{ inputs.targets }}
      TARGETS_OPTIONAL:   ${{ inputs.targets_optional }}
      FROM_DOCKER_REPOSITORY: ${{ inputs.from_docker_repository }}
      FROM_DOCKER_TARGET: ${{ inputs.from_docker_target }}
      FROM_DOCKER_TAG:    ${{ inputs.from_docker_tag }}
      EXTRA_CONFIGURE_ARGS: --enable-fat-binary
      EXTRA_SAGE_PACKAGES:  ${{ inputs.extra_sage_packages }}
    steps:
      - name: Maximize build disk space
        uses: easimon/maximize-build-space@v10
        with:
          # need space in /var for Docker images
          root-reserve-mb:      30000
          remove-dotnet:        true
          remove-android:       true
          remove-haskell:       true
          remove-codeql:        true
          remove-docker-images: true
        continue-on-error: true
        if: inputs.free_disk_space
      - name: Check out SageMath
        uses: actions/checkout@v4
        with:
          repository: ${{ inputs.sage_repo }}
          ref: ${{ inputs.sage_ref }}
          fetch-depth: 10000
      - name: Download upstream artifact
        uses: actions/download-artifact@v4
        with:
          path: upstream
          name: ${{ inputs.upstream_artifact }}
        if: inputs.upstream_artifact
      - name: Install test prerequisites
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
          sudo apt-get clean
          df -h
      - name: Update Sage packages from upstream artifact
        # Handle both the old and new location of write-dockerfile.sh,
        # because docker.yml is a reusable workflow.
        run: |
          (export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore; for a in build/bin/write-dockerfile.sh .ci/write-dockerfile.sh; do if [ -r $a ]; then echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - $a; fi; done; git diff)
        if: inputs.upstream_artifact

      - name: Try to login to ghcr.io
        if: inputs.docker_push_repository != ''
        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
        run: |
            TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
            if [ -z "$TOKEN" ]; then
              TOKEN="${{ secrets.GITHUB_TOKEN }}"
            fi
            if echo "$TOKEN" | docker login ghcr.io -u ${{ github.actor }} --password-stdin; then
              echo "DOCKER_PUSH_REPOSITORY=$(echo ${{ inputs.docker_push_repository }} | tr "[:upper:]" "[:lower:]")" >> $GITHUB_ENV
              echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV
            fi

      - name: Determine Docker tags to use
        run: |
            # This line needs to be run before the step "Merge CI fixes from sagemath/sage".
            DOCKER_TAG="$(git describe --dirty --always)"
            echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
            # From the docker documentation via .ci/update-env.sh:
            # "A tag name must be valid ASCII and may
            # contain lowercase and uppercase letters, digits, underscores, periods and
            # dashes. A tag name may not start with a period or a dash and may contain a
            # maximum of 128 characters."
            EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128`
            shopt -s extglob
            case "$GITHUB_REF_NAME" in
              +([0-9]).+([0-9])?(.+([0-9])) )
                EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";;
              +([0-9]).+([0-9])?(.+([0-9]))?(.)@(a|alpha|b|beta|rc)+([0-9]) )
                EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";;
            esac
            echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV

      - name: Merge CI fixes from sagemath/sage
        # This step needs to happen after the commit sha is put in DOCKER_TAG
        # so that multi-stage builds can work correctly.
        run: |
          .ci/merge-fixes.sh
        env:
          GH_TOKEN: ${{ github.token }}
          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}

      - name: Show disk space
        run: |
          df -h
        if: inputs.free_disk_space
      - name: Configure and build Sage distribution within a Docker container
        # The first command below is a self-destruct sequence,
        # which preempts the GitHub Actions 6-hour job cancellation.
        #
        # Using "docker exec", we enter the temporary containers used by
        # "docker build" and kill the "make" processes of the Sage distribution.
        #
        # The arcane "find" command is a replacement for "pkill make",
        # which we use because pkill is not installed in the "minimal" package
        # configuration on many platforms.
        #
        # The "sed" command strips away timestamps from "docker build" (buildkit)
        # such as "#25 1211.0" at the beginning of each line. The timestamps are
        # redundant because GH Actions provides timestamps for each line already.
        # Stripping the timestamps from the beginnings of lines also allows
        # GitHub Actions to recognize workflow commands such as ::error etc.
        #
        run: |
          (sleep ${{ inputs.timeout }}; for id in $(docker ps -q); do docker exec $id find /proc -maxdepth 2 -name cmdline -exec bash -c "grep -l [m][a][k][e] {} | cut -d/ -f3 | xargs --no-run-if-empty kill" \;; done) &
          set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg NUMPROC=9 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=5\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed -E --unbuffered "s/^#[0-9]+ [0-9]+[.][0-9]+ //;/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
      - name: Copy logs from the Docker image or build container
        run: |
          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
          cp -r .tox/$TOX_ENV/* "artifacts/$LOGS_ARTIFACT_NAME"
          rm -rf "artifacts/$LOGS_ARTIFACT_NAME"/{bin,lib,pyvenv.cfg}
        if: always()
      - name: Upload logs artifact
        uses: actions/upload-artifact@v4
        with:
          path: artifacts
          name: ${{ env.LOGS_ARTIFACT_NAME }}
        if: always() && inputs.logs_artifact
      - name: Print out logs for immediate inspection
        # and markup the output with GitHub Actions logging commands
        run: |
          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
        if: always()
      - name: List Docker images
        run: |
          if [ -n "$DOCKER_PUSH_REPOSITORY" -a -f .tox/$TOX_ENV/Dockertags.pushed ]; then
             set -- $(cat .tox/$TOX_ENV/Dockertags.pushed)
             case $# in
                 1) images="image";  one_image="the image";;
                 *) images="images"; one_image="one of the images";;
             esac
             echo "::notice title=Docker $images pushed::Pushed $images $*)"
             echo
             echo "To pull $one_image and enter the container, type:"
             echo
             for TAG in $*; do
                 echo "  \$ docker run -it $TAG bash"
             done
             echo
             echo "To use $one_image as the base for an incremental build, type:"
             echo
             TOX_ENV_SANS_INCREMENTAL=${TOX_ENV/-incremental/}
             DOCKER_IMAGE=${TOX_ENV_SANS_INCREMENTAL#docker-}
             for TAG in $*; do
                 echo -n "  \$"
                 if [ "$DOCKER_PUSH_REPOSITORY" != "ghcr.io/sagemath/sage/" ]; then
                     echo -n " FROM_DOCKER_REPOSITORY=$DOCKER_PUSH_REPOSITORY"
                 fi
                 eval DOCKER_TARGET=\${TAG#*$DOCKER_IMAGE-}
                 DOCKER_TARGET=${DOCKER_TARGET%:*}
                 if [ "$DOCKER_TARGET" != "with-targets" ]; then
                     echo -n " FROM_DOCKER_TARGET=$DOCKER_TARGET"
                 fi
                 echo " FROM_DOCKER_TAG=${TAG#*:} tox -e $TOX_ENV_SANS_INCREMENTAL-incremental"
             done
          elif [ -n "$DOCKER_PUSH_REPOSITORY" -a -f .tox/$TOX_ENV/Dockertags ]; then
             echo "Unable to push Docker images to $DOCKER_PUSH_REPOSITORY."
             echo "This is normal in a pull request to sagemath/sage or to another user's repository."
             echo
             echo "If you need Docker images, "
             echo " - either run this GitHub Actions workflow in your repository fork"
             echo " - or use the method described in https://doc.sagemath.org/html/en/developer/portability_testing.html#automatic-docker-based-build-testing-using-tox"
          else
             echo "No Docker images created."
          fi
        if: always() && ${{ inputs.docker_push_repository }}