build.yml

---
name: build

on:
  push:
    branches:
      - "**"
    tags:
      - "v*.*.*"
  pull_request:
  schedule:
    - cron: "0 10 * * *"  # everyday at 10am
  repository_dispatch:
    # Respond to rebuild requests. See: https://github.com/cisagov/action-apb/
    types: [apb]
  workflow_dispatch:
    inputs:
      remote-shell:
        description: "Debug with remote shell"
        required: true
        default: "false"
      image-tag:
        description: "Tag to apply to pushed images"
        required: true
        default: "dispatch"

env:
  BUILDX_CACHE_DIR: ~/.cache/buildx
  CURL_CACHE_DIR: ~/.cache/curl
  IMAGE_NAME: cisagov/code-gov-update
  PIP_CACHE_DIR: ~/.cache/pip
  # We have dropped support for the linux/arm/v6, linux/ppc64le, and linux/s390x
  # platforms until the run time for the build-push-all job can get below the six
  # hour limit that exists for GitHub Actions runners. This could either be through
  # some kind of optimization, when we matrix out the build so each platform runs
  # on its own runner, or if we start using our own runners that have a higher
  # time limit. Please see #150 for tracking.
  PLATFORMS: "linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8"
  PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit
  RUN_TMATE: ${{ secrets.RUN_TMATE }}

jobs:
  diagnostics:
    name: Run diagnostics
    runs-on: ubuntu-latest
    steps:
      # Note that a duplicate of this step must be added at the top of
      # each job.
      - id: harden-runner
        name: Harden the runner
        uses: step-security/harden-runner@v2
        with:
          egress-policy: audit
      - id: github-status
        name: Check GitHub status
        uses: crazy-max/ghaction-github-status@v3
      - id: dump-context
        name: Dump context
        uses: crazy-max/ghaction-dump-context@v2
  lint:
    # Checks out the source and runs pre-commit hooks.  Detects coding errors
    # and style deviations.
    name: Lint sources
    needs:
      - diagnostics
    runs-on: ubuntu-latest
    steps:
      - id: harden-runner
        name: Harden the runner
        uses: step-security/harden-runner@v2
        with:
          egress-policy: audit
      - id: setup-env
        uses: cisagov/setup-env-github-action@develop
      - uses: actions/checkout@v4
      - id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      # We need the Go version and Go cache location for the actions/cache step,
      # so the Go installation must happen before that.
      - id: setup-go
        uses: actions/setup-go@v4
        with:
          # There is no expectation for actual Go code so we disable caching as
          # it relies on the existence of a go.sum file.
          cache: false
          go-version: "1.20"
      - name: Lookup Go cache directory
        id: go-cache
        run: |
          echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT
      - uses: actions/cache@v3
        env:
          BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
            py${{ steps.setup-python.outputs.python-version }}-\
            go${{ steps.setup-go.outputs.go-version }}-\
            packer${{ steps.setup-env.outputs.packer-version }}-\
            tf${{ steps.setup-env.outputs.terraform-version }}-"
        with:
          # Note that the .terraform directory IS NOT included in the
          # cache because if we were caching, then we would need to use
          # the `-upgrade=true` option. This option blindly pulls down the
          # latest modules and providers instead of checking to see if an
          # update is required. That behavior defeats the benefits of caching.
          # so there is no point in doing it for the .terraform directory.
          path: |
            ${{ env.PIP_CACHE_DIR }}
            ${{ env.PRE_COMMIT_CACHE_DIR }}
            ${{ env.CURL_CACHE_DIR }}
            ${{ steps.go-cache.outputs.dir }}
          key: "${{ env.BASE_CACHE_KEY }}\
            ${{ hashFiles('**/requirements-test.txt') }}-\
            ${{ hashFiles('**/requirements.txt') }}-\
            ${{ hashFiles('**/.pre-commit-config.yaml') }}"
          restore-keys: |
            ${{ env.BASE_CACHE_KEY }}
      - name: Setup curl cache
        run: mkdir -p ${{ env.CURL_CACHE_DIR }}
      - name: Install Packer
        env:
          PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }}
        run: |
          PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip"
          curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \
            --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \
            --location \
            "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}"
          sudo unzip -d /opt/packer \
            ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}"
          sudo mv /usr/local/bin/packer /usr/local/bin/packer-default
          sudo ln -s /opt/packer/packer /usr/local/bin/packer
      - uses: hashicorp/setup-terraform@v2
        with:
          terraform_version: ${{ steps.setup-env.outputs.terraform-version }}
      - name: Install go-critic
        env:
          PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic
          PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }}
        run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
      - name: Install gosec
        env:
          PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec
          PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }}
        run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
      - name: Install shfmt
        env:
          PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt
          PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }}
        run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
      - name: Install staticcheck
        env:
          PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck
          PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }}
        run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
      - name: Install Terraform-docs
        env:
          PACKAGE_URL: github.com/terraform-docs/terraform-docs
          PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }}
        run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip setuptools wheel
          pip install --upgrade --requirement requirements-test.txt
      - name: Set up pre-commit hook environments
        run: pre-commit install-hooks
      - name: Run pre-commit on all files
        run: pre-commit run --all-files
      - name: Setup tmate debug session
        uses: mxschmitt/action-tmate@v3
        if: env.RUN_TMATE
  prepare:
    # Calculates and publishes outputs that are used by other jobs.
    #
    # Outputs:
    #   created:
    #     The current date-time in RFC3339 format.
    #   repometa:
    #     The json metadata describing this repository.
    #   source_version:
    #     The source version as reported by the `bump_version.sh show` command.
    #   tags:
    #     A comma separated list of Docker tags to be applied to the images on
    #     Docker Hub.  The tags will vary depending on:
    #     - The event that triggered the build.
    #     - The branch the build is based upon.
    #     - The git tag the build is based upon.
    #
    #     When a build is based on a git tag of the form `v*.*.*` the image will
    #     be tagged on Docker Hub with multiple levels of version specificity.
    #     For example, a git tag of `v1.2.3+a` will generate Docker tags of
    #     `:1.2.3_a`, `:1.2.3`, `:1.2`, `:1`, and `:latest`.
    #
    #     Builds targeting the default branch will be tagged with `:edge`.
    #
    #     Builds from other branches will be tagged with the branch name. Solidi
    #     (`/` characters - commonly known as slashes) in branch names are
    #     replaced with hyphen-minuses (`-` characters) in the Docker tag.  For
    #     more information about the solidus see these links:
    #       * https://www.compart.com/en/unicode/U+002F
    #       * https://en.wikipedia.org/wiki/Slash_(punctuation)#Encoding
    #
    #     Builds triggered by a push event are tagged with a short hash in the
    #     form: sha-12345678
    #
    #     Builds triggered by a pull request are tagged with the pull request
    #     number in the form pr-123.
    #
    #     Builds triggered using the GitHub GUI (workflow_dispatch) are tagged
    #     with the value specified by the user.
    #
    #     Scheduled builds are tagged with `:nightly`.
    name: Prepare build variables
    needs:
      - diagnostics
    outputs:
      created: ${{ steps.prep.outputs.created }}
      repometa: ${{ steps.repo.outputs.result }}
      source_version: ${{ steps.prep.outputs.source_version }}
      tags: ${{ steps.prep.outputs.tags }}
    runs-on: ubuntu-latest
    steps:
      - id: harden-runner
        name: Harden the runner
        uses: step-security/harden-runner@v2
        with:
          egress-policy: audit
      - uses: actions/checkout@v4
      - name: Gather repository metadata
        id: repo
        uses: actions/github-script@v7
        with:
          script: |
            const repo = await github.rest.repos.get(context.repo)
            return repo.data
      - name: Calculate output values
        id: prep
        run: |
          VERSION=noop
          SEMVER="^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?$"
          if [ "${{ github.event_name }}" = "schedule" ]; then
            VERSION=nightly
          elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            VERSION=${{ github.event.inputs.image-tag }}
          elif [[ $GITHUB_REF == refs/tags/* ]]; then
            VERSION=${GITHUB_REF#refs/tags/}
          elif [[ $GITHUB_REF == refs/heads/* ]]; then
            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
            if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ];
              then
              VERSION=edge
            fi
          elif [[ $GITHUB_REF == refs/pull/* ]]; then
            VERSION=pr-${{ github.event.number }}
          fi
          if [[ $VERSION =~ $SEMVER ]]; then
            VERSION_NO_V=${VERSION#v}
            MAJOR="${BASH_REMATCH[1]}"
            MINOR="${BASH_REMATCH[2]}"
            PATCH="${BASH_REMATCH[3]}"
            TAGS="${IMAGE_NAME}:${VERSION_NO_V//+/_},${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH},${IMAGE_NAME}:${MAJOR}.${MINOR},${IMAGE_NAME}:${MAJOR},${IMAGE_NAME}:latest"
          else
            TAGS="${IMAGE_NAME}:${VERSION}"
          fi
          if [ "${{ github.event_name }}" = "push" ]; then
            TAGS="${TAGS},${IMAGE_NAME}:sha-${GITHUB_SHA::8}"
          fi
          for i in ${TAGS//,/ }
          do
            TAGS="${TAGS},ghcr.io/${i}"
          done
          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
          echo "source_version=$(./bump_version.sh show)" >> $GITHUB_OUTPUT
          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
          echo tags=${TAGS}
      - name: Setup tmate debug session
        uses: mxschmitt/action-tmate@v3
        if: github.event.inputs.remote-shell == 'true' || env.RUN_TMATE
  build:
    # Builds a single test image for the native platform.  This image is saved
    # as an artifact and loaded by the test job.
    name: Build test image
    needs:
      - diagnostics
      - prepare
    runs-on: ubuntu-latest
    steps:
      - id: harden-runner
        name: Harden the runner
        uses: step-security/harden-runner@v2
        with:
          egress-policy: audit
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Cache Docker layers
        uses: actions/cache@v3
        env:
          BASE_CACHE_KEY: buildx-${{ runner.os }}-
        with:
          path: ${{ env.BUILDX_CACHE_DIR }}
          key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }}
          restore-keys: |
            ${{ env.BASE_CACHE_KEY }}
      - name: Create dist directory
        run: mkdir -p dist
      - name: Build image
        id: docker_build
        uses: docker/build-push-action@v5
        with:
          build-args: |
            VERSION=${{ needs.prepare.outputs.source_version }}
          cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }}
          cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }}
          context: .
          file: ./Dockerfile
          outputs: type=docker,dest=dist/image.tar
          # Uncomment the following option if you are building an image for use
          # on Google Cloud Run or AWS Lambda. The current default image output
          # is unable to run on either. Please see the following issue for more
          # information: https://github.com/docker/buildx/issues/1533
          # provenance: false
          tags: ${{ env.IMAGE_NAME }}:latest  # not to be pushed
          # For a list of pre-defined annotation keys and value types see:
          # https://github.com/opencontainers/image-spec/blob/master/annotations.md
          labels: "\
            org.opencontainers.image.created=${{
              needs.prepare.outputs.created }}

            org.opencontainers.image.description=${{
              fromJson(needs.prepare.outputs.repometa).description }}

            org.opencontainers.image.licenses=${{
              fromJson(needs.prepare.outputs.repometa).license.spdx_id }}

            org.opencontainers.image.revision=${{ github.sha }}

            org.opencontainers.image.source=${{
              fromJson(needs.prepare.outputs.repometa).clone_url }}

            org.opencontainers.image.title=${{
              fromJson(needs.prepare.outputs.repometa).name }}

            org.opencontainers.image.url=${{
              fromJson(needs.prepare.outputs.repometa).html_url }}

            org.opencontainers.image.version=${{
              needs.prepare.outputs.source_version }}"
      - name: Compress image
        run: gzip dist/image.tar
      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: dist
          path: dist
      - name: Setup tmate debug session
        uses: mxschmitt/action-tmate@v3
        if: env.RUN_TMATE
  test:
    # Executes tests on the single-platform image created in the "build" job.
    name: Test image
    needs:
      - diagnostics
      - build
    runs-on: ubuntu-latest
    steps:
      - id: harden-runner
        name: Harden the runner
        uses: step-security/harden-runner@v2
        with:
          egress-policy: audit
      - uses: actions/checkout@v4
      - id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      - name: Cache testing environments
        uses: actions/cache@v3
        env:
          BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
            py${{ steps.setup-python.outputs.python-version }}-"
        with:
          path: ${{ env.PIP_CACHE_DIR }}
          key: "${{ env.BASE_CACHE_KEY }}\
            ${{ hashFiles('**/requirements-test.txt') }}-\
            ${{ hashFiles('**/requirements.txt') }}"
          restore-keys: |
            ${{ env.BASE_CACHE_KEY }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip setuptools wheel
          pip install --upgrade --requirement requirements-test.txt
      - name: Download docker image artifact
        uses: actions/download-artifact@v3
        with:
          name: dist
          path: dist
      - name: Load docker image
        run: docker load < dist/image.tar.gz
      - name: Run tests
        env:
          RELEASE_TAG: ${{ github.event.release.tag_name }}
        run: pytest --runslow
      - name: Setup tmate debug session
        uses: mxschmitt/action-tmate@v3
        if: env.RUN_TMATE
  build-push-all:
    # Builds the final set of images for each of the platforms listed in
    # PLATFORMS environment variable.  These images are tagged with the Docker
    # tags calculated in the "prepare" job and pushed to Docker Hub and the
    # GitHub Container Registry.  The contents of README.md are pushed as the
    # image's description to Docker Hub.  This job is skipped when the
    # triggering event is a pull request.
    if: github.event_name != 'pull_request'
    name: Build and push all platforms
    needs:
      - diagnostics
      - lint
      - prepare
      - test
    # When Dependabot creates a PR it requires this permission in
    # order to push Docker images to ghcr.io.
    permissions:
      packages: write
    runs-on: ubuntu-latest
    steps:
      - id: harden-runner
        name: Harden the runner
        uses: step-security/harden-runner@v2
        with:
          egress-policy: audit
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Cache Docker layers
        uses: actions/cache@v3
        env:
          BASE_CACHE_KEY: buildx-${{ runner.os }}-
        with:
          path: ${{ env.BUILDX_CACHE_DIR }}
          key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }}
          restore-keys: |
            ${{ env.BASE_CACHE_KEY }}
      - name: Create cross-platform support Dockerfile-x
        run: ./buildx-dockerfile.sh
      - name: Build and push platform images to registries
        id: docker_build
        uses: docker/build-push-action@v5
        with:
          build-args: |
            VERSION=${{ needs.prepare.outputs.source_version }}
          cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }}
          cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }}
          context: .
          file: ./Dockerfile-x
          platforms: ${{ env.PLATFORMS }}
          # Uncomment the following option if you are building an image for use
          # on Google Cloud Run or AWS Lambda. The current default image output
          # is unable to run on either. Please see the following issue for more
          # information: https://github.com/docker/buildx/issues/1533
          # provenance: false
          push: true
          tags: ${{ needs.prepare.outputs.tags }}
          # For a list of pre-defined annotation keys and value types see:
          # https://github.com/opencontainers/image-spec/blob/master/annotations.md
          labels: "\
            org.opencontainers.image.created=${{
              needs.prepare.outputs.created }}

            org.opencontainers.image.description=${{
              fromJson(needs.prepare.outputs.repometa).description }}

            org.opencontainers.image.licenses=${{
              fromJson(needs.prepare.outputs.repometa).license.spdx_id }}

            org.opencontainers.image.revision=${{ github.sha }}

            org.opencontainers.image.source=${{
              fromJson(needs.prepare.outputs.repometa).clone_url }}

            org.opencontainers.image.title=${{
              fromJson(needs.prepare.outputs.repometa).name }}

            org.opencontainers.image.url=${{
              fromJson(needs.prepare.outputs.repometa).html_url }}

            org.opencontainers.image.version=${{
              needs.prepare.outputs.source_version }}"
      - name: Publish README.md to Docker Hub
        env:
          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
        run: ./push_readme.sh
      - name: Setup tmate debug session
        uses: mxschmitt/action-tmate@v3
        if: env.RUN_TMATE